@milaboratories/pframes-rs-node 1.0.106 → 1.0.109

package/export/addon-def.ts

@@ -14,9 +14,6 @@ import type {
 } from '@milaboratories/pl-model-common';
 import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
 
-export type FrameId = string & { __brand: 'FrameId' };
-export type TableId = string & { __brand: 'TableId' };
-
 export interface NodeFrameSymbol extends Symbol {}
 
 export interface NodeTableSymbol extends Symbol {}
@@ -57,16 +54,16 @@ interface NodeFrame {
   pFrameGetColumnSpec: (
     pFrame: NodeFrameSymbol,
     columnId: PObjectId
-  ) => Promise<PColumnSpec>;
+  ) => Promise<PColumnSpec | null>;
   pFrameListColumns: (pFrame: NodeFrameSymbol) => Promise<PColumnInfo[]>;
   pFrameCreateTable: (
     pFrame: NodeFrameSymbol,
-    id: TableId,
+    id: PFrameInternal.PTableId,
     request: PFrameInternal.CreateTableRequestV4
   ) => NodeTableSymbol;
   pFrameGetUniqueValues: (
     pFrame: NodeFrameSymbol,
-    id: TableId,
+    id: PFrameInternal.PTableId,
     request: UniqueValuesRequest,
     signal: AbortSignal | undefined
   ) => Promise<UniqueValuesResponse>;
@@ -88,19 +85,19 @@ interface NodeTable {
   ) => Promise<PTableShape>;
   pTableGetData: (
     boxed: NodeTableSymbol,
-    id: TableId,
+    id: PFrameInternal.PTableId,
     columnIndices: number[],
     range: TableRange | undefined,
     signal: AbortSignal | undefined
   ) => Promise<PTableVector[]>;
   pTableFilter: (
     boxed: NodeTableSymbol,
-    id: TableId,
+    id: PFrameInternal.PTableId,
     request: PTableRecordFilter[]
   ) => NodeTableSymbol;
   pTableSort: (
     boxed: NodeTableSymbol,
-    id: TableId,
+    id: PFrameInternal.PTableId,
     request: PTableSorting[]
   ) => NodeTableSymbol;
   pTableDispose: (boxed: NodeTableSymbol) => void;

package/export/export.ts

@@ -1,10 +1,10 @@
 import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
 import { PFrame, pprofDump } from './wrapper';
 
-export const PFrameFactory: PFrameInternal.PFrameFactoryV3 = {
+export const PFrameFactory: PFrameInternal.PFrameFactoryV4 = {
   createPFrame: (
-    options: PFrameInternal.PFrameOptions
-  ): PFrameInternal.PFrameV12 => {
+    options: PFrameInternal.PFrameOptionsV2
+  ): PFrameInternal.PFrameV13 => {
     return new PFrame(options);
   },
   pprofDump: async (): Promise<Uint8Array> => {

package/export/tests/pframes.test.ts

@@ -0,0 +1,537 @@
+// Node.js core modules
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import { tmpdir } from 'node:os';
+import type { Buffer } from 'node:buffer';
+
+// Testing framework
+import { describe, test } from 'vitest';
+
+// Project types and modules
+import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import type {
+  PObjectId,
+  PTableSorting,
+  PTableVector,
+  UniqueValuesResponse
+} from '@milaboratories/pl-model-common';
+import { PFrameFactory } from '..';
+import { ulid } from 'ulid';
+
+// ============================================================================
+// Test Utility Functions
+// ============================================================================
+
+/**
+ * Get the path to a test case directory
+ */
+function getTestCaseDataPath(testCase: string): string {
+  const currentDirectoryPath = path.relative('.', __dirname);
+  const testDataPath = path.join(currentDirectoryPath, 'test_data');
+  return path.join(testDataPath, testCase);
+}
+
+/**
+ * Get the full path to a test file
+ */
+function getTestFilePath(testCase: string, fileName: string): string {
+  return path.join(getTestCaseDataPath(testCase), fileName);
+}
+
+/**
+ * Read a test file as Buffer
+ */
+function readTestFile(testCase: string, fileName: string): Buffer {
+  return fs.readFileSync(getTestFilePath(testCase, fileName));
+}
+
+/**
+ * Read and parse a JSON test file
+ */
+function readJsonTestFile(testCase: string, fileName: string): any {
+  return JSON.parse(readTestFile(testCase, fileName).toString());
+}
+
+/**
+ * Create a standard data source for tests that reads from test files
+ */
+function createTestDataSource(
+  testCase: string
+): PFrameInternal.PFrameDataSourceV2 {
+  return {
+    preloadBlob: function (
+      blobIds: PFrameInternal.PFrameBlobId[]
+    ): Promise<void> {
+      return new Promise((resolve) => {
+        if (new Set(blobIds).size !== blobIds.length) {
+          throw new Error('preload requires array of unique values');
+        }
+        resolve();
+      });
+    },
+    resolveBlobContent: function (
+      blobId: PFrameInternal.PFrameBlobId
+    ): Promise<Uint8Array> {
+      return new Promise((resolve) => {
+        resolve(readTestFile(testCase, blobId));
+      });
+    }
+  };
+}
+
+/**
+ * Create a failing data source for error testing
+ */
+function createFailingDataSource(): PFrameInternal.PFrameDataSourceV2 {
+  return {
+    preloadBlob: (): Promise<void> => Promise.resolve(),
+    resolveBlobContent: (): Promise<Uint8Array> =>
+      Promise.reject(new Error('fail'))
+  };
+}
+
+/**
+ * Read a complete PFrame from test data directory
+ * Automatically loads all .spec files and their corresponding .datainfo files
+ */
+async function readPFrame(testCase: string): Promise<PFrameInternal.PFrameV13> {
+  const dataSource = createTestDataSource(testCase);
+  const pframe = PFrameFactory.createPFrame({
+    frameId: ulid() as PFrameInternal.PFrameId,
+    spillPath: tmpdir() /*, logger: console.log */
+  });
+  pframe.setDataSource(dataSource);
+
+  const testCasePath = getTestCaseDataPath(testCase);
+  const files = fs.readdirSync(testCasePath);
+
+  for (const file of files) {
+    if (file.endsWith('.spec')) {
+      const columnId = file.replace('.spec', '') as PObjectId;
+      const columnSpec = readJsonTestFile(testCase, file);
+      pframe.addColumnSpec(columnId, columnSpec);
+
+      const dataInfoFile = file.replace('.spec', '.datainfo');
+      if (files.includes(dataInfoFile)) {
+        const dataInfo = readJsonTestFile(testCase, dataInfoFile);
+        await pframe.setColumnData(columnId, dataInfo);
+      }
+    }
+  }
+
+  return pframe;
+}
+
+// ============================================================================
+// Test Suites
+// ============================================================================
+
+describe('PFrame Column Operations', () => {
+  test.concurrent('find_columns_request', async ({ expect }) => {
+    const testCase = 'find_columns_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.addColumnSpec(columnId, columnSpec);
+
+    const actualResponse = await pframe.findColumns(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('get_column_spec_request', async ({ expect }) => {
+    const testCase = 'get_column_spec_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.addColumnSpec(columnId, columnSpec);
+
+    const actualResponse = await pframe.getColumnSpec(columnId);
+    expect(actualResponse).toEqual(expectedResponse);
+
+    const unknownColumnId = 'unknown_column1' as PObjectId;
+    const unknownColumnSpec = await pframe.getColumnSpec(unknownColumnId);
+    expect(unknownColumnSpec).toBeNull();
+
+    const columnList = await pframe.listColumns();
+    expect(columnList).toEqual([
+      { columnId: columnId, spec: columnSpec, hasData: false }
+    ]);
+  });
+
+  test.concurrent('delete_column_request', async ({ expect }) => {
+    const testCase = 'delete_column_request';
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+
+    const actualResponse = await pframe.deleteColumn(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('set_column_data_request', async ({ expect }) => {
+    const testCase = 'set_column_data_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createTestDataSource(testCase);
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    const columnList = await pframe.listColumns();
+    expect(columnList).toEqual([
+      { columnId: columnId, spec: columnSpec, hasData: true }
+    ]);
+  });
+
+  test.concurrent('unique_values_request', async ({ expect }) => {
+    const testCase = 'unique_values_request';
+
+    const columnId = 'column' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createTestDataSource(testCase);
+    const request = readJsonTestFile(testCase, 'request.json');
+
+    const expectedResponse: UniqueValuesResponse = {
+      values: {
+        type: 'Long',
+        data: new BigInt64Array([BigInt(10), BigInt(11), BigInt(12)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      overflow: false
+    };
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    const actualResponse = await pframe.getUniqueValues(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+});
+
+describe('PFrame Table Creation', () => {
+  // Expected responses for table creation tests
+  const createTableExpectedResponses = {
+    request1: [
+      {
+        type: 'Int' as const,
+        data: new Int32Array([10]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Float' as const,
+        data: new Float32Array([10.0]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request2: [
+      {
+        type: 'Long' as const,
+        data: new BigInt64Array([BigInt(10)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Double' as const,
+        data: new Float64Array([10.0]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request3: [
+      {
+        type: 'String' as const,
+        data: ['10'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Bytes' as const,
+        data: [new Uint8Array([49, 48])],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request4: [
+      {
+        type: 'String' as const,
+        data: ['10'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Float' as const,
+        data: new Float32Array([NaN]),
+        isNA: new Uint8Array([128]),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[]
+  };
+
+  test.concurrent('create_table_request_1', async ({ expect }) => {
+    const testCase = 'create_table_request_1';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request1;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_2', async ({ expect }) => {
+    const testCase = 'create_table_request_2';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request2;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_3', async ({ expect }) => {
+    const testCase = 'create_table_request_3';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request3;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_4', async ({ expect }) => {
+    const testCase = 'create_table_request_4';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request4;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_fail', async ({ expect }) => {
+    const testCase = 'create_table_request_fail';
+
+    const columnId = 'column' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createFailingDataSource();
+    const request = readJsonTestFile(testCase, 'request.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    await expect(table.getShape()).rejects.toThrow();
+  });
+});
+
+describe('PFrame Table Operations', () => {
+  test.concurrent('sort_table_request', async ({ expect }) => {
+    const testCase = 'sort_table_request';
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    const sorting: PTableSorting[] = readJsonTestFile(testCase, 'sorting.json');
+
+    // Expected response for full sorted data
+    let expectedFullResponse: PTableVector[] = [
+      {
+        type: 'Long',
+        data: new BigInt64Array([BigInt(0), BigInt(0), BigInt(1), BigInt(2)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'String',
+        data: ['FloatOne', 'FloatTwo', null, null],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([48])
+      },
+      {
+        type: 'String',
+        data: [null, null, 'IntOne', 'IntTwo'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([192])
+      },
+      {
+        type: 'Float',
+        data: new Float32Array([1.0, 2.0, NaN, NaN]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([48])
+      }
+    ];
+
+    // Expected response for range query
+    const expectedRangeResponse: PTableVector[] = [
+      {
+        type: 'Float',
+        data: new Float32Array([2.0, NaN]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([64])
+      },
+      {
+        type: 'String',
+        data: [null, 'IntOne'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([128])
+      },
+      {
+        type: 'String',
+        data: ['FloatTwo', null],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array([64])
+      },
+      {
+        type: 'Long',
+        data: new BigInt64Array([BigInt(0), BigInt(1)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ];
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    using sortedTable = table.sort(ulid() as PFrameInternal.PTableId, sorting);
+    const footprint = await sortedTable.getFootprint();
+    expect(footprint).toBeGreaterThan(0);
+
+    // Test full sorted data
+    const columnIndices = sortedTable.getColumnIndices(
+      Array.from(sorting, (entry) => entry.column)
+    );
+    let actualResponse = await sortedTable.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedFullResponse);
+
+    // Test range query with reordered columns
+    actualResponse = await sortedTable.getData(
+      [columnIndices[3], columnIndices[2], columnIndices[1], columnIndices[0]],
+      { range: { offset: 1, length: 2 } }
+    );
+    expect(actualResponse).toEqual(expectedRangeResponse);
+  });
+});
+
+describe('PFrame Complex Operations', () => {
+  test.concurrent('complex_join', async ({ expect }) => {
+    const testCase = 'complex_join';
+    using pframe = await readPFrame(testCase);
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const shape = await table.getShape();
+    expect(shape).toEqual({
+      columns: 6,
+      rows: 7124
+    });
+
+    // Test profiling functionality (platform-dependent)
+    if (process.platform === 'win32') {
+      expect(PFrameFactory.pprofDump()).rejects.toThrow();
+    } else {
+      const profile = await PFrameFactory.pprofDump();
+      const profilePath = path.join(tmpdir(), `profile_${Date.now()}.pb.gz`);
+      fs.writeFileSync(profilePath, profile);
+      console.log(`Profile saved to ${profilePath}`);
+    }
+
+    // Test out of bounds range
+    const result = await table.getData([0], {
+      range: { offset: 10000, length: 20000 }
+    });
+    expect(result).toMatchObject([{ data: [] }]);
+
+    const result2 = await table.getData([0], {
+      range: { offset: 7123, length: 2 }
+    });
+    expect(result2[0].data.length).toBe(1);
+  });
+});

package/export/tests/setup.ts

@@ -0,0 +1,38 @@
+import { expect } from 'vitest';
+import { Tester, TesterContext } from '@vitest/expect';
+
+function areTypedArraysEqual(
+  this: TesterContext,
+  lhs: unknown,
+  rhs: unknown,
+  customTesters: Array<Tester>
+): boolean | undefined {
+  const isTypedArray = (value: unknown): value is Object & unknown[] =>
+    ArrayBuffer.isView(value);
+
+  const lhsIsTypedArray = isTypedArray(lhs);
+  const rhsIsTypedArray = isTypedArray(rhs);
+
+  if (lhsIsTypedArray && rhsIsTypedArray) {
+    if (!this.equals(lhs.constructor.name, rhs.constructor.name, customTesters))
+      return false;
+    if (
+      !this.equals(
+        lhs.constructor.length,
+        rhs.constructor.length,
+        customTesters
+      )
+    )
+      return false;
+    for (let j = 0; j < lhs.constructor.length; ++j) {
+      if (!this.equals(lhs[j], rhs[j], customTesters)) return false;
+    }
+    return true;
+  } else if (lhsIsTypedArray === rhsIsTypedArray) {
+    return undefined;
+  } else {
+    return false;
+  }
+}
+
+expect.addEqualityTesters([areTypedArraysEqual]);