@milaboratories/pframes-rs-node 1.0.108 → 1.1.0

package/export/tests/pframes.test.ts

@@ -0,0 +1,736 @@
+// Node.js core modules
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import { tmpdir } from 'node:os';
+import type { Buffer } from 'node:buffer';
+
+// Testing framework
+import { describe, test } from 'vitest';
+
+// Project types and modules
+import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import type {
+  AxisId,
+  PColumnIdAndSpec,
+  PObjectId,
+  PTableSorting,
+  PTableVector,
+  QueryData,
+  UniqueValuesResponse
+} from '@milaboratories/pl-model-common';
+import { PFrameFactory } from '..';
+import { ulid } from 'ulid';
+
+// ============================================================================
+// Test Utility Functions
+// ============================================================================
+
+/**
+ * Get the path to a test case directory
+ */
+function getTestCaseDataPath(testCase: string): string {
+  const currentDirectoryPath = path.relative('.', __dirname);
+  const testDataPath = path.join(currentDirectoryPath, 'test_data');
+  return path.join(testDataPath, testCase);
+}
+
+/**
+ * Get the full path to a test file
+ */
+function getTestFilePath(testCase: string, fileName: string): string {
+  return path.join(getTestCaseDataPath(testCase), fileName);
+}
+
+/**
+ * Read a test file as Buffer
+ */
+function readTestFile(testCase: string, fileName: string): Buffer {
+  return fs.readFileSync(getTestFilePath(testCase, fileName));
+}
+
+/**
+ * Read and parse a JSON test file
+ */
+function readJsonTestFile(testCase: string, fileName: string): any {
+  return JSON.parse(readTestFile(testCase, fileName).toString());
+}
+
+/**
+ * Create a standard data source for tests that reads from test files
+ */
+function createTestDataSource(
+  testCase: string
+): PFrameInternal.PFrameDataSourceV2 {
+  return {
+    preloadBlob: function (
+      blobIds: PFrameInternal.PFrameBlobId[]
+    ): Promise<void> {
+      return new Promise((resolve) => {
+        if (new Set(blobIds).size !== blobIds.length) {
+          throw new Error('preload requires array of unique values');
+        }
+        resolve();
+      });
+    },
+    resolveBlobContent: function (
+      blobId: PFrameInternal.PFrameBlobId
+    ): Promise<Uint8Array> {
+      return new Promise((resolve) => {
+        resolve(readTestFile(testCase, blobId));
+      });
+    }
+  };
+}
+
+/**
+ * Create a failing data source for error testing
+ */
+function createFailingDataSource(): PFrameInternal.PFrameDataSourceV2 {
+  return {
+    preloadBlob: (): Promise<void> => Promise.resolve(),
+    resolveBlobContent: (): Promise<Uint8Array> =>
+      Promise.reject(new Error('fail'))
+  };
+}
+
+/**
+ * Read a complete PFrame from test data directory
+ * Automatically loads all .spec files and their corresponding .datainfo files
+ */
+async function readPFrame(testCase: string): Promise<PFrameInternal.PFrameV13> {
+  const dataSource = createTestDataSource(testCase);
+  const pframe = PFrameFactory.createPFrame({
+    frameId: ulid() as PFrameInternal.PFrameId,
+    spillPath: tmpdir() /*, logger: console.log */
+  });
+  pframe.setDataSource(dataSource);
+
+  const testCasePath = getTestCaseDataPath(testCase);
+  const files = fs.readdirSync(testCasePath);
+
+  for (const file of files) {
+    if (file.endsWith('.spec')) {
+      const columnId = file.replace('.spec', '') as PObjectId;
+      const columnSpec = readJsonTestFile(testCase, file);
+      pframe.addColumnSpec(columnId, columnSpec);
+
+      const dataInfoFile = file.replace('.spec', '.datainfo');
+      if (files.includes(dataInfoFile)) {
+        const dataInfo = readJsonTestFile(testCase, dataInfoFile);
+        await pframe.setColumnData(columnId, dataInfo);
+      }
+    }
+  }
+
+  return pframe;
+}
+
+// ============================================================================
+// Test Suites
+// ============================================================================
+
+describe('PFrame Column Operations', () => {
+  test.concurrent('find_columns_request', async ({ expect }) => {
+    const testCase = 'find_columns_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.addColumnSpec(columnId, columnSpec);
+
+    const actualResponse = await pframe.findColumns(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('get_column_spec_request', async ({ expect }) => {
+    const testCase = 'get_column_spec_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.addColumnSpec(columnId, columnSpec);
+
+    const actualResponse = await pframe.getColumnSpec(columnId);
+    expect(actualResponse).toEqual(expectedResponse);
+
+    const unknownColumnId = 'unknown_column1' as PObjectId;
+    const unknownColumnSpec = await pframe.getColumnSpec(unknownColumnId);
+    expect(unknownColumnSpec).toBeNull();
+
+    const columnList = await pframe.listColumns();
+    expect(columnList).toEqual([
+      { columnId: columnId, spec: columnSpec, hasData: false }
+    ]);
+  });
+
+  test.concurrent('delete_column_request', async ({ expect }) => {
+    const testCase = 'delete_column_request';
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = readJsonTestFile(testCase, 'response.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+
+    const actualResponse = await pframe.deleteColumn(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('set_column_data_request', async ({ expect }) => {
+    const testCase = 'set_column_data_request';
+
+    const columnId = 'column1' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createTestDataSource(testCase);
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    const columnList = await pframe.listColumns();
+    expect(columnList).toEqual([
+      { columnId: columnId, spec: columnSpec, hasData: true }
+    ]);
+  });
+
+  test.concurrent('unique_values_request', async ({ expect }) => {
+    const testCase = 'unique_values_request';
+
+    const columnId = 'column' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createTestDataSource(testCase);
+    const request = readJsonTestFile(testCase, 'request.json');
+
+    const expectedResponse: UniqueValuesResponse = {
+      values: {
+        type: 'Long',
+        data: new BigInt64Array([BigInt(10), BigInt(11), BigInt(12)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      overflow: false
+    };
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    const actualResponse = await pframe.getUniqueValues(request);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+});
+
+describe('PFrame Table Creation', () => {
+  // Expected responses for table creation tests
+  const createTableExpectedResponses = {
+    request1: [
+      {
+        type: 'Int' as const,
+        data: new Int32Array([10]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Float' as const,
+        data: new Float32Array([10.0]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request2: [
+      {
+        type: 'Long' as const,
+        data: new BigInt64Array([BigInt(10)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Double' as const,
+        data: new Float64Array([10.0]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request3: [
+      {
+        type: 'String' as const,
+        data: ['10'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Bytes' as const,
+        data: [new Uint8Array([49, 48])],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[],
+    request4: [
+      {
+        type: 'String' as const,
+        data: ['10'],
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Float' as const,
+        data: new Float32Array([NaN]),
+        isNA: new Uint8Array([128]),
+        absent: new Uint8Array()
+      }
+    ] as PTableVector[]
+  };
+
+  test.concurrent('create_table_request_1', async ({ expect }) => {
+    const testCase = 'create_table_request_1';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request1;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_2', async ({ expect }) => {
+    const testCase = 'create_table_request_2';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request2;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_3', async ({ expect }) => {
+    const testCase = 'create_table_request_3';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request3;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_4', async ({ expect }) => {
+    const testCase = 'create_table_request_4';
+    const request = readJsonTestFile(testCase, 'request.json');
+    const expectedResponse = createTableExpectedResponses.request4;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_request_fail', async ({ expect }) => {
+    const testCase = 'create_table_request_fail';
+
+    const columnId = 'column' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createFailingDataSource();
+    const request = readJsonTestFile(testCase, 'request.json');
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    await expect(table.getShape()).rejects.toThrow();
+  });
+
+  // QueryData requests for createTableByDataQuery tests
+  const createTableByDataQueryRequests = {
+    request1: {
+      tableSpec: {
+        axes: [{ name: 'axis', type: 'Int' }] as AxisId[],
+        columns: [
+          {
+            columnId: 'column',
+            spec: {
+              kind: 'PColumn',
+              axesSpec: [{ name: 'axis', type: 'Int' }],
+              name: 'column',
+              valueType: 'Float'
+            }
+          }
+        ] as PColumnIdAndSpec[]
+      },
+      dataQuery: { type: 'column', columnId: 'column' } as QueryData
+    },
+    request2: {
+      tableSpec: {
+        axes: [{ name: 'axis', type: 'Long' }] as AxisId[],
+        columns: [
+          {
+            columnId: 'column',
+            spec: {
+              kind: 'PColumn',
+              axesSpec: [{ name: 'axis', type: 'Long' }],
+              name: 'column',
+              valueType: 'Double'
+            }
+          }
+        ] as PColumnIdAndSpec[]
+      },
+      dataQuery: { type: 'column', columnId: 'column' } as QueryData
+    },
+    request3: {
+      tableSpec: {
+        axes: [{ name: 'axis', type: 'String' }] as AxisId[],
+        columns: [
+          {
+            columnId: 'column',
+            spec: {
+              kind: 'PColumn',
+              axesSpec: [{ name: 'axis', type: 'String' }],
+              name: 'column',
+              valueType: 'Bytes'
+            }
+          }
+        ] as PColumnIdAndSpec[]
+      },
+      dataQuery: { type: 'column', columnId: 'column' } as QueryData
+    },
+    request4: {
+      tableSpec: {
+        axes: [{ name: 'axis', type: 'String' }] as AxisId[],
+        columns: [
+          {
+            columnId: 'column',
+            spec: {
+              kind: 'PColumn',
+              axesSpec: [{ name: 'axis', type: 'String' }],
+              name: 'column',
+              valueType: 'Float'
+            }
+          }
+        ] as PColumnIdAndSpec[]
+      },
+      dataQuery: { type: 'column', columnId: 'column' } as QueryData
+    }
+  };
+
+  test.concurrent('create_table_by_data_query_1', async ({ expect }) => {
+    const testCase = 'create_table_request_1';
+    const expectedResponse = createTableExpectedResponses.request1;
+    const queryDataRequest = createTableByDataQueryRequests.request1;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTableByDataQuery(
+      ulid() as PFrameInternal.PTableId,
+      queryDataRequest
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_by_data_query_2', async ({ expect }) => {
+    const testCase = 'create_table_request_2';
+    const expectedResponse = createTableExpectedResponses.request2;
+    const queryDataRequest = createTableByDataQueryRequests.request2;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTableByDataQuery(
+      ulid() as PFrameInternal.PTableId,
+      queryDataRequest
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_by_data_query_3', async ({ expect }) => {
+    const testCase = 'create_table_request_3';
+    const expectedResponse = createTableExpectedResponses.request3;
+    const queryDataRequest = createTableByDataQueryRequests.request3;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTableByDataQuery(
+      ulid() as PFrameInternal.PTableId,
+      queryDataRequest
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_by_data_query_4', async ({ expect }) => {
+    const testCase = 'create_table_request_4';
+    const expectedResponse = createTableExpectedResponses.request4;
+    const queryDataRequest = createTableByDataQueryRequests.request4;
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTableByDataQuery(
+      ulid() as PFrameInternal.PTableId,
+      queryDataRequest
+    );
+
+    const columnIndices = Array.from(
+      { length: table.getSpec().length },
+      (_, i) => i
+    );
+    const actualResponse = await table.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedResponse);
+  });
+
+  test.concurrent('create_table_by_data_query_fail', async ({ expect }) => {
+    const testCase = 'create_table_request_fail';
+
+    const columnId = 'column' as PObjectId;
+    const columnSpec = readJsonTestFile(testCase, 'column.spec');
+    const dataInfo = readJsonTestFile(testCase, 'column.datainfo');
+    const dataSource = createFailingDataSource();
+
+    using pframe = PFrameFactory.createPFrame({
+      frameId: ulid() as PFrameInternal.PFrameId,
+      spillPath: tmpdir()
+    });
+    pframe.setDataSource(dataSource);
+
+    pframe.addColumnSpec(columnId, columnSpec);
+    await pframe.setColumnData(columnId, dataInfo);
+
+    // Construct QueryData request for the failing column
+    const queryDataRequest = {
+      tableSpec: {
+        axes: [{ name: 'axis', type: 'Long' }] as AxisId[],
+        columns: [
+          {
+            columnId: 'column',
+            spec: {
+              kind: 'PColumn',
+              axesSpec: [{ name: 'axis', type: 'Long' }],
+              name: 'column',
+              valueType: 'Long'
+            }
+          }
+        ] as PColumnIdAndSpec[]
+      },
+      dataQuery: {
+        type: 'column',
+        columnId: 'column'
+      }
+    };
+
+    // Cast to access the method which may not be on the interface yet
+    using table = pframe.createTableByDataQuery(
+      ulid() as PFrameInternal.PTableId,
+      queryDataRequest as any
+    );
+
+    await expect(table.getShape()).rejects.toThrow();
+  });
+});
+
+describe('PFrame Table Operations', () => {
+  test.concurrent('sort_table_request', async ({ expect }) => {
+    const testCase = 'sort_table_request';
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    const sorting: PTableSorting[] = readJsonTestFile(testCase, 'sorting.json');
+
+    // Expected response for full sorted data
+    let expectedFullResponse: PTableVector[] = [
+      {
+        type: 'Long', // Axis A1 - all values present
+        data: new BigInt64Array([BigInt(0), BigInt(0), BigInt(1), BigInt(2)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'String', // Axis A2 - positions 2,3 absent (no record in column2 for A1=1,2)
+        data: ['FloatOne', 'FloatTwo', null, null],
+        isNA: new Uint8Array([48]),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'String', // Column column1 - positions 0,1 are NA (no data for A1=0)
+        data: [null, null, 'IntOne', 'IntTwo'],
+        isNA: new Uint8Array([192]),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Float', // Column column2 - positions 2,3 are NA (no data for A1=1,2)
+        data: new Float32Array([1.0, 2.0, NaN, NaN]),
+        isNA: new Uint8Array([48]),
+        absent: new Uint8Array()
+      }
+    ];
+
+    // Expected response for range query (rows 1-2, reordered columns)
+    const expectedRangeResponse: PTableVector[] = [
+      {
+        type: 'Float', // Column column2 - position 1 is NA
+        data: new Float32Array([2.0, NaN]),
+        isNA: new Uint8Array([64]),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'String', // Column column1 - position 0 is NA
+        data: [null, 'IntOne'],
+        isNA: new Uint8Array([128]),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'String', // Axis A2 - position 1 is absent
+        data: ['FloatTwo', null],
+        isNA: new Uint8Array([64]),
+        absent: new Uint8Array()
+      },
+      {
+        type: 'Long', // Axis A1 - all present
+        data: new BigInt64Array([BigInt(0), BigInt(1)]),
+        isNA: new Uint8Array(),
+        absent: new Uint8Array()
+      }
+    ];
+
+    using pframe = await readPFrame(testCase);
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    using sortedTable = table.sort(ulid() as PFrameInternal.PTableId, sorting);
+    const footprint = await sortedTable.getFootprint();
+    expect(footprint).toBeGreaterThan(0);
+
+    // Test full sorted data
+    const columnIndices = sortedTable.getColumnIndices(
+      Array.from(sorting, (entry) => entry.column)
+    );
+    let actualResponse = await sortedTable.getData(columnIndices);
+    expect(actualResponse).toEqual(expectedFullResponse);
+
+    // Test range query with reordered columns
+    actualResponse = await sortedTable.getData(
+      [columnIndices[3], columnIndices[2], columnIndices[1], columnIndices[0]],
+      { range: { offset: 1, length: 2 } }
+    );
+    expect(actualResponse).toEqual(expectedRangeResponse);
+  });
+});
+
+describe('PFrame Complex Operations', () => {
+  test.concurrent('complex_join', async ({ expect }) => {
+    const testCase = 'complex_join';
+    using pframe = await readPFrame(testCase);
+
+    const request = readJsonTestFile(testCase, 'request.json');
+    using table = pframe.createTable(
+      ulid() as PFrameInternal.PTableId,
+      request
+    );
+
+    const shape = await table.getShape();
+    expect(shape).toEqual({
+      columns: 6,
+      rows: 7124
+    });
+
+    // Test profiling functionality (platform-dependent)
+    if (process.platform === 'win32') {
+      expect(PFrameFactory.pprofDump()).rejects.toThrow();
+    } else {
+      const profile = await PFrameFactory.pprofDump();
+      const profilePath = path.join(tmpdir(), `profile_${Date.now()}.pb.gz`);
+      fs.writeFileSync(profilePath, profile);
+      console.log(`Profile saved to ${profilePath}`);
+    }
+
+    // Test out of bounds range
+    const result = await table.getData([0], {
+      range: { offset: 10000, length: 20000 }
+    });
+    expect(result).toMatchObject([{ data: [] }]);
+
+    const result2 = await table.getData([0], {
+      range: { offset: 7123, length: 2 }
+    });
+    expect(result2[0].data.length).toBe(1);
+  });
+});