@milaboratories/pf-driver 1.0.42 → 1.0.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pf-driver",
-  "version": "1.0.42",
+  "version": "1.0.44",
   "description": "PFrameDriver implementation abstracted from Middle Layer",
   "keywords": [],
   "license": "UNLICENSED",
@@ -20,11 +20,12 @@
     }
   },
   "dependencies": {
-    "@milaboratories/pframes-rs-node": "1.0.109",
+    "@milaboratories/pframes-rs-node": "1.1.2",
+    "@milaboratories/pframes-rs-wasm": "0.1.0",
     "es-toolkit": "^1.39.10",
     "lru-cache": "^11.2.2",
-    "@milaboratories/pl-model-middle-layer": "1.11.7",
     "@milaboratories/ts-helpers": "1.7.2",
+    "@milaboratories/pl-model-middle-layer": "1.11.8",
     "@platforma-sdk/model": "1.53.15"
   },
   "devDependencies": {
@@ -32,9 +33,9 @@
     "@vitest/coverage-istanbul": "^4.0.16",
     "typescript": "~5.6.3",
     "vitest": "^4.0.16",
-    "@milaboratories/build-configs": "1.4.3",
-    "@milaboratories/ts-builder": "1.2.7",
-    "@milaboratories/ts-configs": "1.2.1"
+    "@milaboratories/ts-configs": "1.2.1",
+    "@milaboratories/ts-builder": "1.2.9",
+    "@milaboratories/build-configs": "1.4.3"
   },
   "engines": {
     "node": ">=22.19.0"

package/src/driver_decl.ts

@@ -39,6 +39,9 @@ export interface AbstractInternalPFrameDriver<PColumnData> extends PFrameDriver,
   /** Create a new PTable */
   createPTable(def: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle>;
 
+  /** Create a new PTable by new Pframe-rs api */
+  createPTableV2(def: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle>;
+
   /** Calculates data for the table and returns complete data representation of it */
   calculateTableData(
     handle: PFrameHandle,

package/src/driver_double.test.ts

@@ -142,3 +142,50 @@ test.for([{ testCase: "01_json" }, { testCase: "02_binary" }, { testCase: "03_pa
     expect(result).toEqual(expected);
   },
 );
+
+test.skip("createTableV2 support", async ({ expect }) => {
+  await using driver = await createPFrameDriverDouble({});
+
+  const columnId = "column1" as PObjectId;
+  const columnSpec = {
+    kind: "PColumn" as const,
+    axesSpec: [
+      {
+        name: "axis1",
+        type: "String" as const,
+      },
+    ],
+    name: "column1",
+    valueType: "Int" as const,
+  };
+
+  using pTable = driver.createPTableV2({
+    src: {
+      type: "column",
+      column: {
+        id: columnId,
+        spec: columnSpec,
+        data: [
+          { key: ["a"], val: 10 },
+          { key: ["b"], val: 20 },
+        ],
+      },
+    },
+    partitionFilters: [],
+    filters: [],
+    sorting: [],
+  });
+
+  const uiDriver: PFrameDriver = driver;
+  const shape = await uiDriver.getShape(pTable.key);
+  expect(shape.rows).toBe(2);
+  expect(shape.columns).toBe(2); // 1 axis + 1 value column
+
+  const data = await uiDriver.getData(pTable.key, [0, 1]);
+  // axis column
+  expect(data[0].type).toBe("String");
+  expect([...data[0].data]).toEqual(["a", "b"]);
+  // value column
+  expect(data[1].type).toBe("Int");
+  expect([...data[1].data]).toEqual([10, 20]);
+});

package/src/driver_double.ts

@@ -125,6 +125,7 @@ export type InternalPFrameDriverDouble = AbstractInternalPFrameDriver<
   PFrameInternal.DataInfo<FileName> | PColumnValues
 >;
 
+// It's mock for testing purposes, not a real test!
 export async function createPFrameDriverDouble({
   dataFolder = tmpdir() as FolderPath,
   logger = () => {},

package/src/driver_impl.ts

@@ -57,6 +57,7 @@ import {
   PTableCachePlainOpsDefaults,
   type PTableCachePlainOps,
 } from "./ptable_cache_plain";
+// import { createPFrame as createSpecFrame } from "@milaboratories/pframes-rs-wasm";
 
 // eslint-disable-next-line @typescript-eslint/no-empty-object-type
 export interface LocalBlobProvider<
@@ -181,8 +182,11 @@ export class AbstractPFrameDriver<
         this.logger,
       ),
     );
-
-    const pTableEntry = this.pTableDefs.acquire({ def: sortedDef, pFrameHandle: pFrameEntry.key });
+    const pTableEntry = this.pTableDefs.acquire({
+      type: "v1",
+      def: sortedDef,
+      pFrameHandle: pFrameEntry.key,
+    });
     if (logPFrames()) {
       this.logger(
         "info",
@@ -202,6 +206,52 @@ export class AbstractPFrameDriver<
     };
   }
 
+  public createPTableV2(_rawDef: PTableDef<PColumn<PColumnData>>): PoolEntry<PTableHandle> {
+    throw new Error("createPTableV2 is not implemented yet");
+    // const columns = extractAllColumns(rawDef.src);
+    // const pFrameEntry = this.createPFrame(columns);
+
+    // const columnsMap = columns.reduce(
+    //   (acc, col) => ((acc[col.id] = col.spec), acc),
+    //   {} as Record<string, PColumnSpec>,
+    // );
+    // const sortedDef = sortPTableDef(
+    //   migratePTableFilters(
+    //     mapPTableDef(rawDef, (c) => c.id),
+    //     this.logger,
+    //   ),
+    // );
+    // const specFrame = createSpecFrame(columnsMap);
+    // const specQuery = specFrame.rewriteLegacyQuery(sortedDef);
+    // const { tableSpec, dataQuery } = specFrame.evaluateQuery(specQuery);
+
+    // const pTableEntry = this.pTableDefs.acquire({
+    //   type: "v2",
+    //   pFrameHandle: pFrameEntry.key,
+    //   def: {
+    //     tableSpec,
+    //     dataQuery,
+    //   },
+    // });
+    // if (logPFrames()) {
+    //   this.logger(
+    //     "info",
+    //     `Create PTable call (pFrameHandle = ${pFrameEntry.key}; pTableHandle = ${pTableEntry.key})`,
+    //   );
+    // }
+
+    // const unref = () => {
+    //   pTableEntry.unref();
+    //   pFrameEntry.unref();
+    // };
+    // return {
+    //   key: pTableEntry.key,
+    //   resource: pTableEntry.resource,
+    //   unref,
+    //   [Symbol.dispose]: unref,
+    // };
+  }
+
   //
   // PFrame instance methods
   //
@@ -276,6 +326,7 @@ export class AbstractPFrameDriver<
     }
 
     const table = this.pTables.acquire({
+      type: "v1",
       pFrameHandle: handle,
       def: sortPTableDef(migratePTableFilters(request, this.logger)),
     });

package/src/ptable_def_pool.ts

@@ -2,7 +2,8 @@ import { PFrameDriverError, type PTableHandle } from "@platforma-sdk/model";
 import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
 import { RefCountPoolBase } from "@milaboratories/ts-helpers";
 import { logPFrames } from "./logging";
-import { stableKeyFromFullPTableDef, type FullPTableDef } from "./ptable_shared";
+import type { FullPTableDef } from "./ptable_shared";
+import { stableKeyFromFullPTableDef } from "./ptable_shared";
 
 export class PTableDefHolder implements Disposable {
   private readonly abortController = new AbortController();

package/src/ptable_pool.ts

@@ -13,7 +13,12 @@ import type { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
 import { RefCountPoolBase, type PoolEntry } from "@milaboratories/ts-helpers";
 import { logPFrames } from "./logging";
 import type { PFramePool } from "./pframe_pool";
-import { stableKeyFromFullPTableDef, type FullPTableDef } from "./ptable_shared";
+import {
+  FullPTableDefV1,
+  FullPTableDefV2,
+  stableKeyFromFullPTableDef,
+  type FullPTableDef,
+} from "./ptable_shared";
 import type { PTableDefPool } from "./ptable_def_pool";
 
 export class PTableHolder implements Disposable {
@@ -72,53 +77,83 @@ export class PTablePool<TreeEntry extends JsonSerializable> extends RefCountPool
       );
     }
 
-    const handle = params.pFrameHandle;
-    const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
+    switch (params.type) {
+      case "v1":
+        return this.createNewResourceV1(params, key);
+      case "v2":
+        return this.createNewResourceV2(params, key);
+      default:
+        // @ts-expect-error `params.type` is a string, but we want to make sure all cases are handled
+        throw new PFrameDriverError(`Unsupported FullPTableDef type: ${params.type}`);
+    }
+  }
+
+  protected createNewResourceV1(params: FullPTableDefV1, key: PTableHandle): PTableHolder {
+    const { def, pFrameHandle } = params;
+    const { pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);
 
     const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
     const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
 
     // 3. Sort
-    if (params.def.sorting.length > 0) {
+    if (def.sorting.length > 0) {
       const predecessor = this.acquire({
         ...params,
         def: {
-          ...params.def,
+          ...def,
           sorting: [],
         },
       });
       const {
         resource: { pTablePromise },
       } = predecessor;
-      const sortedTable = pTablePromise.then((pTable) => pTable.sort(key, params.def.sorting));
-      return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
+      const sortedTable = pTablePromise.then((pTable) => pTable.sort(key, def.sorting));
+      return new PTableHolder(pFrameHandle, combinedSignal, sortedTable, predecessor);
     }
 
     // 2. Filter (except the case with artificial columns where cartesian creates too many rows)
-    if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
+    if (!hasArtificialColumns(def.src) && def.filters.length > 0) {
       const predecessor = this.acquire({
         ...params,
         def: {
-          ...params.def,
+          ...def,
           filters: [],
         },
       });
       const {
         resource: { pTablePromise },
       } = predecessor;
-      const filteredTable = pTablePromise.then((pTable) => pTable.filter(key, params.def.filters));
-      return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
+      const filteredTable = pTablePromise.then((pTable) => pTable.filter(key, def.filters));
+      return new PTableHolder(pFrameHandle, combinedSignal, filteredTable, predecessor);
     }
 
     // 1. Join
     const table = pFramePromise.then((pFrame) =>
       pFrame.createTable(key, {
-        src: joinEntryToInternal(params.def.src),
-        // `params.def.filters` would be non-empty only when join has artificial columns
-        filters: [...params.def.partitionFilters, ...params.def.filters],
+        src: joinEntryToInternal(def.src),
+        // `def.filters` would be non-empty only when join has artificial columns
+        filters: [...def.partitionFilters, ...def.filters],
       }),
     );
-    return new PTableHolder(handle, combinedSignal, table);
+    return new PTableHolder(pFrameHandle, combinedSignal, table);
+  }
+
+  protected createNewResourceV2(params: FullPTableDefV2, key: PTableHandle): PTableHolder {
+    if (logPFrames()) {
+      this.logger(
+        "info",
+        `PTable creation (pTableHandle = ${key}): ` + `${JSON.stringify(params, bigintReplacer)}`,
+      );
+    }
+
+    const { pFrameHandle } = params;
+    const { pFramePromise, disposeSignal } = this.pFrames.getByKey(pFrameHandle);
+
+    const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
+    const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
+
+    const table = pFramePromise.then((pFrame) => pFrame.createTableV2(key, params.def));
+    return new PTableHolder(pFrameHandle, combinedSignal, table);
   }
 
   public getByKey(key: PTableHandle): PTableHolder {

package/src/ptable_shared.ts

@@ -1,11 +1,30 @@
-import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from "@platforma-sdk/model";
+import type {
+  PObjectId,
+  PFrameHandle,
+  PTableDef,
+  PTableHandle,
+  QueryData,
+  PTableColumnSpec,
+} from "@platforma-sdk/model";
 import { hashJson } from "@milaboratories/pl-model-middle-layer";
 
-export type FullPTableDef = {
+export type FullPTableDefV1 = {
+  type: "v1";
   pFrameHandle: PFrameHandle;
   def: PTableDef<PObjectId>;
 };
 
+export type FullPTableDefV2 = {
+  type: "v2";
+  pFrameHandle: PFrameHandle;
+  def: {
+    tableSpec: PTableColumnSpec[];
+    dataQuery: QueryData;
+  };
+};
+
+export type FullPTableDef = FullPTableDefV1 | FullPTableDefV2;
+
 export function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {
   return hashJson(data) as string as PTableHandle;
 }