@milaboratories/pf-driver 1.0.0

package/dist/ptable_pool.js

@@ -0,0 +1,164 @@
+import { bigintReplacer, PFrameDriverError, assertNever } from '@platforma-sdk/model';
+import { RefCountPoolBase } from '@milaboratories/ts-helpers';
+import { logPFrames } from './logging.js';
+import { stableKeyFromFullPTableDef } from './ptable_shared.js';
+
+class PTableHolder {
+    pFrame;
+    pTablePromise;
+    predecessor;
+    abortController = new AbortController();
+    combinedDisposeSignal;
+    constructor(pFrame, pFrameDisposeSignal, pTablePromise, predecessor) {
+        this.pFrame = pFrame;
+        this.pTablePromise = pTablePromise;
+        this.predecessor = predecessor;
+        this.combinedDisposeSignal = AbortSignal.any([pFrameDisposeSignal, this.abortController.signal]);
+    }
+    get disposeSignal() {
+        return this.combinedDisposeSignal;
+    }
+    [Symbol.dispose]() {
+        this.abortController.abort();
+        this.predecessor?.unref();
+        void this.pTablePromise
+            .then((pTable) => pTable.dispose())
+            .catch(() => { });
+    }
+}
+class PTablePool extends RefCountPoolBase {
+    pFrames;
+    pTableDefs;
+    logger;
+    constructor(pFrames, pTableDefs, logger) {
+        super();
+        this.pFrames = pFrames;
+        this.pTableDefs = pTableDefs;
+        this.logger = logger;
+    }
+    calculateParamsKey(params) {
+        return stableKeyFromFullPTableDef(params);
+    }
+    createNewResource(params, key) {
+        if (logPFrames()) {
+            this.logger('info', `PTable creation (pTableHandle = ${key}): `
+                + `${JSON.stringify(params, bigintReplacer)}`);
+        }
+        const handle = params.pFrameHandle;
+        const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);
+        const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;
+        const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));
+        // 3. Sort
+        if (params.def.sorting.length > 0) {
+            const predecessor = this.acquire({
+                ...params,
+                def: {
+                    ...params.def,
+                    sorting: [],
+                },
+            });
+            const { resource: { pTablePromise } } = predecessor;
+            const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));
+            return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);
+        }
+        // 2. Filter (except the case with artificial columns where cartesian creates too many rows)
+        if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {
+            const predecessor = this.acquire({
+                ...params,
+                def: {
+                    ...params.def,
+                    filters: [],
+                },
+            });
+            const { resource: { pTablePromise } } = predecessor;
+            const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));
+            return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);
+        }
+        // 1. Join
+        const table = pFramePromise.then((pFrame) => pFrame.createTable({
+            src: joinEntryToInternal(params.def.src),
+            // `params.def.filters` would be non-empty only when join has artificial columns
+            filters: [...params.def.partitionFilters, ...params.def.filters],
+        }));
+        return new PTableHolder(handle, combinedSignal, table);
+    }
+    getByKey(key) {
+        const resource = super.tryGetByKey(key);
+        if (!resource)
+            throw new PFrameDriverError(`PTable not found, handle = ${key}`);
+        return resource;
+    }
+}
+function hasArtificialColumns(entry) {
+    switch (entry.type) {
+        case 'column':
+        case 'slicedColumn':
+        case 'inlineColumn':
+            return false;
+        case 'artificialColumn':
+            return true;
+        case 'full':
+        case 'inner':
+            return entry.entries.some(hasArtificialColumns);
+        case 'outer':
+            return hasArtificialColumns(entry.primary) || entry.secondary.some(hasArtificialColumns);
+        default:
+            assertNever(entry);
+    }
+}
+function joinEntryToInternal(entry) {
+    const type = entry.type;
+    switch (type) {
+        case 'column':
+            return {
+                type: 'column',
+                columnId: entry.column,
+            };
+        case 'slicedColumn':
+            return {
+                type: 'slicedColumn',
+                columnId: entry.column,
+                newId: entry.newId,
+                axisFilters: entry.axisFilters,
+            };
+        case 'artificialColumn':
+            return {
+                type: 'artificialColumn',
+                columnId: entry.column,
+                newId: entry.newId,
+                axesIndices: entry.axesIndices,
+            };
+        case 'inlineColumn':
+            return {
+                type: 'inlineColumn',
+                newId: entry.column.id,
+                spec: entry.column.spec,
+                dataInfo: {
+                    type: 'Json',
+                    keyLength: entry.column.spec.axesSpec.length,
+                    data: entry.column.data.reduce((acc, row) => {
+                        acc[JSON.stringify(row.key)] = row.val;
+                        return acc;
+                    }, {}),
+                },
+            };
+        case 'inner':
+        case 'full':
+            return {
+                type: entry.type,
+                entries: entry.entries.map((col) => joinEntryToInternal(col)),
+            };
+        case 'outer':
+            return {
+                type: 'outer',
+                primary: joinEntryToInternal(entry.primary),
+                secondary: entry.secondary.map((col) => joinEntryToInternal(col)),
+            };
+        default:
+            // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+            throw new PFrameDriverError(`unsupported PFrame join entry type: ${type}`);
+    }
+}
+
+export { PTableHolder, PTablePool };
+//# sourceMappingURL=ptable_pool.js.map

package/dist/ptable_pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_pool.js","sources":["../src/ptable_pool.ts"],"sourcesContent":["import {\n  assertNever,\n  bigintReplacer,\n  PFrameDriverError,\n  type PFrameHandle,\n  type PTableHandle,\n  type JoinEntry,\n  type JsonSerializable,\n  type PColumnValue,\n  type PObjectId,\n} from '@platforma-sdk/model';\nimport type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport { RefCountPoolBase, type PoolEntry } from '@milaboratories/ts-helpers';\nimport { logPFrames } from './logging';\nimport type { PFramePool } from './pframe_pool';\nimport { stableKeyFromFullPTableDef, type FullPTableDef } from './ptable_shared';\nimport type { PTableDefPool } from './ptable_def_pool';\n\nexport class PTableHolder implements Disposable {\n  private readonly abortController = new AbortController();\n  private readonly combinedDisposeSignal: AbortSignal;\n\n  constructor(\n    public readonly pFrame: PFrameHandle,\n    pFrameDisposeSignal: AbortSignal,\n    public readonly pTablePromise: Promise<PFrameInternal.PTableV7>,\n    private readonly predecessor?: PoolEntry<PTableHandle, PTableHolder>,\n  ) {\n    this.combinedDisposeSignal = AbortSignal.any([pFrameDisposeSignal, this.abortController.signal]);\n  }\n\n  public get disposeSignal(): AbortSignal {\n    return this.combinedDisposeSignal;\n  }\n\n  [Symbol.dispose](): void {\n    this.abortController.abort();\n    this.predecessor?.unref();\n    void this.pTablePromise\n      .then((pTable) => pTable.dispose())\n      .catch(() => { /* mute error */ });\n  }\n}\n\nexport class PTablePool<TreeEntry extends JsonSerializable>\n  extends RefCountPoolBase<FullPTableDef, PTableHandle, PTableHolder> {\n  constructor(\n    private readonly pFrames: PFramePool<TreeEntry>,\n    private readonly pTableDefs: PTableDefPool,\n    private readonly logger: PFrameInternal.Logger,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: FullPTableDef): PTableHandle {\n    return stableKeyFromFullPTableDef(params);\n  }\n\n  protected createNewResource(params: FullPTableDef, key: PTableHandle): PTableHolder {\n    if (logPFrames()) {\n      this.logger('info',\n        `PTable creation (pTableHandle = ${key}): `\n        + `${JSON.stringify(params, bigintReplacer)}`,\n      );\n    }\n\n    const handle = params.pFrameHandle;\n    const { pFramePromise, disposeSignal } = this.pFrames.getByKey(handle);\n\n    const defDisposeSignal = this.pTableDefs.tryGetByKey(key)?.disposeSignal;\n    const combinedSignal = AbortSignal.any([disposeSignal, defDisposeSignal].filter((s) => !!s));\n\n    // 3. Sort\n    if (params.def.sorting.length > 0) {\n      const predecessor = this.acquire({\n        ...params,\n        def: {\n          ...params.def,\n          sorting: [],\n        },\n      });\n      const { resource: { pTablePromise } } = predecessor;\n      const sortedTable = pTablePromise.then((pTable) => pTable.sort(params.def.sorting));\n      return new PTableHolder(handle, combinedSignal, sortedTable, predecessor);\n    }\n\n    // 2. Filter (except the case with artificial columns where cartesian creates too many rows)\n    if (!hasArtificialColumns(params.def.src) && params.def.filters.length > 0) {\n      const predecessor = this.acquire({\n        ...params,\n        def: {\n          ...params.def,\n          filters: [],\n        },\n      });\n      const { resource: { pTablePromise } } = predecessor;\n      const filteredTable = pTablePromise.then((pTable) => pTable.filter(params.def.filters));\n      return new PTableHolder(handle, combinedSignal, filteredTable, predecessor);\n    }\n\n    // 1. Join\n    const table = pFramePromise.then((pFrame) => pFrame.createTable({\n      src: joinEntryToInternal(params.def.src),\n      // `params.def.filters` would be non-empty only when join has artificial columns\n      filters: [...params.def.partitionFilters, ...params.def.filters],\n    }));\n    return new PTableHolder(handle, combinedSignal, table);\n  }\n\n  public getByKey(key: PTableHandle): PTableHolder {\n    const resource = super.tryGetByKey(key);\n    if (!resource) throw new PFrameDriverError(`PTable not found, handle = ${key}`);\n    return resource;\n  }\n}\n\nfunction hasArtificialColumns<T>(entry: JoinEntry<T>): boolean {\n  switch (entry.type) {\n    case 'column':\n    case 'slicedColumn':\n    case 'inlineColumn':\n      return false;\n    case 'artificialColumn':\n      return true;\n    case 'full':\n    case 'inner':\n      return entry.entries.some(hasArtificialColumns);\n    case 'outer':\n      return hasArtificialColumns(entry.primary) || entry.secondary.some(hasArtificialColumns);\n    default:\n      assertNever(entry);\n  }\n}\n\nfunction joinEntryToInternal(entry: JoinEntry<PObjectId>): PFrameInternal.JoinEntryV4 {\n  const type = entry.type;\n  switch (type) {\n    case 'column':\n      return {\n        type: 'column',\n        columnId: entry.column,\n      };\n    case 'slicedColumn':\n      return {\n        type: 'slicedColumn',\n        columnId: entry.column,\n        newId: entry.newId,\n        axisFilters: entry.axisFilters,\n      };\n    case 'artificialColumn':\n      return {\n        type: 'artificialColumn',\n        columnId: entry.column,\n        newId: entry.newId,\n        axesIndices: entry.axesIndices,\n      };\n    case 'inlineColumn':\n      return {\n        type: 'inlineColumn',\n        newId: entry.column.id,\n        spec: entry.column.spec,\n        dataInfo: {\n          type: 'Json',\n          keyLength: entry.column.spec.axesSpec.length,\n          data: entry.column.data.reduce((acc, row) => {\n            acc[JSON.stringify(row.key)] = row.val;\n            return acc;\n          }, {} as Record<string, PColumnValue>),\n        },\n      };\n    case 'inner':\n    case 'full':\n      return {\n        type: entry.type,\n        entries: entry.entries.map((col) => joinEntryToInternal(col)),\n      };\n    case 'outer':\n      return {\n        type: 'outer',\n        primary: joinEntryToInternal(entry.primary),\n        secondary: entry.secondary.map((col) => joinEntryToInternal(col)),\n      };\n    default:\n      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions\n      throw new PFrameDriverError(`unsupported PFrame join entry type: ${type satisfies never}`);\n  }\n}\n"],"names":[],"mappings":";;;;;MAkBa,YAAY,CAAA;AAKL,IAAA,MAAA;AAEA,IAAA,aAAA;AACC,IAAA,WAAA;AAPF,IAAA,eAAe,GAAG,IAAI,eAAe,EAAE;AACvC,IAAA,qBAAqB;AAEtC,IAAA,WAAA,CACkB,MAAoB,EACpC,mBAAgC,EAChB,aAA+C,EAC9C,WAAmD,EAAA;QAHpD,IAAA,CAAA,MAAM,GAAN,MAAM;QAEN,IAAA,CAAA,aAAa,GAAb,aAAa;QACZ,IAAA,CAAA,WAAW,GAAX,WAAW;AAE5B,QAAA,IAAI,CAAC,qBAAqB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,mBAAmB,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAClG;AAEA,IAAA,IAAW,aAAa,GAAA;QACtB,OAAO,IAAI,CAAC,qBAAqB;IACnC;IAEA,CAAC,MAAM,CAAC,OAAO,CAAC,GAAA;AACd,QAAA,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE;AAC5B,QAAA,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE;QACzB,KAAK,IAAI,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,EAAE;AACjC,aAAA,KAAK,CAAC,MAAK,EAAoB,CAAC,CAAC;IACtC;AACD;AAEK,MAAO,UACX,SAAQ,gBAA2D,CAAA;AAEhD,IAAA,OAAA;AACA,IAAA,UAAA;AACA,IAAA,MAAA;AAHnB,IAAA,WAAA,CACmB,OAA8B,EAC9B,UAAyB,EACzB,MAA6B,EAAA;AAE9C,QAAA,KAAK,EAAE;QAJU,IAAA,CAAA,OAAO,GAAP,OAAO;QACP,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;IAGzB;AAEU,IAAA,kBAAkB,CAAC,MAAqB,EAAA;AAChD,QAAA,OAAO,0BAA0B,CAAC,MAAM,CAAC;IAC3C;IAEU,iBAAiB,CAAC,MAAqB,EAAE,GAAiB,EAAA;QAClE,IAAI,UAAU,EAAE,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,gCAAA,EAAmC,GAAG,CAAA,GAAA;kBACpC,CAAA,EAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA,CAAE,CAC9C;QACH;AAEA,QAAA,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY;AAClC,QAAA,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;AAEtE,QAAA,MAAM,gBAAgB,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,aAAa;QACxE,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;;QAG5F,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,YAAA,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;AAC/B,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,EAAE;oBACH,GAAG,MAAM,CAAC,GAAG;AACb,oBAAA,OAAO,EAAE,EAAE;AACZ,iBAAA;AACF,aAAA,CAAC;YACF,MAAM,EAAE,QAAQ,EAAE,EAAE,aAAa,EAAE,EAAE,GAAG,WAAW;YACnD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnF,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,CAAC;QAC3E;;QAGA,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AAC1E,YAAA,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;AAC/B,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,EAAE;oBACH,GAAG,MAAM,CAAC,GAAG;AACb,oBAAA,OAAO,EAAE,EAAE;AACZ,iBAAA;AACF,aAAA,CAAC;YACF,MAAM,EAAE,QAAQ,EAAE,EAAE,aAAa,EAAE,EAAE,GAAG,WAAW;YACnD,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACvF,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,CAAC;QAC7E;;AAGA,QAAA,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,WAAW,CAAC;YAC9D,GAAG,EAAE,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;;AAExC,YAAA,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;AACjE,SAAA,CAAC,CAAC;QACH,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC;IACxD;AAEO,IAAA,QAAQ,CAAC,GAAiB,EAAA;QAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC;AACvC,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,8BAA8B,GAAG,CAAA,CAAE,CAAC;AAC/E,QAAA,OAAO,QAAQ;IACjB;AACD;AAED,SAAS,oBAAoB,CAAI,KAAmB,EAAA;AAClD,IAAA,QAAQ,KAAK,CAAC,IAAI;AAChB,QAAA,KAAK,QAAQ;AACb,QAAA,KAAK,cAAc;AACnB,QAAA,KAAK,cAAc;AACjB,YAAA,OAAO,KAAK;AACd,QAAA,KAAK,kBAAkB;AACrB,YAAA,OAAO,IAAI;AACb,QAAA,KAAK,MAAM;AACX,QAAA,KAAK,OAAO;YACV,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC;AACjD,QAAA,KAAK,OAAO;AACV,YAAA,OAAO,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC;AAC1F,QAAA;YACE,WAAW,CAAC,KAAK,CAAC;;AAExB;AAEA,SAAS,mBAAmB,CAAC,KAA2B,EAAA;AACtD,IAAA,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI;IACvB,QAAQ,IAAI;AACV,QAAA,KAAK,QAAQ;YACX,OAAO;AACL,gBAAA,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,KAAK,CAAC,MAAM;aACvB;AACH,QAAA,KAAK,cAAc;YACjB,OAAO;AACL,gBAAA,IAAI,EAAE,cAAc;gBACpB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B;AACH,QAAA,KAAK,kBAAkB;YACrB,OAAO;AACL,gBAAA,IAAI,EAAE,kBAAkB;gBACxB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B;AACH,QAAA,KAAK,cAAc;YACjB,OAAO;AACL,gBAAA,IAAI,EAAE,cAAc;AACpB,gBAAA,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE;AACtB,gBAAA,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;AACvB,gBAAA,QAAQ,EAAE;AACR,oBAAA,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM;AAC5C,oBAAA,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,KAAI;AAC1C,wBAAA,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG;AACtC,wBAAA,OAAO,GAAG;oBACZ,CAAC,EAAE,EAAkC,CAAC;AACvC,iBAAA;aACF;AACH,QAAA,KAAK,OAAO;AACZ,QAAA,KAAK,MAAM;YACT,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;AAChB,gBAAA,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC;aAC9D;AACH,QAAA,KAAK,OAAO;YACV,OAAO;AACL,gBAAA,IAAI,EAAE,OAAO;AACb,gBAAA,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC;AAC3C,gBAAA,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC;aAClE;AACH,QAAA;;AAEE,YAAA,MAAM,IAAI,iBAAiB,CAAC,uCAAuC,IAAoB,CAAA,CAAE,CAAC;;AAEhG;;;;"}

package/dist/ptable_shared.cjs

@@ -0,0 +1,10 @@
+'use strict';
+
+var tsHelpers = require('@milaboratories/ts-helpers');
+
+function stableKeyFromFullPTableDef(data) {
+    return tsHelpers.hashJson(data);
+}
+
+exports.stableKeyFromFullPTableDef = stableKeyFromFullPTableDef;
+//# sourceMappingURL=ptable_shared.cjs.map

package/dist/ptable_shared.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_shared.cjs","sources":["../src/ptable_shared.ts"],"sourcesContent":["import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from '@platforma-sdk/model';\nimport { hashJson } from '@milaboratories/ts-helpers';\n\nexport type FullPTableDef = {\n  pFrameHandle: PFrameHandle;\n  def: PTableDef<PObjectId>;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n  return hashJson(data) as string as PTableHandle;\n}\n"],"names":["hashJson"],"mappings":";;;;AAQM,SAAU,0BAA0B,CAAC,IAAmB,EAAA;AAC5D,IAAA,OAAOA,kBAAQ,CAAC,IAAI,CAA2B;AACjD;;;;"}

package/dist/ptable_shared.d.ts

@@ -0,0 +1,7 @@
+import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from '@platforma-sdk/model';
+export type FullPTableDef = {
+    pFrameHandle: PFrameHandle;
+    def: PTableDef<PObjectId>;
+};
+export declare function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle;
+//# sourceMappingURL=ptable_shared.d.ts.map

package/dist/ptable_shared.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_shared.d.ts","sourceRoot":"","sources":["../src/ptable_shared.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAG7F,MAAM,MAAM,aAAa,GAAG;IAC1B,YAAY,EAAE,YAAY,CAAC;IAC3B,GAAG,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;CAC3B,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,aAAa,GAAG,YAAY,CAE5E"}

package/dist/ptable_shared.js

@@ -0,0 +1,8 @@
+import { hashJson } from '@milaboratories/ts-helpers';
+
+function stableKeyFromFullPTableDef(data) {
+    return hashJson(data);
+}
+
+export { stableKeyFromFullPTableDef };
+//# sourceMappingURL=ptable_shared.js.map

package/dist/ptable_shared.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_shared.js","sources":["../src/ptable_shared.ts"],"sourcesContent":["import type { PObjectId, PFrameHandle, PTableDef, PTableHandle } from '@platforma-sdk/model';\nimport { hashJson } from '@milaboratories/ts-helpers';\n\nexport type FullPTableDef = {\n  pFrameHandle: PFrameHandle;\n  def: PTableDef<PObjectId>;\n};\n\nexport function stableKeyFromFullPTableDef(data: FullPTableDef): PTableHandle {\n  return hashJson(data) as string as PTableHandle;\n}\n"],"names":[],"mappings":";;AAQM,SAAU,0BAA0B,CAAC,IAAmB,EAAA;AAC5D,IAAA,OAAO,QAAQ,CAAC,IAAI,CAA2B;AACjD;;;;"}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@milaboratories/pf-driver",
+  "version": "1.0.0",
+  "description": "PFrameDriver implementation abstracted from Middle Layer",
+  "engines": {
+    "node": ">=22.19.0"
+  },
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "./dist/**/*",
+    "./src/**/*"
+  ],
+  "keywords": [],
+  "license": "UNLICENSED",
+  "devDependencies": {
+    "eslint": "^9.25.1",
+    "tsconfig-paths": "^4.2.0",
+    "typescript": "~5.6.3",
+    "tslib": "~2.7.0",
+    "vitest": "^4.0.7",
+    "@vitest/coverage-v8": "^4.0.7",
+    "@types/node": "~24.5.2",
+    "@milaboratories/eslint-config": "1.0.4",
+    "@milaboratories/build-configs": "1.0.8",
+    "@milaboratories/ts-builder": "1.0.5",
+    "@milaboratories/ts-configs": "1.0.6"
+  },
+  "dependencies": {
+    "@milaboratories/pframes-rs-node": "1.0.103",
+    "lru-cache": "^11.2.2",
+    "es-toolkit": "^1.39.10",
+    "@milaboratories/ts-helpers": "1.5.2",
+    "@platforma-sdk/model": "1.45.26",
+    "@milaboratories/pl-model-middle-layer": "1.8.36"
+  },
+  "scripts": {
+    "type-check": "ts-builder types --target node",
+    "build": "ts-builder build --target node",
+    "watch": "ts-builder build --target node --watch",
+    "lint": "eslint .",
+    "test": "vitest run --coverage",
+    "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz"
+  }
+}

package/src/data_info_helpers.ts

@@ -0,0 +1,26 @@
+import {
+  PFrameDriverError,
+  type PColumnSpec,
+  type PColumnValues,
+  type JsonDataInfo,
+  type PColumnValue,
+} from '@platforma-sdk/model';
+
+export function makeJsonDataInfo(
+  spec: PColumnSpec,
+  data: PColumnValues,
+): JsonDataInfo {
+  const keyLength = spec.axesSpec.length;
+  const jsonData: Record<string, PColumnValue> = {};
+  for (const { key, val } of data) {
+    if (key.length !== keyLength)
+      throw new PFrameDriverError(`inline column key length ${key.length} differs from axes count ${keyLength}`);
+    jsonData[JSON.stringify(key)] = val;
+  }
+
+  return {
+    type: 'Json',
+    keyLength,
+    data: jsonData,
+  };
+}

package/src/driver_decl.ts

@@ -0,0 +1,82 @@
+import type {
+  CalculateTableDataRequest,
+  CalculateTableDataResponse,
+  PColumn,
+  PFrameDef,
+  PFrameDriver,
+  PFrameHandle,
+  PObjectId,
+  PTableDef,
+  PTableHandle,
+  PTableShape,
+  PTableVector,
+  TableRange,
+  UniqueValuesRequest,
+  UniqueValuesResponse,
+} from '@platforma-sdk/model';
+import type { PoolEntry } from '@milaboratories/ts-helpers';
+
+/**
+ * Extends public and safe SDK's driver API with methods used internally in the middle
+ * layer and in tests.
+ */
+export interface AbstractInternalPFrameDriver<PColumnData>
+  extends PFrameDriver, AsyncDisposable {
+  /** Dispose the driver and all its resources. */
+  dispose(): Promise<void>;
+
+  /**
+   * Dump active PFrames allocations in pprof format.
+   * The result of this function should be saved as `profile.pb.gz`.
+   * Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
+   * to view the allocation flamechart.
+   * @warning This method will always reject on Windows!
+   */
+  pprofDump(): Promise<Uint8Array>;
+
+  /** Create a new PFrame */
+  createPFrame(
+    def: PFrameDef<PColumn<PColumnData>>,
+  ): PoolEntry<PFrameHandle>;
+
+  /** Create a new PTable */
+  createPTable(
+    def: PTableDef<PColumn<PColumnData>>,
+  ): PoolEntry<PTableHandle>;
+
+  /** Calculates data for the table and returns complete data representation of it */
+  calculateTableData(
+    handle: PFrameHandle,
+    request: CalculateTableDataRequest<PObjectId>,
+    range: TableRange | undefined,
+    signal?: AbortSignal
+  ): Promise<CalculateTableDataResponse>;
+
+  /** Calculate set of unique values for a specific axis for the filtered set of records */
+  getUniqueValues(
+    handle: PFrameHandle,
+    request: UniqueValuesRequest,
+    signal?: AbortSignal
+  ): Promise<UniqueValuesResponse>;
+
+  /** Unified table shape */
+  getShape(
+    handle: PTableHandle,
+    signal?: AbortSignal,
+  ): Promise<PTableShape>;
+
+  /**
+   * Retrieve the data from the table. To retrieve only data required, it can be
+   * sliced both horizontally ({@link columnIndices}) and vertically
+   * ({@link range}).
+   *
+   * @param columnIndices unified indices of columns to be retrieved
+   * @param range optionally limit the range of records to retrieve
+   * */
+  getData(
+    handle: PTableHandle,
+    columnIndices: number[],
+    range: TableRange | undefined,
+    signal?: AbortSignal,
+  ): Promise<PTableVector[]>;
+}

package/src/driver_double.test.ts

@@ -0,0 +1,135 @@
+import {
+    pTableValue,
+  type CalculateTableDataResponse,
+  type PFrameDriver,
+  type PObjectId,
+} from '@platforma-sdk/model';
+import { test } from 'vitest';
+import { join } from 'node:path';
+import {
+    createPFrameDriverDouble,
+    makeFolderPath,
+} from './driver_double';
+import { readJsonSync } from '@milaboratories/ts-helpers';
+
+test('inline column support', async ({ expect }) => {
+    // Model context
+
+    await using driver = await createPFrameDriverDouble({});
+    using pFrame = driver.createPFrame([{
+        id: 'column1' as PObjectId,
+        spec: {
+            kind: 'PColumn',
+            axesSpec: [{
+                name: 'axis1',
+                type: 'String',
+            }],
+            name: 'column1',
+            valueType: 'Int',
+        },
+        data: [
+            {
+                key: ['axis1'],
+                val: 1,
+            },
+        ],
+    }]);
+
+    // UI context
+
+    const uiDriver: PFrameDriver = driver;
+    const pFrameHandle = pFrame.key;
+
+    const data = await uiDriver.calculateTableData(pFrameHandle, {
+        src: {
+            type: 'column',
+            column: 'column1' as PObjectId,
+        },
+        filters: [],
+        sorting: [],
+    });
+
+    expect(data).toEqual([
+        {
+            spec: {
+                type: 'axis',
+                id: {
+                    name: 'axis1',
+                    type: 'String',
+                },
+                spec: {
+                    name: 'axis1',
+                    type: 'String',
+                },
+            },
+            data: {
+                type: 'String',
+                data: ['axis1'],
+                isNA: new Uint8Array(),
+                absent: new Uint8Array(),
+            },
+        },
+        {
+            spec: {
+                type: 'column',
+                id: 'column1' as PObjectId,
+                spec: {
+                    kind: 'PColumn',
+                    axesSpec: [{
+                        name: 'axis1',
+                        type: 'String',
+                    }],
+                    name: 'column1',
+                    valueType: 'Int',
+                },
+            },
+            data: {
+                type: 'Int',
+                data: new Int32Array([1]),
+                isNA: new Uint8Array(),
+                absent: new Uint8Array(),
+            },
+        },
+    ] satisfies CalculateTableDataResponse);
+})
+
+test.for([
+    { testCase: '01_json' },
+    { testCase: '02_binary' },
+    { testCase: '03_parquet' },
+])(`stored column support - $testCase`, async ({ testCase }, { expect }) => {
+    const dataFolder = makeFolderPath(join(__dirname, '..', 'assets', testCase));
+
+    // Model context
+
+    await using driver = await createPFrameDriverDouble({ dataFolder });
+    const pFrame = driver.createPFrame([{
+        id: 'column' as PObjectId,
+        spec: readJsonSync(join(dataFolder, 'column.spec')),
+        data: readJsonSync(join(dataFolder, 'column.datainfo')),
+    }]);
+
+    // UI context
+
+    const uiDriver: PFrameDriver = driver;
+    const pFrameHandle = pFrame.key;
+
+    const data = await uiDriver.calculateTableData(pFrameHandle, {
+        src: {
+            type: 'column',
+            column: 'column' as PObjectId,
+        },
+        filters: [],
+        sorting: [],
+    });
+    const result = {
+        spec: data.map((d) => d.spec),
+        data: data.map((d) => d.data)
+            .map((d) => [...d.data.keys()].map((i) => pTableValue(d, i, {
+                absent: "|~|",
+                na: null,
+            }))),
+    }
+    const expected = readJsonSync(join(dataFolder, 'response.json'));
+    expect(result).toEqual(expected);
+})

package/src/driver_double.ts

@@ -0,0 +1,134 @@
+import {
+  isDataInfo,
+  PFrameDriverError,
+  type Branded,
+  type PColumnSpec,
+  type PColumnValues,
+} from '@platforma-sdk/model';
+import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { RefCountPoolBase, type PoolEntry } from '@milaboratories/ts-helpers';
+import { HttpHelpers } from '@milaboratories/pframes-rs-node';
+import fs from 'node:fs';
+import path from 'node:path';
+import { tmpdir } from 'node:os';
+import type { AbstractInternalPFrameDriver } from './driver_decl';
+import {
+  AbstractPFrameDriver,
+  type LocalBlobProvider,
+  type RemoteBlobProvider,
+} from './driver_impl';
+import { makeJsonDataInfo } from './data_info_helpers';
+
+export type FileName = Branded<string, 'FileName'>;
+export type FilePath = Branded<string, 'FilePath'>;
+export type FolderPath = Branded<string, 'FolderPath'>;
+
+export function makeFolderPath(dataFolder: string): FolderPath {
+  if (!fs.statSync(dataFolder, { throwIfNoEntry: false })?.isDirectory()) {
+    throw new PFrameDriverError(`Data folder ${dataFolder} does not exist`);
+  }
+  return dataFolder as FolderPath;
+}
+
+function makeBlobId(res: FileName): PFrameInternal.PFrameBlobId {
+  return res as string;
+}
+
+class LocalBlobProviderImpl
+  extends RefCountPoolBase<FileName, PFrameInternal.PFrameBlobId, FilePath>
+  implements LocalBlobProvider<FileName> {
+  constructor(private readonly dataFolder: FolderPath) {
+    super();
+  }
+
+  protected calculateParamsKey(params: FileName): PFrameInternal.PFrameBlobId {
+    return makeBlobId(params);
+  }
+
+  protected createNewResource(params: FileName, _key: PFrameInternal.PFrameBlobId): FilePath {
+    const filePath = path.join(this.dataFolder, params);
+    if (!fs.statSync(filePath, { throwIfNoEntry: false })?.isFile()) {
+      throw new PFrameDriverError(`File ${filePath} does not exist`);
+    }
+    return filePath as FilePath;
+  }
+
+  public getByKey(blobId: PFrameInternal.PFrameBlobId): FilePath {
+    const resource = super.tryGetByKey(blobId);
+    if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);
+    return resource;
+  }
+
+  public makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'> {
+    return {
+      preloadBlob: async (_blobIds: PFrameInternal.PFrameBlobId[]) => {},
+      resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {
+        const filePath = this.getByKey(blobId);
+        return await fs.promises.readFile(filePath, { signal });
+      },
+    };
+  }
+}
+
+class RemoteBlobProviderImpl implements RemoteBlobProvider<FileName> {
+  constructor(
+    private readonly pool: LocalBlobProviderImpl,
+    private readonly server: PFrameInternal.HttpServer,
+  ) {}
+
+  public static async init(
+    dataFolder: FolderPath,
+    logger: PFrameInternal.Logger,
+    serverOptions: Omit<PFrameInternal.HttpServerOptions, 'handler'>,
+  ): Promise<RemoteBlobProviderImpl> {
+    const pool = new LocalBlobProviderImpl(dataFolder);
+
+    const underlyingStore = await HttpHelpers.createFsStore({ rootDir: dataFolder, logger });
+    const store: PFrameInternal.ObjectStore = {
+      request: (filename, params) => {
+        const blobId = filename.slice(0, -PFrameInternal.ParquetExtension.length);
+        return underlyingStore.request(blobId as PFrameInternal.ParquetFileName, params);
+      },
+    };
+
+    const handler = HttpHelpers.createRequestHandler({ store });
+    const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });
+
+    return new RemoteBlobProviderImpl(pool, server);
+  }
+
+  public acquire(params: FileName): PoolEntry {
+    return this.pool.acquire(params);
+  }
+
+  public httpServerInfo(): PFrameInternal.HttpServerInfo {
+    return this.server.info;
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.server.stop();
+  }
+}
+
+export async function createPFrameDriverDouble({
+  dataFolder = tmpdir() as FolderPath,
+  logger = () => {},
+}: {
+  dataFolder?: FolderPath;
+  logger?: PFrameInternal.Logger;
+}): Promise<AbstractInternalPFrameDriver<PFrameInternal.DataInfo<FileName> | PColumnValues>> {
+  const localBlobProvider = new LocalBlobProviderImpl(dataFolder);
+  const remoteBlobProvider = await RemoteBlobProviderImpl.init(dataFolder, logger, {});
+
+  const resolveDataInfo = (
+    spec: PColumnSpec,
+    data: PFrameInternal.DataInfo<FileName> | PColumnValues,
+  ) => isDataInfo(data) ? data : makeJsonDataInfo(spec, data);
+
+  return new AbstractPFrameDriver({
+    logger,
+    localBlobProvider,
+    remoteBlobProvider,
+    resolveDataInfo,
+  });
+}