@milaboratories/pf-driver 1.0.0

package/dist/pframe_pool.cjs

@@ -0,0 +1,196 @@
+'use strict';
+
+var model = require('@platforma-sdk/model');
+var plModelMiddleLayer = require('@milaboratories/pl-model-middle-layer');
+var tsHelpers = require('@milaboratories/ts-helpers');
+var pframesRsNode = require('@milaboratories/pframes-rs-node');
+var esToolkit = require('es-toolkit');
+var logging = require('./logging.cjs');
+
+class PFrameHolder {
+    localBlobProvider;
+    remoteBlobProvider;
+    spillPath;
+    pFramePromise;
+    abortController = new AbortController();
+    localBlobs = [];
+    remoteBlobs = [];
+    constructor(localBlobProvider, remoteBlobProvider, logger, spillPath, columns) {
+        this.localBlobProvider = localBlobProvider;
+        this.remoteBlobProvider = remoteBlobProvider;
+        this.spillPath = spillPath;
+        const makeLocalBlobId = (blob) => {
+            const localBlob = this.localBlobProvider.acquire(blob);
+            this.localBlobs.push(localBlob);
+            return localBlob.key;
+        };
+        const makeRemoteBlobId = (blob) => {
+            const remoteBlob = this.remoteBlobProvider.acquire(blob);
+            this.remoteBlobs.push(remoteBlob);
+            return `${remoteBlob.key}${plModelMiddleLayer.PFrameInternal.ParquetExtension}`;
+        };
+        const mapColumnData = (data) => {
+            switch (data.type) {
+                case 'Json':
+                    return { ...data };
+                case 'JsonPartitioned':
+                    return {
+                        ...data,
+                        parts: esToolkit.mapValues(data.parts, makeLocalBlobId),
+                    };
+                case 'BinaryPartitioned':
+                    return {
+                        ...data,
+                        parts: esToolkit.mapValues(data.parts, (v) => ({
+                            index: makeLocalBlobId(v.index),
+                            values: makeLocalBlobId(v.values),
+                        })),
+                    };
+                case 'ParquetPartitioned':
+                    return {
+                        ...data,
+                        parts: esToolkit.mapValues(data.parts, (v) => ({
+                            ...v,
+                            data: makeRemoteBlobId(v.data),
+                        })),
+                    };
+                default:
+                    model.assertNever(data);
+            }
+        };
+        const jsonifiedColumns = columns.map((column) => ({
+            ...column,
+            data: mapColumnData(column.data),
+        }));
+        try {
+            const pFrame = pframesRsNode.PFrameFactory.createPFrame({ spillPath: this.spillPath, logger });
+            pFrame.setDataSource({
+                ...this.localBlobProvider.makeDataSource(this.disposeSignal),
+                parquetServer: this.remoteBlobProvider.httpServerInfo(),
+            });
+            const promises = [];
+            for (const column of jsonifiedColumns) {
+                pFrame.addColumnSpec(column.id, column.spec);
+                promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
+            }
+            this.pFramePromise = Promise.all(promises)
+                .then(() => pFrame)
+                .catch((err) => {
+                this.dispose();
+                pFrame.dispose();
+                throw new model.PFrameDriverError(`PFrame creation failed asynchronously, `
+                    + `columns: ${JSON.stringify(jsonifiedColumns)}, `
+                    + `error: ${model.ensureError(err)}`);
+            });
+        }
+        catch (err) {
+            throw new model.PFrameDriverError(`PFrame creation failed synchronously, `
+                + `columns: ${JSON.stringify(jsonifiedColumns)}, `
+                + `error: ${model.ensureError(err)}`);
+        }
+    }
+    get disposeSignal() {
+        return this.abortController.signal;
+    }
+    dispose() {
+        this.abortController.abort();
+        this.localBlobs.forEach((entry) => entry.unref());
+        this.remoteBlobs.forEach((entry) => entry.unref());
+    }
+    [Symbol.dispose]() {
+        this.dispose();
+        void this.pFramePromise
+            .then((pFrame) => pFrame.dispose())
+            .catch(() => { });
+    }
+}
+class PFramePool extends tsHelpers.RefCountPoolBase {
+    localBlobProvider;
+    remoteBlobProvider;
+    logger;
+    spillPath;
+    constructor(localBlobProvider, remoteBlobProvider, logger, spillPath) {
+        super();
+        this.localBlobProvider = localBlobProvider;
+        this.remoteBlobProvider = remoteBlobProvider;
+        this.logger = logger;
+        this.spillPath = spillPath;
+    }
+    calculateParamsKey(params) {
+        return stableKeyFromPFrameData(params);
+    }
+    createNewResource(params, key) {
+        if (logging.logPFrames()) {
+            this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
+                + `${JSON.stringify(params, model.bigintReplacer)}`);
+        }
+        return new PFrameHolder(this.localBlobProvider, this.remoteBlobProvider, this.logger, this.spillPath, params);
+    }
+    getByKey(key) {
+        const resource = super.tryGetByKey(key);
+        if (!resource)
+            throw new model.PFrameDriverError(`PFrame not found, handle = ${key}`);
+        return resource;
+    }
+}
+function stableKeyFromPFrameData(data) {
+    const orderedData = [...data].map((column) => model.mapPObjectData(column, (r) => {
+        let result;
+        const type = r.type;
+        switch (type) {
+            case 'Json':
+                result = {
+                    type: r.type,
+                    keyLength: r.keyLength,
+                    payload: Object.entries(r.data).map(([part, value]) => ({
+                        key: part,
+                        value,
+                    })),
+                };
+                break;
+            case 'JsonPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: model.canonicalizeJson(info),
+                    })),
+                };
+                break;
+            case 'BinaryPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: [model.canonicalizeJson(info.index), model.canonicalizeJson(info.values)],
+                    })),
+                };
+                break;
+            case 'ParquetPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: info.dataDigest || [
+                            model.canonicalizeJson(info.data),
+                            JSON.stringify({ axes: info.axes, column: info.column }),
+                        ],
+                    })),
+                };
+                break;
+            default:
+                throw new model.PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`);
+        }
+        result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);
+        return result;
+    }));
+    orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);
+    return tsHelpers.hashJson(orderedData);
+}
+
+exports.PFrameHolder = PFrameHolder;
+exports.PFramePool = PFramePool;
+//# sourceMappingURL=pframe_pool.cjs.map

package/dist/pframe_pool.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"pframe_pool.cjs","sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n  assertNever,\n  bigintReplacer,\n  canonicalizeJson,\n  ensureError,\n  mapPObjectData,\n  PFrameDriverError,\n  type JsonSerializable,\n  type PColumn,\n  type PFrameHandle,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  hashJson,\n  RefCountPoolBase,\n  type PoolEntry,\n} from '@milaboratories/ts-helpers';\nimport { PFrameFactory } from '@milaboratories/pframes-rs-node';\nimport { mapValues } from 'es-toolkit';\nimport { logPFrames } from './logging';\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n  acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n  makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'>;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n  acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n  httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n  public readonly pFramePromise: Promise<PFrameInternal.PFrameV12>;\n  private readonly abortController = new AbortController();\n\n  private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n  private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n  constructor(\n    private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n    private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n    logger: PFrameInternal.Logger,\n    private readonly spillPath: string,\n    columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n  ) {\n    const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n      const localBlob = this.localBlobProvider.acquire(blob);\n      this.localBlobs.push(localBlob);\n      return localBlob.key;\n    };\n\n    const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n      const remoteBlob = this.remoteBlobProvider.acquire(blob);\n      this.remoteBlobs.push(remoteBlob);\n      return `${remoteBlob.key}${PFrameInternal.ParquetExtension}`;\n    };\n\n    const mapColumnData = (\n      data: PFrameInternal.DataInfo<TreeEntry>,\n    ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n      switch (data.type) {\n        case 'Json':\n          return { ...data };\n        case 'JsonPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, makeLocalBlobId),\n          };\n        case 'BinaryPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, (v) => ({\n              index: makeLocalBlobId(v.index),\n              values: makeLocalBlobId(v.values),\n            })),\n          };\n        case 'ParquetPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, (v) => ({\n              ...v,\n              data: makeRemoteBlobId(v.data),\n            })),\n          };\n        default:\n          assertNever(data);\n      }\n    };\n\n    const jsonifiedColumns = columns.map((column) => ({\n      ...column,\n      data: mapColumnData(column.data),\n    }));\n\n    try {\n      const pFrame = PFrameFactory.createPFrame({ spillPath: this.spillPath, logger });\n      pFrame.setDataSource({\n        ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n        parquetServer: this.remoteBlobProvider.httpServerInfo(),\n      });\n\n      const promises: Promise<void>[] = [];\n      for (const column of jsonifiedColumns) {\n        pFrame.addColumnSpec(column.id, column.spec);\n        promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));\n      }\n\n      this.pFramePromise = Promise.all(promises)\n        .then(() => pFrame)\n        .catch((err) => {\n          this.dispose();\n          pFrame.dispose();\n          throw new PFrameDriverError(\n            `PFrame creation failed asynchronously, `\n            + `columns: ${JSON.stringify(jsonifiedColumns)}, `\n            + `error: ${ensureError(err)}`,\n          );\n        });\n    } catch (err: unknown) {\n      throw new PFrameDriverError(\n        `PFrame creation failed synchronously, `\n        + `columns: ${JSON.stringify(jsonifiedColumns)}, `\n        + `error: ${ensureError(err)}`,\n      );\n    }\n  }\n\n  public get disposeSignal(): AbortSignal {\n    return this.abortController.signal;\n  }\n\n  private dispose(): void {\n    this.abortController.abort();\n    this.localBlobs.forEach((entry) => entry.unref());\n    this.remoteBlobs.forEach((entry) => entry.unref());\n  }\n\n  [Symbol.dispose](): void {\n    this.dispose();\n    void this.pFramePromise\n      .then((pFrame) => pFrame.dispose())\n      .catch(() => { /* mute error */ });\n  }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable>\n  extends RefCountPoolBase<\n    PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n    PFrameHandle,\n    PFrameHolder<TreeEntry>> {\n  constructor(\n    private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n    private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n    private readonly logger: PFrameInternal.Logger,\n    private readonly spillPath: string,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[]): PFrameHandle {\n    return stableKeyFromPFrameData(params);\n  }\n\n  protected createNewResource(\n    params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n    key: PFrameHandle,\n  ): PFrameHolder<TreeEntry> {\n    if (logPFrames()) {\n      this.logger('info',\n        `PFrame creation (pFrameHandle = ${key}): `\n        + `${JSON.stringify(params, bigintReplacer)}`,\n      );\n    }\n    return new PFrameHolder(\n      this.localBlobProvider,\n      this.remoteBlobProvider,\n      this.logger,\n      this.spillPath,\n      params,\n    );\n  }\n\n  public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n    const resource = super.tryGetByKey(key);\n    if (!resource) throw new PFrameDriverError(`PFrame not found, handle = ${key}`);\n    return resource;\n  }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n  data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n  const orderedData = [...data].map((column) =>\n    mapPObjectData(column, (r) => {\n      let result: {\n        type: string;\n        keyLength: number;\n        payload: {\n          key: string;\n          value: null | number | string | [string, string];\n        }[];\n      };\n      const type = r.type;\n      switch (type) {\n        case 'Json':\n          result = {\n            type: r.type,\n            keyLength: r.keyLength,\n            payload: Object.entries(r.data).map(([part, value]) => ({\n              key: part,\n              value,\n            })),\n          };\n          break;\n        case 'JsonPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: canonicalizeJson(info),\n            })),\n          };\n          break;\n        case 'BinaryPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n            })),\n          };\n          break;\n        case 'ParquetPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: info.dataDigest || [\n                canonicalizeJson(info.data),\n                JSON.stringify({ axes: info.axes, column: info.column }),\n              ] as const,\n            })),\n          };\n          break;\n        default:\n          throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type satisfies never)}`);\n      }\n      result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);\n      return result;\n    }),\n  );\n  orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);\n  return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"names":["PFrameInternal","mapValues","assertNever","PFrameFactory","PFrameDriverError","ensureError","RefCountPoolBase","logPFrames","bigintReplacer","mapPObjectData","canonicalizeJson","hashJson"],"mappings":";;;;;;;;;MA+Ba,YAAY,CAAA;AAQJ,IAAA,iBAAA;AACA,IAAA,kBAAA;AAEA,IAAA,SAAA;AAVH,IAAA,aAAa;AACZ,IAAA,eAAe,GAAG,IAAI,eAAe,EAAE;IAEvC,UAAU,GAA6C,EAAE;IACzD,WAAW,GAA6C,EAAE;IAE3E,WAAA,CACmB,iBAA+C,EAC/C,kBAAiD,EAClE,MAA6B,EACZ,SAAiB,EAClC,OAAsD,EAAA;QAJrC,IAAA,CAAA,iBAAiB,GAAjB,iBAAiB;QACjB,IAAA,CAAA,kBAAkB,GAAlB,kBAAkB;QAElB,IAAA,CAAA,SAAS,GAAT,SAAS;AAG1B,QAAA,MAAM,eAAe,GAAG,CAAC,IAAe,KAAiC;YACvE,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC;AACtD,YAAA,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;YAC/B,OAAO,SAAS,CAAC,GAAG;AACtB,QAAA,CAAC;AAED,QAAA,MAAM,gBAAgB,GAAG,CAAC,IAAe,KAAiC;YACxE,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC;AACxD,YAAA,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;YACjC,OAAO,CAAA,EAAG,UAAU,CAAC,GAAG,GAAGA,iCAAc,CAAC,gBAAgB,CAAA,CAAE;AAC9D,QAAA,CAAC;AAED,QAAA,MAAM,aAAa,GAAG,CACpB,IAAwC,KACgB;AACxD,YAAA,QAAQ,IAAI,CAAC,IAAI;AACf,gBAAA,KAAK,MAAM;AACT,oBAAA,OAAO,EAAE,GAAG,IAAI,EAAE;AACpB,gBAAA,KAAK,iBAAiB;oBACpB,OAAO;AACL,wBAAA,GAAG,IAAI;wBACP,KAAK,EAAEC,mBAAS,CAAC,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC;qBAC9C;AACH,gBAAA,KAAK,mBAAmB;oBACtB,OAAO;AACL,wBAAA,GAAG,IAAI;AACP,wBAAA,KAAK,EAAEA,mBAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,4BAAA,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC;AAC/B,4BAAA,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;AAClC,yBAAA,CAAC,CAAC;qBACJ;AACH,gBAAA,KAAK,oBAAoB;oBACvB,OAAO;AACL,wBAAA,GAAG,IAAI;AACP,wBAAA,KAAK,EAAEA,mBAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,4BAAA,GAAG,CAAC;AACJ,4BAAA,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/B,yBAAA,CAAC,CAAC;qBACJ;AACH,gBAAA;oBACEC,iBAAW,CAAC,IAAI,CAAC;;AAEvB,QAAA,CAAC;QAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,MAAM;AAChD,YAAA,GAAG,MAAM;AACT,YAAA,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC;AACjC,SAAA,CAAC,CAAC;AAEH,QAAA,IAAI;AACF,YAAA,MAAM,MAAM,GAAGC,2BAAa,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YAChF,MAAM,CAAC,aAAa,CAAC;gBACnB,GAAG,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC;AAC5D,gBAAA,aAAa,EAAE,IAAI,CAAC,kBAAkB,CAAC,cAAc,EAAE;AACxD,aAAA,CAAC;YAEF,MAAM,QAAQ,GAAoB,EAAE;AACpC,YAAA,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE;gBACrC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC;gBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;YAC7F;YAEA,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ;AACtC,iBAAA,IAAI,CAAC,MAAM,MAAM;AACjB,iBAAA,KAAK,CAAC,CAAC,GAAG,KAAI;gBACb,IAAI,CAAC,OAAO,EAAE;gBACd,MAAM,CAAC,OAAO,EAAE;gBAChB,MAAM,IAAIC,uBAAiB,CACzB,CAAA,uCAAA;AACE,sBAAA,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA,EAAA;AAC5C,sBAAA,CAAA,OAAA,EAAUC,iBAAW,CAAC,GAAG,CAAC,CAAA,CAAE,CAC/B;AACH,YAAA,CAAC,CAAC;QACN;QAAE,OAAO,GAAY,EAAE;YACrB,MAAM,IAAID,uBAAiB,CACzB,CAAA,sCAAA;AACE,kBAAA,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA,EAAA;AAC5C,kBAAA,CAAA,OAAA,EAAUC,iBAAW,CAAC,GAAG,CAAC,CAAA,CAAE,CAC/B;QACH;IACF;AAEA,IAAA,IAAW,aAAa,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM;IACpC;IAEQ,OAAO,GAAA;AACb,QAAA,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE;AAC5B,QAAA,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;AACjD,QAAA,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;IACpD;IAEA,CAAC,MAAM,CAAC,OAAO,CAAC,GAAA;QACd,IAAI,CAAC,OAAO,EAAE;QACd,KAAK,IAAI,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,EAAE;AACjC,aAAA,KAAK,CAAC,MAAK,EAAoB,CAAC,CAAC;IACtC;AACD;AAEK,MAAO,UACX,SAAQC,0BAGkB,CAAA;AAEP,IAAA,iBAAA;AACA,IAAA,kBAAA;AACA,IAAA,MAAA;AACA,IAAA,SAAA;AAJnB,IAAA,WAAA,CACmB,iBAA+C,EAC/C,kBAAiD,EACjD,MAA6B,EAC7B,SAAiB,EAAA;AAElC,QAAA,KAAK,EAAE;QALU,IAAA,CAAA,iBAAiB,GAAjB,iBAAiB;QACjB,IAAA,CAAA,kBAAkB,GAAlB,kBAAkB;QAClB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,SAAS,GAAT,SAAS;IAG5B;AAEU,IAAA,kBAAkB,CAAC,MAAqD,EAAA;AAChF,QAAA,OAAO,uBAAuB,CAAC,MAAM,CAAC;IACxC;IAEU,iBAAiB,CACzB,MAAqD,EACrD,GAAiB,EAAA;QAEjB,IAAIC,kBAAU,EAAE,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,gCAAA,EAAmC,GAAG,CAAA,GAAA;kBACpC,CAAA,EAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAEC,oBAAc,CAAC,CAAA,CAAE,CAC9C;QACH;QACA,OAAO,IAAI,YAAY,CACrB,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,MAAM,CACP;IACH;AAEO,IAAA,QAAQ,CAAC,GAAiB,EAAA;QAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC;AACvC,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAIJ,uBAAiB,CAAC,8BAA8B,GAAG,CAAA,CAAE,CAAC;AAC/E,QAAA,OAAO,QAAQ;IACjB;AACD;AAED,SAAS,uBAAuB,CAC9B,IAAmD,EAAA;IAEnD,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,KACvCK,oBAAc,CAAC,MAAM,EAAE,CAAC,CAAC,KAAI;AAC3B,QAAA,IAAI,MAOH;AACD,QAAA,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI;QACnB,QAAQ,IAAI;AACV,YAAA,KAAK,MAAM;AACT,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;wBACT,KAAK;AACN,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,iBAAiB;AACpB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAEC,sBAAgB,CAAC,IAAI,CAAC;AAC9B,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,mBAAmB;AACtB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAE,CAACA,sBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAEA,sBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAU;AAC9E,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,oBAAoB;AACvB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAE,IAAI,CAAC,UAAU,IAAI;AACxB,4BAAAA,sBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AAC3B,4BAAA,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AAChD,yBAAA;AACX,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA;AACE,gBAAA,MAAM,IAAIN,uBAAiB,CAAC,CAAA,2BAAA,EAA8B,IAAI,CAAC,SAAS,CAAC,IAAoB,CAAC,CAAA,CAAE,CAAC;;AAErG,QAAA,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;AAC7D,QAAA,OAAO,MAAM;IACf,CAAC,CAAC,CACH;IACD,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACxD,IAAA,OAAOO,kBAAQ,CAAC,WAAW,CAA2B;AACxD;;;;;"}

package/dist/pframe_pool.d.ts

@@ -0,0 +1,35 @@
+import { type JsonSerializable, type PColumn, type PFrameHandle } from '@platforma-sdk/model';
+import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { RefCountPoolBase, type PoolEntry } from '@milaboratories/ts-helpers';
+export interface LocalBlobProvider<TreeEntry extends JsonSerializable> {
+    acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;
+    makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'>;
+}
+export interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {
+    acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;
+    httpServerInfo(): PFrameInternal.HttpServerInfo;
+}
+export declare class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {
+    private readonly localBlobProvider;
+    private readonly remoteBlobProvider;
+    private readonly spillPath;
+    readonly pFramePromise: Promise<PFrameInternal.PFrameV12>;
+    private readonly abortController;
+    private readonly localBlobs;
+    private readonly remoteBlobs;
+    constructor(localBlobProvider: LocalBlobProvider<TreeEntry>, remoteBlobProvider: RemoteBlobProvider<TreeEntry>, logger: PFrameInternal.Logger, spillPath: string, columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[]);
+    get disposeSignal(): AbortSignal;
+    private dispose;
+    [Symbol.dispose](): void;
+}
+export declare class PFramePool<TreeEntry extends JsonSerializable> extends RefCountPoolBase<PColumn<PFrameInternal.DataInfo<TreeEntry>>[], PFrameHandle, PFrameHolder<TreeEntry>> {
+    private readonly localBlobProvider;
+    private readonly remoteBlobProvider;
+    private readonly logger;
+    private readonly spillPath;
+    constructor(localBlobProvider: LocalBlobProvider<TreeEntry>, remoteBlobProvider: RemoteBlobProvider<TreeEntry>, logger: PFrameInternal.Logger, spillPath: string);
+    protected calculateParamsKey(params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[]): PFrameHandle;
+    protected createNewResource(params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[], key: PFrameHandle): PFrameHolder<TreeEntry>;
+    getByKey(key: PFrameHandle): PFrameHolder<TreeEntry>;
+}
+//# sourceMappingURL=pframe_pool.d.ts.map

package/dist/pframe_pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pframe_pool.d.ts","sourceRoot":"","sources":["../src/pframe_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,gBAAgB,EACrB,KAAK,OAAO,EACZ,KAAK,YAAY,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAEL,gBAAgB,EAChB,KAAK,SAAS,EACf,MAAM,4BAA4B,CAAC;AAKpC,MAAM,WAAW,iBAAiB,CAAC,SAAS,SAAS,gBAAgB;IACnE,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;IACnE,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;CAC/F;AAED,MAAM,WAAW,kBAAkB,CAAC,SAAS,SAAS,gBAAgB;IACpE,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;IACnE,cAAc,IAAI,cAAc,CAAC,cAAc,CAAC;CACjD;AAED,qBAAa,YAAY,CAAC,SAAS,SAAS,gBAAgB,CAAE,YAAW,UAAU;IAQ/E,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAEnC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAV5B,SAAgB,aAAa,EAAE,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IACjE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IAEzD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgD;IAC3E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgD;gBAGzD,iBAAiB,EAAE,iBAAiB,CAAC,SAAS,CAAC,EAC/C,kBAAkB,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAClE,MAAM,EAAE,cAAc,CAAC,MAAM,EACZ,SAAS,EAAE,MAAM,EAClC,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE;IAoFxD,IAAW,aAAa,IAAI,WAAW,CAEtC;IAED,OAAO,CAAC,OAAO;IAMf,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAMzB;AAED,qBAAa,UAAU,CAAC,SAAS,SAAS,gBAAgB,CACxD,SAAQ,gBAAgB,CACtB,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,EAC7C,YAAY,EACZ,YAAY,CAAC,SAAS,CAAC,CAAC;IAExB,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS;gBAHT,iBAAiB,EAAE,iBAAiB,CAAC,SAAS,CAAC,EAC/C,kBAAkB,EAAE,kBAAkB,CAAC,SAAS,CAAC,EACjD,MAAM,EAAE,cAAc,CAAC,MAAM,EAC7B,SAAS,EAAE,MAAM;IAKpC,SAAS,CAAC,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,YAAY;IAIjG,SAAS,CAAC,iBAAiB,CACzB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,EACrD,GAAG,EAAE,YAAY,GAChB,YAAY,CAAC,SAAS,CAAC;IAgBnB,QAAQ,CAAC,GAAG,EAAE,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC;CAK5D"}

package/dist/pframe_pool.js

@@ -0,0 +1,193 @@
+import { bigintReplacer, PFrameDriverError, mapPObjectData, canonicalizeJson, ensureError, assertNever } from '@platforma-sdk/model';
+import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { RefCountPoolBase, hashJson } from '@milaboratories/ts-helpers';
+import { PFrameFactory } from '@milaboratories/pframes-rs-node';
+import { mapValues } from 'es-toolkit';
+import { logPFrames } from './logging.js';
+
+class PFrameHolder {
+    localBlobProvider;
+    remoteBlobProvider;
+    spillPath;
+    pFramePromise;
+    abortController = new AbortController();
+    localBlobs = [];
+    remoteBlobs = [];
+    constructor(localBlobProvider, remoteBlobProvider, logger, spillPath, columns) {
+        this.localBlobProvider = localBlobProvider;
+        this.remoteBlobProvider = remoteBlobProvider;
+        this.spillPath = spillPath;
+        const makeLocalBlobId = (blob) => {
+            const localBlob = this.localBlobProvider.acquire(blob);
+            this.localBlobs.push(localBlob);
+            return localBlob.key;
+        };
+        const makeRemoteBlobId = (blob) => {
+            const remoteBlob = this.remoteBlobProvider.acquire(blob);
+            this.remoteBlobs.push(remoteBlob);
+            return `${remoteBlob.key}${PFrameInternal.ParquetExtension}`;
+        };
+        const mapColumnData = (data) => {
+            switch (data.type) {
+                case 'Json':
+                    return { ...data };
+                case 'JsonPartitioned':
+                    return {
+                        ...data,
+                        parts: mapValues(data.parts, makeLocalBlobId),
+                    };
+                case 'BinaryPartitioned':
+                    return {
+                        ...data,
+                        parts: mapValues(data.parts, (v) => ({
+                            index: makeLocalBlobId(v.index),
+                            values: makeLocalBlobId(v.values),
+                        })),
+                    };
+                case 'ParquetPartitioned':
+                    return {
+                        ...data,
+                        parts: mapValues(data.parts, (v) => ({
+                            ...v,
+                            data: makeRemoteBlobId(v.data),
+                        })),
+                    };
+                default:
+                    assertNever(data);
+            }
+        };
+        const jsonifiedColumns = columns.map((column) => ({
+            ...column,
+            data: mapColumnData(column.data),
+        }));
+        try {
+            const pFrame = PFrameFactory.createPFrame({ spillPath: this.spillPath, logger });
+            pFrame.setDataSource({
+                ...this.localBlobProvider.makeDataSource(this.disposeSignal),
+                parquetServer: this.remoteBlobProvider.httpServerInfo(),
+            });
+            const promises = [];
+            for (const column of jsonifiedColumns) {
+                pFrame.addColumnSpec(column.id, column.spec);
+                promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));
+            }
+            this.pFramePromise = Promise.all(promises)
+                .then(() => pFrame)
+                .catch((err) => {
+                this.dispose();
+                pFrame.dispose();
+                throw new PFrameDriverError(`PFrame creation failed asynchronously, `
+                    + `columns: ${JSON.stringify(jsonifiedColumns)}, `
+                    + `error: ${ensureError(err)}`);
+            });
+        }
+        catch (err) {
+            throw new PFrameDriverError(`PFrame creation failed synchronously, `
+                + `columns: ${JSON.stringify(jsonifiedColumns)}, `
+                + `error: ${ensureError(err)}`);
+        }
+    }
+    get disposeSignal() {
+        return this.abortController.signal;
+    }
+    dispose() {
+        this.abortController.abort();
+        this.localBlobs.forEach((entry) => entry.unref());
+        this.remoteBlobs.forEach((entry) => entry.unref());
+    }
+    [Symbol.dispose]() {
+        this.dispose();
+        void this.pFramePromise
+            .then((pFrame) => pFrame.dispose())
+            .catch(() => { });
+    }
+}
+class PFramePool extends RefCountPoolBase {
+    localBlobProvider;
+    remoteBlobProvider;
+    logger;
+    spillPath;
+    constructor(localBlobProvider, remoteBlobProvider, logger, spillPath) {
+        super();
+        this.localBlobProvider = localBlobProvider;
+        this.remoteBlobProvider = remoteBlobProvider;
+        this.logger = logger;
+        this.spillPath = spillPath;
+    }
+    calculateParamsKey(params) {
+        return stableKeyFromPFrameData(params);
+    }
+    createNewResource(params, key) {
+        if (logPFrames()) {
+            this.logger('info', `PFrame creation (pFrameHandle = ${key}): `
+                + `${JSON.stringify(params, bigintReplacer)}`);
+        }
+        return new PFrameHolder(this.localBlobProvider, this.remoteBlobProvider, this.logger, this.spillPath, params);
+    }
+    getByKey(key) {
+        const resource = super.tryGetByKey(key);
+        if (!resource)
+            throw new PFrameDriverError(`PFrame not found, handle = ${key}`);
+        return resource;
+    }
+}
+function stableKeyFromPFrameData(data) {
+    const orderedData = [...data].map((column) => mapPObjectData(column, (r) => {
+        let result;
+        const type = r.type;
+        switch (type) {
+            case 'Json':
+                result = {
+                    type: r.type,
+                    keyLength: r.keyLength,
+                    payload: Object.entries(r.data).map(([part, value]) => ({
+                        key: part,
+                        value,
+                    })),
+                };
+                break;
+            case 'JsonPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: canonicalizeJson(info),
+                    })),
+                };
+                break;
+            case 'BinaryPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: [canonicalizeJson(info.index), canonicalizeJson(info.values)],
+                    })),
+                };
+                break;
+            case 'ParquetPartitioned':
+                result = {
+                    type: r.type,
+                    keyLength: r.partitionKeyLength,
+                    payload: Object.entries(r.parts).map(([part, info]) => ({
+                        key: part,
+                        value: info.dataDigest || [
+                            canonicalizeJson(info.data),
+                            JSON.stringify({ axes: info.axes, column: info.column }),
+                        ],
+                    })),
+                };
+                break;
+            default:
+                throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type)}`);
+        }
+        result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);
+        return result;
+    }));
+    orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);
+    return hashJson(orderedData);
+}
+
+export { PFrameHolder, PFramePool };
+//# sourceMappingURL=pframe_pool.js.map

package/dist/pframe_pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pframe_pool.js","sources":["../src/pframe_pool.ts"],"sourcesContent":["import {\n  assertNever,\n  bigintReplacer,\n  canonicalizeJson,\n  ensureError,\n  mapPObjectData,\n  PFrameDriverError,\n  type JsonSerializable,\n  type PColumn,\n  type PFrameHandle,\n} from '@platforma-sdk/model';\nimport { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport {\n  hashJson,\n  RefCountPoolBase,\n  type PoolEntry,\n} from '@milaboratories/ts-helpers';\nimport { PFrameFactory } from '@milaboratories/pframes-rs-node';\nimport { mapValues } from 'es-toolkit';\nimport { logPFrames } from './logging';\n\nexport interface LocalBlobProvider<TreeEntry extends JsonSerializable> {\n  acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n  makeDataSource(signal: AbortSignal): Omit<PFrameInternal.PFrameDataSourceV2, 'parquetServer'>;\n}\n\nexport interface RemoteBlobProvider<TreeEntry extends JsonSerializable> {\n  acquire(params: TreeEntry): PoolEntry<PFrameInternal.PFrameBlobId>;\n  httpServerInfo(): PFrameInternal.HttpServerInfo;\n}\n\nexport class PFrameHolder<TreeEntry extends JsonSerializable> implements Disposable {\n  public readonly pFramePromise: Promise<PFrameInternal.PFrameV12>;\n  private readonly abortController = new AbortController();\n\n  private readonly localBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n  private readonly remoteBlobs: PoolEntry<PFrameInternal.PFrameBlobId>[] = [];\n\n  constructor(\n    private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n    private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n    logger: PFrameInternal.Logger,\n    private readonly spillPath: string,\n    columns: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n  ) {\n    const makeLocalBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n      const localBlob = this.localBlobProvider.acquire(blob);\n      this.localBlobs.push(localBlob);\n      return localBlob.key;\n    };\n\n    const makeRemoteBlobId = (blob: TreeEntry): PFrameInternal.PFrameBlobId => {\n      const remoteBlob = this.remoteBlobProvider.acquire(blob);\n      this.remoteBlobs.push(remoteBlob);\n      return `${remoteBlob.key}${PFrameInternal.ParquetExtension}`;\n    };\n\n    const mapColumnData = (\n      data: PFrameInternal.DataInfo<TreeEntry>,\n    ): PFrameInternal.DataInfo<PFrameInternal.PFrameBlobId> => {\n      switch (data.type) {\n        case 'Json':\n          return { ...data };\n        case 'JsonPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, makeLocalBlobId),\n          };\n        case 'BinaryPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, (v) => ({\n              index: makeLocalBlobId(v.index),\n              values: makeLocalBlobId(v.values),\n            })),\n          };\n        case 'ParquetPartitioned':\n          return {\n            ...data,\n            parts: mapValues(data.parts, (v) => ({\n              ...v,\n              data: makeRemoteBlobId(v.data),\n            })),\n          };\n        default:\n          assertNever(data);\n      }\n    };\n\n    const jsonifiedColumns = columns.map((column) => ({\n      ...column,\n      data: mapColumnData(column.data),\n    }));\n\n    try {\n      const pFrame = PFrameFactory.createPFrame({ spillPath: this.spillPath, logger });\n      pFrame.setDataSource({\n        ...this.localBlobProvider.makeDataSource(this.disposeSignal),\n        parquetServer: this.remoteBlobProvider.httpServerInfo(),\n      });\n\n      const promises: Promise<void>[] = [];\n      for (const column of jsonifiedColumns) {\n        pFrame.addColumnSpec(column.id, column.spec);\n        promises.push(pFrame.setColumnData(column.id, column.data, { signal: this.disposeSignal }));\n      }\n\n      this.pFramePromise = Promise.all(promises)\n        .then(() => pFrame)\n        .catch((err) => {\n          this.dispose();\n          pFrame.dispose();\n          throw new PFrameDriverError(\n            `PFrame creation failed asynchronously, `\n            + `columns: ${JSON.stringify(jsonifiedColumns)}, `\n            + `error: ${ensureError(err)}`,\n          );\n        });\n    } catch (err: unknown) {\n      throw new PFrameDriverError(\n        `PFrame creation failed synchronously, `\n        + `columns: ${JSON.stringify(jsonifiedColumns)}, `\n        + `error: ${ensureError(err)}`,\n      );\n    }\n  }\n\n  public get disposeSignal(): AbortSignal {\n    return this.abortController.signal;\n  }\n\n  private dispose(): void {\n    this.abortController.abort();\n    this.localBlobs.forEach((entry) => entry.unref());\n    this.remoteBlobs.forEach((entry) => entry.unref());\n  }\n\n  [Symbol.dispose](): void {\n    this.dispose();\n    void this.pFramePromise\n      .then((pFrame) => pFrame.dispose())\n      .catch(() => { /* mute error */ });\n  }\n}\n\nexport class PFramePool<TreeEntry extends JsonSerializable>\n  extends RefCountPoolBase<\n    PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n    PFrameHandle,\n    PFrameHolder<TreeEntry>> {\n  constructor(\n    private readonly localBlobProvider: LocalBlobProvider<TreeEntry>,\n    private readonly remoteBlobProvider: RemoteBlobProvider<TreeEntry>,\n    private readonly logger: PFrameInternal.Logger,\n    private readonly spillPath: string,\n  ) {\n    super();\n  }\n\n  protected calculateParamsKey(params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[]): PFrameHandle {\n    return stableKeyFromPFrameData(params);\n  }\n\n  protected createNewResource(\n    params: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n    key: PFrameHandle,\n  ): PFrameHolder<TreeEntry> {\n    if (logPFrames()) {\n      this.logger('info',\n        `PFrame creation (pFrameHandle = ${key}): `\n        + `${JSON.stringify(params, bigintReplacer)}`,\n      );\n    }\n    return new PFrameHolder(\n      this.localBlobProvider,\n      this.remoteBlobProvider,\n      this.logger,\n      this.spillPath,\n      params,\n    );\n  }\n\n  public getByKey(key: PFrameHandle): PFrameHolder<TreeEntry> {\n    const resource = super.tryGetByKey(key);\n    if (!resource) throw new PFrameDriverError(`PFrame not found, handle = ${key}`);\n    return resource;\n  }\n}\n\nfunction stableKeyFromPFrameData<TreeEntry extends JsonSerializable>(\n  data: PColumn<PFrameInternal.DataInfo<TreeEntry>>[],\n): PFrameHandle {\n  const orderedData = [...data].map((column) =>\n    mapPObjectData(column, (r) => {\n      let result: {\n        type: string;\n        keyLength: number;\n        payload: {\n          key: string;\n          value: null | number | string | [string, string];\n        }[];\n      };\n      const type = r.type;\n      switch (type) {\n        case 'Json':\n          result = {\n            type: r.type,\n            keyLength: r.keyLength,\n            payload: Object.entries(r.data).map(([part, value]) => ({\n              key: part,\n              value,\n            })),\n          };\n          break;\n        case 'JsonPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: canonicalizeJson(info),\n            })),\n          };\n          break;\n        case 'BinaryPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: [canonicalizeJson(info.index), canonicalizeJson(info.values)] as const,\n            })),\n          };\n          break;\n        case 'ParquetPartitioned':\n          result = {\n            type: r.type,\n            keyLength: r.partitionKeyLength,\n            payload: Object.entries(r.parts).map(([part, info]) => ({\n              key: part,\n              value: info.dataDigest || [\n                canonicalizeJson(info.data),\n                JSON.stringify({ axes: info.axes, column: info.column }),\n              ] as const,\n            })),\n          };\n          break;\n        default:\n          throw new PFrameDriverError(`unsupported resource type: ${JSON.stringify(type satisfies never)}`);\n      }\n      result.payload.sort((lhs, rhs) => lhs.key < rhs.key ? -1 : 1);\n      return result;\n    }),\n  );\n  orderedData.sort((lhs, rhs) => lhs.id < rhs.id ? -1 : 1);\n  return hashJson(orderedData) as string as PFrameHandle;\n}\n"],"names":[],"mappings":";;;;;;;MA+Ba,YAAY,CAAA;AAQJ,IAAA,iBAAA;AACA,IAAA,kBAAA;AAEA,IAAA,SAAA;AAVH,IAAA,aAAa;AACZ,IAAA,eAAe,GAAG,IAAI,eAAe,EAAE;IAEvC,UAAU,GAA6C,EAAE;IACzD,WAAW,GAA6C,EAAE;IAE3E,WAAA,CACmB,iBAA+C,EAC/C,kBAAiD,EAClE,MAA6B,EACZ,SAAiB,EAClC,OAAsD,EAAA;QAJrC,IAAA,CAAA,iBAAiB,GAAjB,iBAAiB;QACjB,IAAA,CAAA,kBAAkB,GAAlB,kBAAkB;QAElB,IAAA,CAAA,SAAS,GAAT,SAAS;AAG1B,QAAA,MAAM,eAAe,GAAG,CAAC,IAAe,KAAiC;YACvE,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC;AACtD,YAAA,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC;YAC/B,OAAO,SAAS,CAAC,GAAG;AACtB,QAAA,CAAC;AAED,QAAA,MAAM,gBAAgB,GAAG,CAAC,IAAe,KAAiC;YACxE,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC;AACxD,YAAA,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;YACjC,OAAO,CAAA,EAAG,UAAU,CAAC,GAAG,GAAG,cAAc,CAAC,gBAAgB,CAAA,CAAE;AAC9D,QAAA,CAAC;AAED,QAAA,MAAM,aAAa,GAAG,CACpB,IAAwC,KACgB;AACxD,YAAA,QAAQ,IAAI,CAAC,IAAI;AACf,gBAAA,KAAK,MAAM;AACT,oBAAA,OAAO,EAAE,GAAG,IAAI,EAAE;AACpB,gBAAA,KAAK,iBAAiB;oBACpB,OAAO;AACL,wBAAA,GAAG,IAAI;wBACP,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC;qBAC9C;AACH,gBAAA,KAAK,mBAAmB;oBACtB,OAAO;AACL,wBAAA,GAAG,IAAI;AACP,wBAAA,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,4BAAA,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC;AAC/B,4BAAA,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;AAClC,yBAAA,CAAC,CAAC;qBACJ;AACH,gBAAA,KAAK,oBAAoB;oBACvB,OAAO;AACL,wBAAA,GAAG,IAAI;AACP,wBAAA,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM;AACnC,4BAAA,GAAG,CAAC;AACJ,4BAAA,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/B,yBAAA,CAAC,CAAC;qBACJ;AACH,gBAAA;oBACE,WAAW,CAAC,IAAI,CAAC;;AAEvB,QAAA,CAAC;QAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,MAAM;AAChD,YAAA,GAAG,MAAM;AACT,YAAA,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC;AACjC,SAAA,CAAC,CAAC;AAEH,QAAA,IAAI;AACF,YAAA,MAAM,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YAChF,MAAM,CAAC,aAAa,CAAC;gBACnB,GAAG,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC;AAC5D,gBAAA,aAAa,EAAE,IAAI,CAAC,kBAAkB,CAAC,cAAc,EAAE;AACxD,aAAA,CAAC;YAEF,MAAM,QAAQ,GAAoB,EAAE;AACpC,YAAA,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE;gBACrC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC;gBAC5C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;YAC7F;YAEA,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ;AACtC,iBAAA,IAAI,CAAC,MAAM,MAAM;AACjB,iBAAA,KAAK,CAAC,CAAC,GAAG,KAAI;gBACb,IAAI,CAAC,OAAO,EAAE;gBACd,MAAM,CAAC,OAAO,EAAE;gBAChB,MAAM,IAAI,iBAAiB,CACzB,CAAA,uCAAA;AACE,sBAAA,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA,EAAA;AAC5C,sBAAA,CAAA,OAAA,EAAU,WAAW,CAAC,GAAG,CAAC,CAAA,CAAE,CAC/B;AACH,YAAA,CAAC,CAAC;QACN;QAAE,OAAO,GAAY,EAAE;YACrB,MAAM,IAAI,iBAAiB,CACzB,CAAA,sCAAA;AACE,kBAAA,CAAA,SAAA,EAAY,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA,EAAA;AAC5C,kBAAA,CAAA,OAAA,EAAU,WAAW,CAAC,GAAG,CAAC,CAAA,CAAE,CAC/B;QACH;IACF;AAEA,IAAA,IAAW,aAAa,GAAA;AACtB,QAAA,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM;IACpC;IAEQ,OAAO,GAAA;AACb,QAAA,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE;AAC5B,QAAA,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;AACjD,QAAA,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;IACpD;IAEA,CAAC,MAAM,CAAC,OAAO,CAAC,GAAA;QACd,IAAI,CAAC,OAAO,EAAE;QACd,KAAK,IAAI,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,EAAE;AACjC,aAAA,KAAK,CAAC,MAAK,EAAoB,CAAC,CAAC;IACtC;AACD;AAEK,MAAO,UACX,SAAQ,gBAGkB,CAAA;AAEP,IAAA,iBAAA;AACA,IAAA,kBAAA;AACA,IAAA,MAAA;AACA,IAAA,SAAA;AAJnB,IAAA,WAAA,CACmB,iBAA+C,EAC/C,kBAAiD,EACjD,MAA6B,EAC7B,SAAiB,EAAA;AAElC,QAAA,KAAK,EAAE;QALU,IAAA,CAAA,iBAAiB,GAAjB,iBAAiB;QACjB,IAAA,CAAA,kBAAkB,GAAlB,kBAAkB;QAClB,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,SAAS,GAAT,SAAS;IAG5B;AAEU,IAAA,kBAAkB,CAAC,MAAqD,EAAA;AAChF,QAAA,OAAO,uBAAuB,CAAC,MAAM,CAAC;IACxC;IAEU,iBAAiB,CACzB,MAAqD,EACrD,GAAiB,EAAA;QAEjB,IAAI,UAAU,EAAE,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,MAAM,EAChB,CAAA,gCAAA,EAAmC,GAAG,CAAA,GAAA;kBACpC,CAAA,EAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA,CAAE,CAC9C;QACH;QACA,OAAO,IAAI,YAAY,CACrB,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,MAAM,CACP;IACH;AAEO,IAAA,QAAQ,CAAC,GAAiB,EAAA;QAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC;AACvC,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,iBAAiB,CAAC,8BAA8B,GAAG,CAAA,CAAE,CAAC;AAC/E,QAAA,OAAO,QAAQ;IACjB;AACD;AAED,SAAS,uBAAuB,CAC9B,IAAmD,EAAA;IAEnD,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,KACvC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,KAAI;AAC3B,QAAA,IAAI,MAOH;AACD,QAAA,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI;QACnB,QAAQ,IAAI;AACV,YAAA,KAAK,MAAM;AACT,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;wBACT,KAAK;AACN,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,iBAAiB;AACpB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC;AAC9B,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,mBAAmB;AACtB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAU;AAC9E,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA,KAAK,oBAAoB;AACvB,gBAAA,MAAM,GAAG;oBACP,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,kBAAkB;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM;AACtD,wBAAA,GAAG,EAAE,IAAI;AACT,wBAAA,KAAK,EAAE,IAAI,CAAC,UAAU,IAAI;AACxB,4BAAA,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;AAC3B,4BAAA,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AAChD,yBAAA;AACX,qBAAA,CAAC,CAAC;iBACJ;gBACD;AACF,YAAA;AACE,gBAAA,MAAM,IAAI,iBAAiB,CAAC,CAAA,2BAAA,EAA8B,IAAI,CAAC,SAAS,CAAC,IAAoB,CAAC,CAAA,CAAE,CAAC;;AAErG,QAAA,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;AAC7D,QAAA,OAAO,MAAM;IACf,CAAC,CAAC,CACH;IACD,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACxD,IAAA,OAAO,QAAQ,CAAC,WAAW,CAA2B;AACxD;;;;"}

package/dist/ptable_cache_per_frame.cjs

@@ -0,0 +1,69 @@
+'use strict';
+
+var lruCache = require('lru-cache');
+var logging = require('./logging.cjs');
+
+const PTableCachePerFrameOpsDefaults = {
+    pFrameCacheMaxCount: 18, // SHM trees create 3 PTables per graphic, we want to cache 6 graphics per PFrame
+    pFramesCacheMaxSize: 8 * 1024 * 1024 * 1024, // 8 GB, same as blob driver cache (must be at least 2GB)
+};
+class PTableCachePerFrame {
+    logger;
+    ops;
+    perFrame = new Map();
+    global;
+    disposeListeners = new Set();
+    constructor(logger, ops) {
+        this.logger = logger;
+        this.ops = ops;
+        this.global = new lruCache.LRUCache({
+            maxSize: this.ops.pFramesCacheMaxSize,
+            dispose: (resource, key, reason) => {
+                if (reason === 'evict') {
+                    this.perFrame.get(resource.resource.pFrame)?.delete(key);
+                }
+                if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {
+                    this.perFrame.delete(resource.resource.pFrame);
+                }
+                resource.unref();
+                if (logging.logPFrames()) {
+                    logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);
+                }
+            },
+        });
+    }
+    cache(resource, size) {
+        const key = resource.key;
+        if (logging.logPFrames()) {
+            this.logger('info', `calculateTableData cache - added PTable ${key} with size ${size}`);
+        }
+        this.global.set(key, resource, { size: Math.max(size, 1) }); // 1 is minimum size to avoid cache evictions
+        let perFrame = this.perFrame.get(resource.resource.pFrame);
+        if (!perFrame) {
+            perFrame = new lruCache.LRUCache({
+                max: this.ops.pFrameCacheMaxCount,
+                dispose: (_resource, key, reason) => {
+                    if (reason === 'evict') {
+                        this.global.delete(key);
+                    }
+                },
+            });
+            this.perFrame.set(resource.resource.pFrame, perFrame);
+        }
+        perFrame.set(key, resource);
+        if (!this.disposeListeners.has(key)) {
+            const disposeListener = () => {
+                this.perFrame.get(resource.resource.pFrame)?.delete(key);
+                this.global.delete(key);
+                this.disposeListeners.delete(key);
+                resource.resource.disposeSignal.removeEventListener('abort', disposeListener);
+            };
+            this.disposeListeners.add(key);
+            resource.resource.disposeSignal.addEventListener('abort', disposeListener);
+        }
+    }
+}
+
+exports.PTableCachePerFrame = PTableCachePerFrame;
+exports.PTableCachePerFrameOpsDefaults = PTableCachePerFrameOpsDefaults;
+//# sourceMappingURL=ptable_cache_per_frame.cjs.map

package/dist/ptable_cache_per_frame.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_cache_per_frame.cjs","sources":["../src/ptable_cache_per_frame.ts"],"sourcesContent":["import type { PFrameHandle, PTableHandle } from '@platforma-sdk/model';\nimport type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';\nimport type { PoolEntry } from '@milaboratories/ts-helpers';\nimport { LRUCache } from 'lru-cache';\nimport { logPFrames } from './logging';\nimport type { PTableHolder } from './ptable_pool';\n\nexport type PTableCachePerFrameOps = {\n  /** Maximum number of `calculateTableData` results cached for each PFrame */\n  pFrameCacheMaxCount: number;\n  /**\n   * Maximum size of `calculateTableData` results cached for PFrames overall.\n   * The limit is soft, as the same table could be materialized with other requests and will not be deleted in such case.\n   * Also each table has predeccessors, overlapping predecessors will be counted twice, so the effective limit is smaller.\n   */\n  pFramesCacheMaxSize: number;\n};\n\nexport const PTableCachePerFrameOpsDefaults: PTableCachePerFrameOps = {\n  pFrameCacheMaxCount: 18, // SHM trees create 3 PTables per graphic, we want to cache 6 graphics per PFrame\n  pFramesCacheMaxSize: 8 * 1024 * 1024 * 1024, // 8 GB, same as blob driver cache (must be at least 2GB)\n};\n\nexport class PTableCachePerFrame {\n  private readonly perFrame = new Map<PFrameHandle, LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>>();\n  private readonly global: LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>;\n  private readonly disposeListeners = new Set<PTableHandle>();\n\n  constructor(\n    private readonly logger: PFrameInternal.Logger,\n    private readonly ops: PTableCachePerFrameOps,\n  ) {\n    this.global = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({\n      maxSize: this.ops.pFramesCacheMaxSize,\n      dispose: (resource, key, reason) => {\n        if (reason === 'evict') {\n          this.perFrame.get(resource.resource.pFrame)?.delete(key);\n        }\n\n        if (this.perFrame.get(resource.resource.pFrame)?.size === 0) {\n          this.perFrame.delete(resource.resource.pFrame);\n        }\n\n        resource.unref();\n        if (logPFrames()) {\n          logger('info', `calculateTableData cache - removed PTable ${key} (reason: ${reason})`);\n        }\n      },\n    });\n  }\n\n  public cache(resource: PoolEntry<PTableHandle, PTableHolder>, size: number): void {\n    const key = resource.key;\n    if (logPFrames()) {\n      this.logger('info', `calculateTableData cache - added PTable ${key} with size ${size}`);\n    }\n\n    this.global.set(key, resource, { size: Math.max(size, 1) }); // 1 is minimum size to avoid cache evictions\n\n    let perFrame = this.perFrame.get(resource.resource.pFrame);\n    if (!perFrame) {\n      perFrame = new LRUCache<PTableHandle, PoolEntry<PTableHandle, PTableHolder>>({\n        max: this.ops.pFrameCacheMaxCount,\n        dispose: (_resource, key, reason) => {\n          if (reason === 'evict') {\n            this.global.delete(key);\n          }\n        },\n      });\n      this.perFrame.set(resource.resource.pFrame, perFrame);\n    }\n    perFrame.set(key, resource);\n\n    if (!this.disposeListeners.has(key)) {\n      const disposeListener = () => {\n        this.perFrame.get(resource.resource.pFrame)?.delete(key);\n        this.global.delete(key);\n\n        this.disposeListeners.delete(key);\n        resource.resource.disposeSignal.removeEventListener('abort', disposeListener);\n      };\n      this.disposeListeners.add(key);\n      resource.resource.disposeSignal.addEventListener('abort', disposeListener);\n    }\n  }\n}\n"],"names":["LRUCache","logPFrames"],"mappings":";;;;;AAkBO,MAAM,8BAA8B,GAA2B;IACpE,mBAAmB,EAAE,EAAE;IACvB,mBAAmB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;;MAGhC,mBAAmB,CAAA;AAMX,IAAA,MAAA;AACA,IAAA,GAAA;AANF,IAAA,QAAQ,GAAG,IAAI,GAAG,EAA+E;AACjG,IAAA,MAAM;AACN,IAAA,gBAAgB,GAAG,IAAI,GAAG,EAAgB;IAE3D,WAAA,CACmB,MAA6B,EAC7B,GAA2B,EAAA;QAD3B,IAAA,CAAA,MAAM,GAAN,MAAM;QACN,IAAA,CAAA,GAAG,GAAH,GAAG;AAEpB,QAAA,IAAI,CAAC,MAAM,GAAG,IAAIA,iBAAQ,CAAsD;AAC9E,YAAA,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB;YACrC,OAAO,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,KAAI;AACjC,gBAAA,IAAI,MAAM,KAAK,OAAO,EAAE;AACtB,oBAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC;gBAC1D;AAEA,gBAAA,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE;oBAC3D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAChD;gBAEA,QAAQ,CAAC,KAAK,EAAE;gBAChB,IAAIC,kBAAU,EAAE,EAAE;oBAChB,MAAM,CAAC,MAAM,EAAE,CAAA,0CAAA,EAA6C,GAAG,CAAA,UAAA,EAAa,MAAM,CAAA,CAAA,CAAG,CAAC;gBACxF;YACF,CAAC;AACF,SAAA,CAAC;IACJ;IAEO,KAAK,CAAC,QAA+C,EAAE,IAAY,EAAA;AACxE,QAAA,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG;QACxB,IAAIA,kBAAU,EAAE,EAAE;YAChB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA,wCAAA,EAA2C,GAAG,CAAA,WAAA,EAAc,IAAI,CAAA,CAAE,CAAC;QACzF;QAEA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;AAE5D,QAAA,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC1D,IAAI,CAAC,QAAQ,EAAE;YACb,QAAQ,GAAG,IAAID,iBAAQ,CAAsD;AAC3E,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB;gBACjC,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,MAAM,KAAI;AAClC,oBAAA,IAAI,MAAM,KAAK,OAAO,EAAE;AACtB,wBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;oBACzB;gBACF,CAAC;AACF,aAAA,CAAC;AACF,YAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;QACvD;AACA,QAAA,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC;QAE3B,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACnC,MAAM,eAAe,GAAG,MAAK;AAC3B,gBAAA,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC;AACxD,gBAAA,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;AAEvB,gBAAA,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC;gBACjC,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC;AAC/E,YAAA,CAAC;AACD,YAAA,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC;YAC9B,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC;QAC5E;IACF;AACD;;;;;"}

package/dist/ptable_cache_per_frame.d.ts

@@ -0,0 +1,25 @@
+import type { PTableHandle } from '@platforma-sdk/model';
+import type { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import type { PoolEntry } from '@milaboratories/ts-helpers';
+import type { PTableHolder } from './ptable_pool';
+export type PTableCachePerFrameOps = {
+    /** Maximum number of `calculateTableData` results cached for each PFrame */
+    pFrameCacheMaxCount: number;
+    /**
+     * Maximum size of `calculateTableData` results cached for PFrames overall.
+     * The limit is soft, as the same table could be materialized with other requests and will not be deleted in such case.
+     * Also each table has predeccessors, overlapping predecessors will be counted twice, so the effective limit is smaller.
+     */
+    pFramesCacheMaxSize: number;
+};
+export declare const PTableCachePerFrameOpsDefaults: PTableCachePerFrameOps;
+export declare class PTableCachePerFrame {
+    private readonly logger;
+    private readonly ops;
+    private readonly perFrame;
+    private readonly global;
+    private readonly disposeListeners;
+    constructor(logger: PFrameInternal.Logger, ops: PTableCachePerFrameOps);
+    cache(resource: PoolEntry<PTableHandle, PTableHolder>, size: number): void;
+}
+//# sourceMappingURL=ptable_cache_per_frame.d.ts.map

package/dist/ptable_cache_per_frame.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ptable_cache_per_frame.d.ts","sourceRoot":"","sources":["../src/ptable_cache_per_frame.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAgB,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAG5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,MAAM,sBAAsB,GAAG;IACnC,4EAA4E;IAC5E,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,sBAG5C,CAAC;AAEF,qBAAa,mBAAmB;IAM5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IANtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0F;IACnH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgE;IACvF,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA2B;gBAGzC,MAAM,EAAE,cAAc,CAAC,MAAM,EAC7B,GAAG,EAAE,sBAAsB;IAqBvC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;CAkClF"}