@milaboratories/pl-middle-layer 1.36.1 → 1.36.3

package/dist/middle_layer/driver_kit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver_kit.d.ts","sourceRoot":"","sources":["../../src/middle_layer/driver_kit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EACV,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAKL,cAAc,EACd,uBAAuB,EACvB,UAAU,EAGV,YAAY,EACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,KAAK,GAAG,MAAM,iCAAiC,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,KAAK,EAEV,uBAAuB,EACxB,MAAM,OAAO,CAAC;AAMf;;;;;;KAMK;AACL,MAAM,WAAW,oBAAqB,SAAQ,GAAG,CAAC,SAAS;IAEzD,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;IAEpC,QAAQ,CAAC,eAAe,EAAE,uBAAuB,CAAC;IAElD,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAE/B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC,QAAQ,CAAC,YAAY,EAAE,oBAAoB,CAAC;IAE5C;;;SAGK;IACL,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB;;;SAGK;IACL,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CA4D/B"}
+{"version":3,"file":"driver_kit.d.ts","sourceRoot":"","sources":["../../src/middle_layer/driver_kit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EACV,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAKL,cAAc,EACd,uBAAuB,EACvB,UAAU,EAGV,YAAY,EACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,KAAK,GAAG,MAAM,iCAAiC,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,KAAK,EAEV,uBAAuB,EACxB,MAAM,OAAO,CAAC;AAMf;;;;;;KAMK;AACL,MAAM,WAAW,oBAAqB,SAAQ,GAAG,CAAC,SAAS;IAEzD,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;IAEpC,QAAQ,CAAC,eAAe,EAAE,uBAAuB,CAAC;IAElD,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAE/B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC,QAAQ,CAAC,YAAY,EAAE,oBAAoB,CAAC;IAE5C;;;SAGK;IACL,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB;;;SAGK;IACL,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAgE/B"}

package/dist/middle_layer/ops.d.ts

@@ -25,6 +25,8 @@ export type DriverKitOpsPaths = {
      * If undefined, default list will be created.
      * */
     readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];
+    /** Path to the directory where pframes will spill temporary files and store materialized views */
+    readonly pframesSpillPath: string;
 };
 /** Options required to initialize full set of middle layer driver kit */
 export type DriverKitOpsSettings = {
@@ -59,7 +61,7 @@ export type DriverKitOpsSettings = {
 export type DriverKitOps = DriverKitOpsPaths & DriverKitOpsSettings;
 /** Some defaults fot MiddleLayerOps. */
 export declare const DefaultDriverKitOpsSettings: Pick<DriverKitOpsSettings, 'logger' | 'blobDriverOps' | 'downloadBlobToURLDriverOps' | 'uploadDriverOps' | 'logStreamDriverOps'>;
-export declare function DefaultDriverKitOpsPaths(workDir: string): Pick<DriverKitOpsPaths, 'blobDownloadPath' | 'downloadBlobToURLPath'>;
+export declare function DefaultDriverKitOpsPaths(workDir: string): Pick<DriverKitOpsPaths, 'blobDownloadPath' | 'downloadBlobToURLPath' | 'pframesSpillPath'>;
 /** Fields with default values are marked as optional here. */
 export type DriverKitOpsConstructor = Omit<DriverKitOpsSettings, keyof typeof DefaultDriverKitOpsSettings> & Partial<typeof DefaultDriverKitOpsSettings> & Omit<DriverKitOpsPaths, keyof ReturnType<typeof DefaultDriverKitOpsPaths>> & Partial<ReturnType<typeof DefaultDriverKitOpsPaths>>;
 export type MiddleLayerOpsPaths = DriverKitOpsPaths & {

package/dist/middle_layer/ops.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ops.d.ts","sourceRoot":"","sources":["../../src/middle_layer/ops.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EACV,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,uBAAuB,EACxB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAGzE,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC,qDAAqD;IACrD,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAEvC;;;;;;;;;;;SAWK;IACL,QAAQ,CAAC,gBAAgB,EAAE,sBAAsB,EAAE,CAAC;IAEpD;;;SAGK;IACL,QAAQ,CAAC,4BAA4B,CAAC,EAAE,uBAAuB,EAAE,CAAC;CACnE,CAAC;AAEF,yEAAyE;AACzE,MAAM,MAAM,oBAAoB,GAAG;IAKjC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAM1B;;;;;SAKK;IACL,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAM7B;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAC;IAM1C,QAAQ,CAAC,0BAA0B,EAAE,0BAA0B,CAAC;IAMhE;;;SAGK;IACL,QAAQ,CAAC,eAAe,EAAE,eAAe,CAAC;IAO1C,mDAAmD;IACnD,QAAQ,CAAC,kBAAkB,EAAE,mBAAmB,CAAC;IAMjD;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,EAAE,sBAAsB,CAAC;CACzD,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,oBAAoB,CAAC;AAEpE,wCAAwC;AACxC,eAAO,MAAM,2BAA2B,EAAE,IAAI,CAC5C,oBAAoB,EAClB,QAAQ,GACR,eAAe,GACf,4BAA4B,GAC5B,iBAAiB,GACjB,oBAAoB,CAsBvB,CAAC;AAEF,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,GACd,IAAI,CAAC,iBAAiB,EACvB,kBAAkB,GAClB,uBAAuB,CAAC,CAKzB;AAED,8DAA8D;AAE9D,MAAM,MAAM,uBAAuB,GACjC,IAAI,CAAC,oBAAoB,EAAE,MAAM,OAAO,2BAA2B,CAAC,GAClE,OAAO,CAAC,OAAO,2BAA2B,CAAC,GAC3C,IAAI,CAAC,iBAAiB,EAAE,MAAM,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,GAC1E,OAAO,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AAEzD,MAAM,MAAM,mBAAmB,GAAG,iBAAiB,GAAG;IACpD,8CAA8C;IAC9C,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC,CAAC;AAEF,sCAAsC;AACtC,MAAM,MAAM,uBAAuB,GAAG;IACpC,uFAAuF;IACvF,oBAAoB,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,6EAA6E;AAC7E,MAAM,MAAM,sBAAsB,GAAG,oBAAoB,GAAG;IAC1D,qBAAqB;IACrB,QAAQ,CAAC,QAAQ,EAAE,uBAAuB,CAAC;IAE3C;0CACsC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,2BAA2B,CAAC;IAEzD;4FACwF;IACxF,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IAExC;0DACsD;IACtD,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAEtC,+CAA+C;IAC/C,QAAQ,CAAC,6BAA6B,EAAE,MAAM,CAAC;IAE/C,mEAAmE;IACnE,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,sBAAsB,GAAG,mBAAmB,CAAC;AAE1E,wCAAwC;AACxC,eAAO,MAAM,6BAA6B,EAAE,IAAI,CAC9C,cAAc,EACZ,MAAM,OAAO,2BAA2B,GACxC,oBAAoB,GACpB,wBAAwB,GACxB,sBAAsB,GACtB,+BAA+B,GAC/B,UAAU,CAcb,CAAC;AAEF,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,MAAM,GACd,IAAI,CACH,mBAAmB,EACrB,MAAM,UAAU,CAAC,OAAO,wBAAwB,CAAC,GAAG,sBAAsB,CACzE,CAKF;AAED,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,sBAAsB,EACtB,MAAM,OAAO,6BAA6B,CAC3C,GACD,OAAO,CAAC,OAAO,6BAA6B,CAAC,GAC7C,IAAI,CAAC,mBAAmB,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC,GACvF,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC,CAAC"}
+{"version":3,"file":"ops.d.ts","sourceRoot":"","sources":["../../src/middle_layer/ops.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,KAAK,EACV,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,uBAAuB,EACxB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAE3D,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAGzE,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC,qDAAqD;IACrD,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAEvC;;;;;;;;;;;SAWK;IACL,QAAQ,CAAC,gBAAgB,EAAE,sBAAsB,EAAE,CAAC;IAEpD;;;SAGK;IACL,QAAQ,CAAC,4BAA4B,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAElE,kGAAkG;IAClG,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,yEAAyE;AACzE,MAAM,MAAM,oBAAoB,GAAG;IAKjC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAM1B;;;;;SAKK;IACL,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAM7B;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAC;IAM1C,QAAQ,CAAC,0BAA0B,EAAE,0BAA0B,CAAC;IAMhE;;;SAGK;IACL,QAAQ,CAAC,eAAe,EAAE,eAAe,CAAC;IAO1C,mDAAmD;IACnD,QAAQ,CAAC,kBAAkB,EAAE,mBAAmB,CAAC;IAMjD;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,EAAE,sBAAsB,CAAC;CACzD,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,oBAAoB,CAAC;AAEpE,wCAAwC;AACxC,eAAO,MAAM,2BAA2B,EAAE,IAAI,CAC5C,oBAAoB,EAClB,QAAQ,GACR,eAAe,GACf,4BAA4B,GAC5B,iBAAiB,GACjB,oBAAoB,CAsBvB,CAAC;AAEF,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,GACd,IAAI,CAAC,iBAAiB,EACvB,kBAAkB,GAClB,uBAAuB,GACvB,kBAAkB,CAAC,CAMpB;AAED,8DAA8D;AAE9D,MAAM,MAAM,uBAAuB,GACjC,IAAI,CAAC,oBAAoB,EAAE,MAAM,OAAO,2BAA2B,CAAC,GAClE,OAAO,CAAC,OAAO,2BAA2B,CAAC,GAC3C,IAAI,CAAC,iBAAiB,EAAE,MAAM,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,GAC1E,OAAO,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AAEzD,MAAM,MAAM,mBAAmB,GAAG,iBAAiB,GAAG;IACpD,8CAA8C;IAC9C,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC,CAAC;AAEF,sCAAsC;AACtC,MAAM,MAAM,uBAAuB,GAAG;IACpC,uFAAuF;IACvF,oBAAoB,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,6EAA6E;AAC7E,MAAM,MAAM,sBAAsB,GAAG,oBAAoB,GAAG;IAC1D,qBAAqB;IACrB,QAAQ,CAAC,QAAQ,EAAE,uBAAuB,CAAC;IAE3C;0CACsC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,2BAA2B,CAAC;IAEzD;4FACwF;IACxF,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IAExC;0DACsD;IACtD,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAEtC,+CAA+C;IAC/C,QAAQ,CAAC,6BAA6B,EAAE,MAAM,CAAC;IAE/C,mEAAmE;IACnE,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,sBAAsB,GAAG,mBAAmB,CAAC;AAE1E,wCAAwC;AACxC,eAAO,MAAM,6BAA6B,EAAE,IAAI,CAC9C,cAAc,EACZ,MAAM,OAAO,2BAA2B,GACxC,oBAAoB,GACpB,wBAAwB,GACxB,sBAAsB,GACtB,+BAA+B,GAC/B,UAAU,CAcb,CAAC;AAEF,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,MAAM,GACd,IAAI,CACH,mBAAmB,EACrB,MAAM,UAAU,CAAC,OAAO,wBAAwB,CAAC,GAAG,sBAAsB,CACzE,CAKF;AAED,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,sBAAsB,EACtB,MAAM,OAAO,6BAA6B,CAC3C,GACD,OAAO,CAAC,OAAO,6BAA6B,CAAC,GAC7C,IAAI,CAAC,mBAAmB,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC,GACvF,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,CAAC,CAAC"}

package/dist/pool/driver.d.ts

@@ -20,10 +20,11 @@ export interface InternalPFrameDriver extends SdkPFrameDriver {
 export declare class PFrameDriver implements InternalPFrameDriver {
     private readonly blobDriver;
     private readonly logger;
+    private readonly spillPath;
     private readonly pFrames;
     private readonly pTables;
-    private readonly blobContentCache;
-    constructor(blobDriver: DownloadDriver, logger: MiLogger);
+    static init(blobDriver: DownloadDriver, logger: MiLogger, spillPath: string): Promise<PFrameDriver>;
+    private constructor();
     createPFrame(def: PFrameDef<PlTreeNodeAccessor | PColumnValues | DataInfo<PlTreeNodeAccessor>>, ctx: ComputableCtx): PFrameHandle;
     createPTable(def: PTableDef<PColumn<PlTreeNodeAccessor | PColumnValues | DataInfo<PlTreeNodeAccessor>>>, ctx: ComputableCtx, signal?: AbortSignal): PTableHandle;
     findColumns(handle: PFrameHandle, request: FindColumnsRequest): Promise<FindColumnsResponse>;

package/dist/pool/driver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EACT,MAAM,sBAAsB,CAAC;AAW9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AA2J3D;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EACjF,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAC1F,GAAG,EAAE,aAAa,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,YAAY,CAAC;IAEhB,mFAAmF;IACnF,kBAAkB,CAChB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAEvC,yFAAyF;IACzF,eAAe,CACb,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC;AAED,qBAAa,YAAa,YAAW,oBAAoB;IAMrD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA+B;gBAG7C,UAAU,EAAE,cAAc,EAC1B,MAAM,EAAE,QAAQ;IAuE5B,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EACjF,GAAG,EAAE,aAAa,GACjB,YAAY;IAiBR,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAC1F,GAAG,EAAE,aAAa,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,YAAY;IAmBF,WAAW,CACtB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,mBAAmB,CAAC;IA2BlB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAI9E,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAI9D,kBAAkB,CAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC;IA8BzB,eAAe,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC;IAgBnB,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAKpD,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAK1D,OAAO,CAClB,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,CAAC,EAAE,UAAU,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;CAI3B"}
+{"version":3,"file":"driver.d.ts","sourceRoot":"","sources":["../../src/pool/driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,kBAAkB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,4BAA4B,CAAC;AACzF,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EAEnB,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,IAAI,eAAe,EAC/B,OAAO,EACP,SAAS,EAET,SAAS,EAIT,aAAa,EACb,QAAQ,EACT,MAAM,sBAAsB,CAAC;AAW9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AAmJ3D;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EACjF,GAAG,EAAE,aAAa,GACjB,YAAY,CAAC;IAEhB,0BAA0B;IAC1B,YAAY,CACV,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAC1F,GAAG,EAAE,aAAa,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,YAAY,CAAC;IAEhB,mFAAmF;IACnF,kBAAkB,CAChB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAEvC,yFAAyF;IACzF,eAAe,CACb,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC;AAED,qBAAa,YAAa,YAAW,oBAAoB;IAerD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAhB5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;WAExD,IAAI,CACtB,UAAU,EAAE,cAAc,EAC1B,MAAM,EAAE,QAAQ,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,YAAY,CAAC;IAMxB,OAAO;IA0EA,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EACjF,GAAG,EAAE,aAAa,GACjB,YAAY;IAiBR,YAAY,CACjB,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAC1F,GAAG,EAAE,aAAa,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,YAAY;IAmBF,WAAW,CACtB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,mBAAmB,CAAC;IA2BlB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;IAI9E,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAI9D,kBAAkB,CAC7B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,0BAA0B,CAAC;IA8BzB,eAAe,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,oBAAoB,CAAC;IAgBnB,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAKpD,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAK1D,OAAO,CAClB,MAAM,EAAE,YAAY,EACpB,aAAa,EAAE,MAAM,EAAE,EACvB,KAAK,CAAC,EAAE,UAAU,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;CAI3B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-middle-layer",
-  "version": "1.36.1",
+  "version": "1.36.3",
   "description": "Pl Middle Layer",
   "engines": {
     "node": ">=20.16.0"
@@ -22,7 +22,7 @@
   "keywords": [],
   "license": "UNLICENSED",
   "dependencies": {
-    "@milaboratories/pframes-rs-node": "1.0.27",
+    "@milaboratories/pframes-rs-node": "1.0.29",
     "canonicalize": "~2.1.0",
     "denque": "^2.1.0",
     "lru-cache": "^11.1.0",
@@ -33,21 +33,21 @@
     "zod": "~3.23.8",
     "remeda": "^2.21.2",
     "@milaboratories/pl-http": "^1.1.2",
+    "@milaboratories/computable": "^2.4.6",
     "@milaboratories/resolve-helper": "^1.1.0",
-    "@milaboratories/computable": "^2.4.5",
-    "@platforma-sdk/block-tools": "^2.5.40",
-    "@milaboratories/pl-client": "^2.8.1",
-    "@milaboratories/pl-drivers": "^1.5.48",
-    "@milaboratories/pl-model-middle-layer": "^1.7.26",
+    "@platforma-sdk/block-tools": "^2.5.41",
+    "@milaboratories/pl-client": "^2.8.2",
     "@milaboratories/pl-model-common": "^1.13.8",
+    "@milaboratories/pl-drivers": "^1.5.49",
     "@milaboratories/pl-model-backend": "^1.1.0",
-    "@milaboratories/pl-tree": "^1.5.7",
+    "@milaboratories/pl-tree": "^1.5.8",
+    "@milaboratories/pl-model-middle-layer": "^1.7.26",
     "@platforma-sdk/model": "^1.29.22",
-    "@milaboratories/ts-helpers": "^1.1.6",
     "@platforma-sdk/workflow-tengo": "4.1.3",
-    "@milaboratories/pl-config": "^1.4.7",
-    "@milaboratories/pl-errors": "^1.0.7",
-    "@milaboratories/pl-deployments": "^2.1.0"
+    "@milaboratories/pl-errors": "^1.0.8",
+    "@milaboratories/pl-config": "^1.4.8",
+    "@milaboratories/ts-helpers": "^1.1.7",
+    "@milaboratories/pl-deployments": "^2.1.1"
   },
   "devDependencies": {
     "semver": "^7.6.3",
@@ -60,8 +60,8 @@
     "jest": "^29.7.0",
     "@jest/globals": "^29.7.0",
     "ts-jest": "^29.2.6",
-    "@milaboratories/eslint-config": "^1.0.4",
-    "@milaboratories/platforma-build-configs": "1.0.3"
+    "@milaboratories/platforma-build-configs": "1.0.3",
+    "@milaboratories/eslint-config": "^1.0.4"
   },
   "scripts": {
     "type-check": "tsc --noEmit --composite false",

package/src/middle_layer/driver_kit.ts

@@ -112,7 +112,11 @@ export async function initDriverKit(
     ops.virtualLocalStoragesOverride,
   );
 
-  const pFrameDriver = new PFrameDriver(blobDriver, ops.logger);
+  const pFrameDriver = await PFrameDriver.init(
+    blobDriver,
+    ops.logger,
+    ops.pframesSpillPath,
+  );
 
   return {
     blobDriver,

package/src/middle_layer/ops.ts

@@ -39,6 +39,9 @@ export type DriverKitOpsPaths = {
    * If undefined, default list will be created.
    * */
   readonly virtualLocalStoragesOverride?: VirtualLocalStorageSpec[];
+
+  /** Path to the directory where pframes will spill temporary files and store materialized views */
+  readonly pframesSpillPath: string;
 };
 
 /** Options required to initialize full set of middle layer driver kit */
@@ -144,10 +147,12 @@ export function DefaultDriverKitOpsPaths(
   workDir: string,
 ): Pick<DriverKitOpsPaths,
 | 'blobDownloadPath'
-| 'downloadBlobToURLPath'> {
+| 'downloadBlobToURLPath'
+| 'pframesSpillPath'> {
   return {
     blobDownloadPath: path.join(workDir, 'download'),
     downloadBlobToURLPath: path.join(workDir, 'downloadToURL'),
+    pframesSpillPath: path.join(workDir, 'pframes'),
   };
 }
 

package/src/pool/driver.ts

@@ -42,10 +42,11 @@ import { RefCountResourcePool } from './ref_count_pool';
 import { allBlobs, makeDataInfoFromPColumnValues, mapBlobs, parseDataInfoResource } from './data';
 import { createHash } from 'node:crypto';
 import type { MiLogger } from '@milaboratories/ts-helpers';
-import { assertNever } from '@milaboratories/ts-helpers';
+import { assertNever, emptyDir } from '@milaboratories/ts-helpers';
 import canonicalize from 'canonicalize';
-import { PFrame as PFrameRs } from '@milaboratories/pframes-rs-node';
+import { PFrame } from '@milaboratories/pframes-rs-node';
 import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
 import { LRUCache } from 'lru-cache';
 import { getDebugFlags } from '../debug';
 
@@ -83,7 +84,7 @@ function migrateFilters(filters: PTableRecordFilter[]): PTableRecordFilter[] {
 const bigintReplacer = (_: string, v: unknown) => (typeof v === 'bigint' ? v.toString() : v);
 
 class PFrameHolder implements PFrameInternal.PFrameDataSource, Disposable {
-  public readonly rustPFrame: PFrameInternal.PFrameV4;
+  public readonly pFrame: PFrameInternal.PFrameV4;
   private readonly blobIdToResource = new Map<string, ResourceInfo>();
   private readonly blobHandleComputables = new Map<
     string,
@@ -93,20 +94,11 @@ class PFrameHolder implements PFrameInternal.PFrameDataSource, Disposable {
   constructor(
     private readonly blobDriver: DownloadDriver,
     private readonly logger: MiLogger,
+    private readonly spillPath: string,
     private readonly blobContentCache: LRUCache<string, Uint8Array>,
     columns: InternalPFrameData,
   ) {
-    const logFunc: PFrameInternal.Logger = (level: 'info' | 'warn' | 'error', message: string) => {
-      switch (level) {
-        default:
-        case 'info':
-          return this.logger.info(message);
-        case 'warn':
-          return this.logger.warn(message);
-        case 'error':
-          return this.logger.error(message);
-      }
-    };
+    const logFunc: PFrameInternal.Logger = (level, message) => this.logger[level](message);
 
     for (const column of columns) {
       for (const blob of allBlobs(column.data)) {
@@ -123,13 +115,13 @@ class PFrameHolder implements PFrameInternal.PFrameDataSource, Disposable {
     ];
 
     try {
-      const pFrame = new PFrameRs(getDebugFlags().logPFrameRequests ? logFunc : undefined);
+      const pFrame = new PFrame(this.spillPath, getDebugFlags().logPFrameRequests ? logFunc : undefined);
       pFrame.setDataSource(this);
       for (const column of distinctСolumns) {
         pFrame.addColumnSpec(column.id, column.spec);
         pFrame.setColumnData(column.id, column.data);
       }
-      this.rustPFrame = pFrame;
+      this.pFrame = pFrame;
     } catch (err: unknown) {
       throw new Error(
         `Rust PFrame creation failed, columns: ${JSON.stringify(distinctСolumns)}, error: ${err as Error}`,
@@ -176,7 +168,7 @@ class PFrameHolder implements PFrameInternal.PFrameDataSource, Disposable {
 
   [Symbol.dispose](): void {
     for (const computable of this.blobHandleComputables.values()) computable.resetState();
-    this.rustPFrame.dispose();
+    this.pFrame.dispose();
   }
 }
 
@@ -232,23 +224,33 @@ export interface InternalPFrameDriver extends SdkPFrameDriver {
 export class PFrameDriver implements InternalPFrameDriver {
   private readonly pFrames: RefCountResourcePool<InternalPFrameData, PFrameHolder>;
   private readonly pTables: RefCountResourcePool<FullPTableDef, PTableHolder>;
-  private readonly blobContentCache: LRUCache<string, Uint8Array>;
 
-  constructor(
+  public static async init(
+    blobDriver: DownloadDriver,
+    logger: MiLogger,
+    spillPath: string,
+  ): Promise<PFrameDriver> {
+    const resolvedSpillPath = path.resolve(spillPath);
+    await emptyDir(resolvedSpillPath);
+    return new PFrameDriver(blobDriver, logger, resolvedSpillPath);
+  }
+
+  private constructor(
     private readonly blobDriver: DownloadDriver,
     private readonly logger: MiLogger,
+    private readonly spillPath: string,
   ) {
     const blobContentCache = new LRUCache<string, Uint8Array>({
       maxSize: 1_000_000_000, // 1Gb
       fetchMethod: async (key) => await fsp.readFile(key),
       sizeCalculation: (v) => v.length,
     });
-    this.blobContentCache = blobContentCache;
 
     this.pFrames = new (class extends RefCountResourcePool<InternalPFrameData, PFrameHolder> {
       constructor(
         private readonly blobDriver: DownloadDriver,
         private readonly logger: MiLogger,
+        private readonly spillPath: string,
       ) {
         super();
       }
@@ -258,13 +260,13 @@ export class PFrameDriver implements InternalPFrameDriver {
           logger.info(
             `PFrame creation (pFrameHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`,
           );
-        return new PFrameHolder(this.blobDriver, this.logger, blobContentCache, params);
+        return new PFrameHolder(this.blobDriver, this.logger, this.spillPath, blobContentCache, params);
       }
 
       protected calculateParamsKey(params: InternalPFrameData): string {
         return stableKeyFromPFrameData(params);
       }
-    })(this.blobDriver, this.logger);
+    })(this.blobDriver, this.logger, this.spillPath);
 
     this.pTables = new (class extends RefCountResourcePool<
       FullPTableDef,
@@ -283,7 +285,7 @@ export class PFrameDriver implements InternalPFrameDriver {
             `PTable creation (pTableHandle = ${this.calculateParamsKey(params)}): ${JSON.stringify(params, bigintReplacer)}`,
           );
         }
-        const tablePromise = this.pFrames.getByKey(handle).rustPFrame.createTable({
+        const tablePromise = this.pFrames.getByKey(handle).pFrame.createTable({
           src: joinEntryToInternal(params.def.src),
           filters: migrateFilters(params.def.filters),
         }, params.signal).then(async (table) => {
@@ -368,7 +370,7 @@ export class PFrameDriver implements InternalPFrameDriver {
             }]
           : [],
     };
-    const responce = await this.pFrames.getByKey(handle).rustPFrame.findColumns(iRequest);
+    const responce = await this.pFrames.getByKey(handle).pFrame.findColumns(iRequest);
     return {
       hits: responce.hits
         .filter((h) => // only exactly matching columns
@@ -381,11 +383,11 @@ export class PFrameDriver implements InternalPFrameDriver {
   }
 
   public async getColumnSpec(handle: PFrameHandle, columnId: PObjectId): Promise<PColumnSpec> {
-    return await this.pFrames.getByKey(handle).rustPFrame.getColumnSpec(columnId);
+    return await this.pFrames.getByKey(handle).pFrame.getColumnSpec(columnId);
   }
 
   public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {
-    return await this.pFrames.getByKey(handle).rustPFrame.listColumns();
+    return await this.pFrames.getByKey(handle).pFrame.listColumns();
   }
 
   public async calculateTableData(
@@ -398,7 +400,7 @@ export class PFrameDriver implements InternalPFrameDriver {
         `Call calculateTableData, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,
       );
     }
-    return await this.pFrames.getByKey(handle).rustPFrame.createTable({
+    return await this.pFrames.getByKey(handle).pFrame.createTable({
       src: joinEntryToInternal(request.src),
       filters: migrateFilters(request.filters),
     }, signal).then(async (table) => {
@@ -432,7 +434,7 @@ export class PFrameDriver implements InternalPFrameDriver {
         `Call getUniqueValues, handle = ${handle}, request = ${JSON.stringify(request, bigintReplacer)}`,
       );
     }
-    return await this.pFrames.getByKey(handle).rustPFrame.getUniqueValues({
+    return await this.pFrames.getByKey(handle).pFrame.getUniqueValues({
       ...request,
       filters: migrateFilters(request.filters),
     }, signal);