@milaboratories/pl-model-middle-layer 1.8.16 → 1.8.17

package/dist/pframe/internal_api/api_factory.d.ts

@@ -1,4 +1,5 @@
-import { BinaryPartitionedDataInfo, JsonDataInfo, JsonPartitionedDataInfo, ParquetChunk, ParquetPartitionedDataInfo, PColumnSpec, PObjectId } from '@milaboratories/pl-model-common';
+import { Base64Encoded, BinaryPartitionedDataInfo, JsonDataInfo, JsonPartitionedDataInfo, ParquetChunk, ParquetPartitionedDataInfo, PColumnSpec, PObjectId } from '@milaboratories/pl-model-common';
+import { HttpAuthorizationToken, ObjectStoreUrl, PemCertificate } from './http_helpers';
 /** Abstract identifier of a data blob that can be requested from the storage backend */
 export type PFrameBlobId = string;
 /** Path of the file containing requested data (blob). This path is returned by
@@ -15,6 +16,31 @@ export type PFrameDataSource = {
     /** Returns raw blob data given the blob id from {@link DataInfo}. */
     resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
 };
+/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
+export type ParquetServerConfig = {
+    /** URL of the parquet HTTP(S) server */
+    url: ObjectStoreUrl;
+    /** Authorization token for Bearer scheme */
+    authToken?: HttpAuthorizationToken;
+    /** CA certificate of HTTPS server */
+    caCert?: Base64Encoded<PemCertificate>;
+};
+/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
+export interface PFrameDataSourceV2 {
+    /**
+     * PFrame may notify storage backend about its plans to use particular blobs in the future.
+     * Storage backend will do its best to preload specified blob so the subsequent
+     * {@link resolveBlob} will quickly return preloaded file path.
+     */
+    preloadBlob(blobIds: PFrameBlobId[]): Promise<void>;
+    /** Returns raw blob data given the blob id from {@link DataInfo}. */
+    resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
+    /**
+     * Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
+     * When not provided, parquet BlobIds would be treated as local file paths.
+     */
+    parquetServer?: ParquetServerConfig;
+}
 /**
  * Union type representing all possible data storage formats for PColumn data.
  * The specific format used depends on data size, access patterns, and performance requirements.
@@ -24,12 +50,15 @@ export type PFrameDataSource = {
 export type DataInfo<Blob> = JsonDataInfo | JsonPartitionedDataInfo<Blob> | BinaryPartitionedDataInfo<Blob> | ParquetPartitionedDataInfo<ParquetChunk<Blob>>;
 /** API exposed by PFrames library allowing to create and provide data for
  * PFrame objects */
-export interface PFrameFactoryAPIV2 {
+export interface PFrameFactoryAPIV3 extends Disposable {
     /** Associates data source with this PFrame */
     setDataSource(dataSource: PFrameDataSource): void;
-    /** Adds PColumn without spec */
+    /** Adds PColumn without data info */
     addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
-    /** Associates data info with cpecific column */
+    /**
+     * Assign data info to the specified PColumn.
+     * For parquet data info, schema resolution via network is performed during this call.
+     */
     setColumnData(columnId: PObjectId, dataInfo: DataInfo<PFrameBlobId>, options?: {
         signal?: AbortSignal;
     }): Promise<void>;
@@ -39,9 +68,9 @@ export interface PFrameFactoryAPIV2 {
 }
 /** API exposed by PFrames library allowing to create and provide data for
  * PFrame objects */
-export interface PFrameFactoryAPIV3 extends Disposable {
+export interface PFrameFactoryAPIV4 extends Disposable {
     /** Associates data source with this PFrame */
-    setDataSource(dataSource: PFrameDataSource): void;
+    setDataSource(dataSource: PFrameDataSourceV2): void;
     /** Adds PColumn without data info */
     addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
     /**

package/dist/pframe/internal_api/api_factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yBAAyB,EACzB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACV,MAAM,iCAAiC,CAAC;AAEzC,wFAAwF;AACxF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAGlC;+EAC+E;AAC/E,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,gGAAgG;AAChG,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,qEAAqE;IACrE,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC/D,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,CAAC,IAAI,IACrB,YAAY,GACZ,uBAAuB,CAAC,IAAI,CAAC,GAC7B,yBAAyB,CAAC,IAAI,CAAC,GAC/B,0BAA0B,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AAEnD;oBACoB;AACpB,MAAM,WAAW,kBAAkB;IACjC,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAElD,gCAAgC;IAChC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE,gDAAgD;IAChD,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;oBACoB;AACpB,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAElD,qCAAqC;IACrC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE;;;OAGG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB"}
+{"version":3,"file":"api_factory.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_factory.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,yBAAyB,EACzB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,0BAA0B,EAC1B,WAAW,EACX,SAAS,EACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,sBAAsB,EACtB,cAAc,EACd,cAAc,EACf,MAAM,gBAAgB,CAAC;AAExB,wFAAwF;AACxF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAGlC;+EAC+E;AAC/E,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,gGAAgG;AAChG,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,qEAAqE;IACrE,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC/D,CAAC;AAEF,uFAAuF;AACvF,MAAM,MAAM,mBAAmB,GAAG;IAChC,wCAAwC;IACxC,GAAG,EAAE,cAAc,CAAC;IAEpB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IAEnC,qCAAqC;IACrC,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACxC,CAAC;AAEF,gGAAgG;AAChG,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpD,qEAAqE;IACrE,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAE9D;;;OAGG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,CAAC,IAAI,IACrB,YAAY,GACZ,uBAAuB,CAAC,IAAI,CAAC,GAC7B,yBAAyB,CAAC,IAAI,CAAC,GAC/B,0BAA0B,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AAEnD;oBACoB;AACpB,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAElD,qCAAqC;IACrC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE;;;OAGG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;oBACoB;AACpB,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,8CAA8C;IAC9C,aAAa,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAEpD,qCAAqC;IACrC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI,CAAC;IAElE;;;OAGG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC,EAChC,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;wEACoE;IACpE,OAAO,IAAI,IAAI,CAAC;CACjB"}

package/dist/pframe/internal_api/api_read.d.ts

@@ -2,31 +2,7 @@ import { FindColumnsRequest, FindColumnsResponse } from './find_columns';
 import { DeleteColumnFromColumnsRequest, DeleteColumnFromColumnsResponse } from './delete_column';
 import { PColumnInfo, PColumnSpec, PObjectId, UniqueValuesRequest, UniqueValuesResponse } from '@milaboratories/pl-model-common';
 import { CreateTableRequestV3 } from './create_table';
-import { PTableV6, PTableV7 } from './table';
-/** Read interface exposed by PFrames library */
-export interface PFrameReadAPIV8 {
-    /**
-     * Finds columns given filtering criteria on column name, annotations etc.
-     * and a set of qualified axes specs to find only columns with compatible
-     * axes spec.
-     *
-     * Only column specs are used, this method will work even for columns
-     * with no assigned data.
-     * */
-    findColumns(request: FindColumnsRequest): Promise<FindColumnsResponse>;
-    /** To be reviewed in the future */
-    deleteColumn(request: DeleteColumnFromColumnsRequest): Promise<DeleteColumnFromColumnsResponse>;
-    /** Retrieve single column spec */
-    getColumnSpec(columnId: PObjectId): Promise<PColumnSpec>;
-    /** Retrieve information about all columns currently added to the PFrame */
-    listColumns(): Promise<PColumnInfo[]>;
-    /** Calculates data for the table and returns an object to access it */
-    createTable(request: CreateTableRequestV3): PTableV6;
-    /** Calculate set of unique values for a specific axis for the filtered set of records */
-    getUniqueValues(request: UniqueValuesRequest, ops?: {
-        signal?: AbortSignal;
-    }): Promise<UniqueValuesResponse>;
-}
+import { PTableV7 } from './table';
 /** Read interface exposed by PFrames library */
 export interface PFrameReadAPIV9 {
     /**

package/dist/pframe/internal_api/api_read.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api_read.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_read.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,8BAA8B,EAAE,+BAA+B,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,mBAAmB,EACnB,oBAAoB,EACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE7C,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B;;;;;;;SAOK;IACL,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEvE,mCAAmC;IACnC,YAAY,CAAC,OAAO,EAAE,8BAA8B,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;IAEhG,kCAAkC;IAClC,aAAa,CAAC,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzD,2EAA2E;IAC3E,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEtC,uEAAuE;IACvE,WAAW,CAAC,OAAO,EAAE,oBAAoB,GAAG,QAAQ,CAAC;IAErD,yFAAyF;IACzF,eAAe,CACb,OAAO,EAAE,mBAAmB,EAC5B,GAAG,CAAC,EAAE;QACJ,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC;AAED,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B;;;;;;;SAOK;IACL,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEvE;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,8BAA8B,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;IAEhG,kCAAkC;IAClC,aAAa,CAAC,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzD,2EAA2E;IAC3E,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEtC,uEAAuE;IACvE,WAAW,CAAC,OAAO,EAAE,oBAAoB,GAAG,QAAQ,CAAC;IAErD,yFAAyF;IACzF,eAAe,CACb,OAAO,EAAE,mBAAmB,EAC5B,GAAG,CAAC,EAAE;QACJ,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC"}
+{"version":3,"file":"api_read.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/api_read.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,8BAA8B,EAAE,+BAA+B,EAAE,MAAM,iBAAiB,CAAC;AAClG,OAAO,EACL,WAAW,EACX,WAAW,EACX,SAAS,EACT,mBAAmB,EACnB,oBAAoB,EACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B;;;;;;;SAOK;IACL,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAEvE;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,8BAA8B,GAAG,OAAO,CAAC,+BAA+B,CAAC,CAAC;IAEhG,kCAAkC;IAClC,aAAa,CAAC,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzD,2EAA2E;IAC3E,WAAW,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEtC,uEAAuE;IACvE,WAAW,CAAC,OAAO,EAAE,oBAAoB,GAAG,QAAQ,CAAC;IAErD,yFAAyF;IACzF,eAAe,CACb,OAAO,EAAE,mBAAmB,EAC5B,GAAG,CAAC,EAAE;QACJ,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC"}

package/dist/pframe/internal_api/pframe.d.ts

@@ -1,10 +1,10 @@
-import type { PFrameFactoryAPIV2, PFrameFactoryAPIV3 } from './api_factory';
-import type { PFrameReadAPIV8, PFrameReadAPIV9 } from './api_read';
+import type { PFrameFactoryAPIV3, PFrameFactoryAPIV4 } from './api_factory';
+import type { PFrameReadAPIV9 } from './api_read';
 import type { Logger } from './common';
-export interface PFrameV9 extends PFrameFactoryAPIV2, PFrameReadAPIV8 {
-}
 export interface PFrameV10 extends PFrameFactoryAPIV3, PFrameReadAPIV9 {
 }
+export interface PFrameV11 extends PFrameFactoryAPIV4, PFrameReadAPIV9 {
+}
 export type PFrameOptions = {
     /** Path to directory where PFrame can create temporary files */
     spillPath: string;
@@ -27,4 +27,20 @@ export interface PFrameFactory {
      */
     pprofDump: () => Promise<Uint8Array>;
 }
+/** List of PFrame management functions exposed by PFrame module */
+export interface PFrameFactoryV2 {
+    /**
+     * Create a new PFrame instance.
+     * @warning Use concurrency limiting to avoid OOM crashes when multiple instances are simultaneously in use.
+     */
+    createPFrame(options: PFrameOptions): PFrameV11;
+    /**
+     * Dump active allocations from all PFrames instances in pprof format.
+     * The result of this function should be saved as `profile.pb.gz`.
+     * Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
+     * to view the allocation flamechart.
+     * @warning This method will always reject on Windows!
+     */
+    pprofDump: () => Promise<Uint8Array>;
+}
 //# sourceMappingURL=pframe.d.ts.map

package/dist/pframe/internal_api/pframe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,MAAM,WAAW,QAAS,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAExE,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,MAAM,aAAa,GAAG;IAC1B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAA;AAED,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC"}
+{"version":3,"file":"pframe.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/pframe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAEvC,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,WAAW,SAAU,SAAQ,kBAAkB,EAAE,eAAe;CAAG;AAEzE,MAAM,MAAM,aAAa,GAAG;IAC1B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAA;AAED,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC;AAED,mEAAmE;AACnE,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,CAAC;IAEhD;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;CACtC"}

package/dist/pframe/internal_api/table.d.ts

@@ -1,58 +1,4 @@
 import { PTableColumnId, PTableColumnSpec, PTableRecordFilter, PTableShape, PTableSorting, PTableVector, TableRange } from '@milaboratories/pl-model-common';
-/**
- * Table view returned as a result of create table operation.
- *
- * PTable can be thought as having a composite primary key, consisting of axes,
- * and a set of data columns derived from PColumn values.
- * */
-export interface PTableV6 {
-    /**
-     * Returns ordered array of table axes specs (primary key "columns" in SQL
-     * terms) and data column specs (regular "columns" in SQL terms).
-     *
-     * Data for a specific table column can be retrieved using unified indexing
-     * corresponding to elements in this array.
-     *
-     * Axes are always listed first.
-     * */
-    getSpec(): PTableColumnSpec[];
-    /** Transforms unified column identifiers into unified indices of columns. */
-    getColumnIndices(columnIds: PTableColumnId[]): number[];
-    /**
-     * Get PTable disk footprint in bytes
-     * Warning: This call materializes the join.
-     */
-    getFootprint(ops?: {
-        withPredecessors?: boolean;
-        signal?: AbortSignal;
-    }): Promise<number>;
-    /**
-     * Unified table shape
-     * Warning: This call materializes the join.
-     */
-    getShape(ops?: {
-        signal?: AbortSignal;
-    }): Promise<PTableShape>;
-    /**
-     * Retrieve the data from the table. To retrieve only data required, it can be
-     * sliced both horizontally ({@link columnIndices}) and vertically
-     * ({@link range}).
-     * This call materializes the join.
-     *
-     * @param columnIndices unified indices of columns to be retrieved
-     * @param range optionally limit the range of records to retrieve
-     * */
-    getData(columnIndices: number[], ops?: {
-        range?: TableRange;
-        signal?: AbortSignal;
-    }): Promise<PTableVector[]>;
-    /** Filters the table and returns new PTable instance */
-    filter(request: PTableRecordFilter[]): PTableV6;
-    /** Sorts the table and returns new PTable instance. */
-    sort(request: PTableSorting[]): PTableV6;
-    /** Deallocates all underlying resources */
-    dispose(): void;
-}
 /**
  * Table view returned as a result of create table operation.
  *

package/dist/pframe/internal_api/table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/table.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,kBAAkB,EAClB,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACX,MAAM,iCAAiC,CAAC;AAEzC;;;;;KAKK;AACL,MAAM,WAAW,QAAQ;IACvB;;;;;;;;SAQK;IACL,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAE9B,6EAA6E;IAC7E,gBAAgB,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IAExD;;;OAGG;IACH,YAAY,CAAC,GAAG,CAAC,EAAE;QACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzB;;;;;;;;SAQK;IACL,OAAO,CACL,aAAa,EAAE,MAAM,EAAE,EACvB,GAAG,CAAC,EAAE;QACJ,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3B,wDAAwD;IACxD,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,QAAQ,CAAC;IAEhD,uDAAuD;IACvD,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,QAAQ,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;;KAKK;AACL,MAAM,WAAW,QAAS,SAAQ,UAAU;IAC1C;;;;;;;;SAQK;IACL,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAE9B,6EAA6E;IAC7E,gBAAgB,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IAExD;;;OAGG;IACH,YAAY,CAAC,GAAG,CAAC,EAAE;QACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzB;;;;;;;;SAQK;IACL,OAAO,CACL,aAAa,EAAE,MAAM,EAAE,EACvB,GAAG,CAAC,EAAE;QACJ,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3B,wDAAwD;IACxD,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,QAAQ,CAAC;IAEhD,uDAAuD;IACvD,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,QAAQ,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,IAAI,IAAI,CAAC;CACjB"}
+{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../../src/pframe/internal_api/table.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,kBAAkB,EAClB,WAAW,EACX,aAAa,EACb,YAAY,EACZ,UAAU,EACX,MAAM,iCAAiC,CAAC;AAEzC;;;;;KAKK;AACL,MAAM,WAAW,QAAS,SAAQ,UAAU;IAC1C;;;;;;;;SAQK;IACL,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAE9B,6EAA6E;IAC7E,gBAAgB,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IAExD;;;OAGG;IACH,YAAY,CAAC,GAAG,CAAC,EAAE;QACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzB;;;;;;;;SAQK;IACL,OAAO,CACL,aAAa,EAAE,MAAM,EAAE,EACvB,GAAG,CAAC,EAAE;QACJ,KAAK,CAAC,EAAE,UAAU,CAAC;QACnB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3B,wDAAwD;IACxD,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,QAAQ,CAAC;IAEhD,uDAAuD;IACvD,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,QAAQ,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,IAAI,IAAI,CAAC;CACjB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-model-middle-layer",
-  "version": "1.8.16",
+  "version": "1.8.17",
   "description": "Common model between middle layer and non-block UI code",
   "type": "module",
   "types": "./dist/index.d.ts",

package/src/pframe/internal_api/api_factory.ts

@@ -1,4 +1,5 @@
 import {
+  Base64Encoded,
   BinaryPartitionedDataInfo,
   JsonDataInfo,
   JsonPartitionedDataInfo,
@@ -7,6 +8,12 @@ import {
   PColumnSpec,
   PObjectId,
 } from '@milaboratories/pl-model-common';
+import {
+  HttpHelpers,
+  HttpAuthorizationToken,
+  ObjectStoreUrl,
+  PemCertificate,
+} from './http_helpers';
 
 /** Abstract identifier of a data blob that can be requested from the storage backend */
 export type PFrameBlobId = string;
@@ -29,6 +36,37 @@ export type PFrameDataSource = {
   resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
 };
 
+/** Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer} */
+export type ParquetServerConfig = {
+  /** URL of the parquet HTTP(S) server */
+  url: ObjectStoreUrl;
+
+  /** Authorization token for Bearer scheme */
+  authToken?: HttpAuthorizationToken;
+
+  /** CA certificate of HTTPS server */
+  caCert?: Base64Encoded<PemCertificate>;
+};
+
+/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
+export interface PFrameDataSourceV2 {
+  /**
+   * PFrame may notify storage backend about its plans to use particular blobs in the future.
+   * Storage backend will do its best to preload specified blob so the subsequent
+   * {@link resolveBlob} will quickly return preloaded file path.
+   */
+  preloadBlob(blobIds: PFrameBlobId[]): Promise<void>;
+
+  /** Returns raw blob data given the blob id from {@link DataInfo}. */
+  resolveBlobContent(blobId: PFrameBlobId): Promise<Uint8Array>;
+
+  /**
+   * Parquet HTTP(S) server connection settings, {@link HttpHelpers.createHttpServer}
+   * When not provided, parquet BlobIds would be treated as local file paths.
+   */
+  parquetServer?: ParquetServerConfig;
+}
+
 /**
  * Union type representing all possible data storage formats for PColumn data.
  * The specific format used depends on data size, access patterns, and performance requirements.
@@ -43,14 +81,17 @@ export type DataInfo<Blob> =
 
 /** API exposed by PFrames library allowing to create and provide data for
  * PFrame objects */
-export interface PFrameFactoryAPIV2 {
+export interface PFrameFactoryAPIV3 extends Disposable {
   /** Associates data source with this PFrame */
   setDataSource(dataSource: PFrameDataSource): void;
 
-  /** Adds PColumn without spec */
+  /** Adds PColumn without data info */
   addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
 
-  /** Associates data info with cpecific column */
+  /**
+   * Assign data info to the specified PColumn.
+   * For parquet data info, schema resolution via network is performed during this call.
+   */
   setColumnData(
     columnId: PObjectId,
     dataInfo: DataInfo<PFrameBlobId>,
@@ -66,9 +107,9 @@ export interface PFrameFactoryAPIV2 {
 
 /** API exposed by PFrames library allowing to create and provide data for
  * PFrame objects */
-export interface PFrameFactoryAPIV3 extends Disposable {
+export interface PFrameFactoryAPIV4 extends Disposable {
   /** Associates data source with this PFrame */
-  setDataSource(dataSource: PFrameDataSource): void;
+  setDataSource(dataSource: PFrameDataSourceV2): void;
 
   /** Adds PColumn without data info */
   addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;

package/src/pframe/internal_api/api_read.ts

@@ -8,40 +8,7 @@ import {
   UniqueValuesResponse
 } from '@milaboratories/pl-model-common';
 import { CreateTableRequestV3 } from './create_table';
-import { PTableV6, PTableV7 } from './table';
-
-/** Read interface exposed by PFrames library */
-export interface PFrameReadAPIV8 {
-  /**
-   * Finds columns given filtering criteria on column name, annotations etc.
-   * and a set of qualified axes specs to find only columns with compatible
-   * axes spec.
-   *
-   * Only column specs are used, this method will work even for columns
-   * with no assigned data.
-   * */
-  findColumns(request: FindColumnsRequest): Promise<FindColumnsResponse>;
-
-  /** To be reviewed in the future */
-  deleteColumn(request: DeleteColumnFromColumnsRequest): Promise<DeleteColumnFromColumnsResponse>;
-
-  /** Retrieve single column spec */
-  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec>;
-
-  /** Retrieve information about all columns currently added to the PFrame */
-  listColumns(): Promise<PColumnInfo[]>;
-
-  /** Calculates data for the table and returns an object to access it */
-  createTable(request: CreateTableRequestV3): PTableV6;
-
-  /** Calculate set of unique values for a specific axis for the filtered set of records */
-  getUniqueValues(
-    request: UniqueValuesRequest,
-    ops?: {
-      signal?: AbortSignal,
-    }
-  ): Promise<UniqueValuesResponse>;
-}
+import { PTableV7 } from './table';
 
 /** Read interface exposed by PFrames library */
 export interface PFrameReadAPIV9 {

package/src/pframe/internal_api/pframe.ts

@@ -1,11 +1,11 @@
-import type { PFrameFactoryAPIV2, PFrameFactoryAPIV3 } from './api_factory';
-import type { PFrameReadAPIV8, PFrameReadAPIV9 } from './api_read';
+import type { PFrameFactoryAPIV3, PFrameFactoryAPIV4 } from './api_factory';
+import type { PFrameReadAPIV9 } from './api_read';
 import type { Logger } from './common';
 
-export interface PFrameV9 extends PFrameFactoryAPIV2, PFrameReadAPIV8 {}
-
 export interface PFrameV10 extends PFrameFactoryAPIV3, PFrameReadAPIV9 {}
 
+export interface PFrameV11 extends PFrameFactoryAPIV4, PFrameReadAPIV9 {}
+
 export type PFrameOptions = {
   /** Path to directory where PFrame can create temporary files */
   spillPath: string;
@@ -30,3 +30,21 @@ export interface PFrameFactory {
    */
   pprofDump: () => Promise<Uint8Array>;
 };
+
+/** List of PFrame management functions exposed by PFrame module */
+export interface PFrameFactoryV2 {
+  /**
+   * Create a new PFrame instance.
+   * @warning Use concurrency limiting to avoid OOM crashes when multiple instances are simultaneously in use.
+   */
+  createPFrame(options: PFrameOptions): PFrameV11;
+
+  /**
+   * Dump active allocations from all PFrames instances in pprof format.
+   * The result of this function should be saved as `profile.pb.gz`.
+   * Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
+   * to view the allocation flamechart.
+   * @warning This method will always reject on Windows!
+   */
+  pprofDump: () => Promise<Uint8Array>;
+};

package/src/pframe/internal_api/table.ts

@@ -8,71 +8,6 @@ import {
   TableRange
 } from '@milaboratories/pl-model-common';
 
-/**
- * Table view returned as a result of create table operation.
- *
- * PTable can be thought as having a composite primary key, consisting of axes,
- * and a set of data columns derived from PColumn values.
- * */
-export interface PTableV6 {
-  /**
-   * Returns ordered array of table axes specs (primary key "columns" in SQL
-   * terms) and data column specs (regular "columns" in SQL terms).
-   *
-   * Data for a specific table column can be retrieved using unified indexing
-   * corresponding to elements in this array.
-   *
-   * Axes are always listed first.
-   * */
-  getSpec(): PTableColumnSpec[];
-
-  /** Transforms unified column identifiers into unified indices of columns. */
-  getColumnIndices(columnIds: PTableColumnId[]): number[];
-
-  /**
-   * Get PTable disk footprint in bytes
-   * Warning: This call materializes the join.
-   */
-  getFootprint(ops?: {
-    withPredecessors?: boolean,
-    signal?: AbortSignal,
-  }): Promise<number>;
-
-  /**
-   * Unified table shape
-   * Warning: This call materializes the join.
-   */
-  getShape(ops?: {
-    signal?: AbortSignal,
-  }): Promise<PTableShape>;
-
-  /**
-   * Retrieve the data from the table. To retrieve only data required, it can be
-   * sliced both horizontally ({@link columnIndices}) and vertically
-   * ({@link range}).
-   * This call materializes the join.
-   *
-   * @param columnIndices unified indices of columns to be retrieved
-   * @param range optionally limit the range of records to retrieve
-   * */
-  getData(
-    columnIndices: number[],
-    ops?: {
-      range?: TableRange,
-      signal?: AbortSignal,
-    },
-  ): Promise<PTableVector[]>;
-
-  /** Filters the table and returns new PTable instance */
-  filter(request: PTableRecordFilter[]): PTableV6;
-
-  /** Sorts the table and returns new PTable instance. */
-  sort(request: PTableSorting[]): PTableV6;
-
-  /** Deallocates all underlying resources */
-  dispose(): void;
-}
-
 /**
  * Table view returned as a result of create table operation.
  *