@milaboratories/pl-model-middle-layer 1.8.41 → 1.8.42

package/src/block_registry/registry_spec.ts

@@ -2,35 +2,35 @@ import { z } from 'zod';
 
 export const LocalDevFolder = z.object({
   type: z.literal('local-dev'),
-  path: z.string()
+  path: z.string(),
 });
 export type LocalDevFolder = z.infer<typeof LocalDevFolder>;
 
 /** @deprecated don't use */
 export const RemoteRegistryV1Spec = z.object({
   type: z.literal('remote-v1'),
-  url: z.string().url()
+  url: z.string().url(),
 });
 /** @deprecated don't use */
 export type RemoteRegistryV1Spec = z.infer<typeof RemoteRegistryV1Spec>;
 
 export const RemoteRegistryV2Spec = z.object({
   type: z.literal('remote-v2'),
-  url: z.string().url()
+  url: z.string().url(),
 });
 export type RemoteRegistryV2Spec = z.infer<typeof RemoteRegistryV2Spec>;
 
 export const RegistrySpec = z.discriminatedUnion('type', [
   RemoteRegistryV1Spec,
   RemoteRegistryV2Spec,
-  LocalDevFolder
+  LocalDevFolder,
 ]);
 export type RegistrySpec = z.infer<typeof RegistrySpec>;
 
 export const RegistryEntry = z.object({
   id: z.string(),
   title: z.string().optional(),
-  spec: RegistrySpec
+  spec: RegistrySpec,
 });
 export type RegistryEntry = z.infer<typeof RegistryEntry>;
 

package/src/block_state.ts

@@ -1,9 +1,9 @@
-import { BlockOutputsBase, BlockState } from '@milaboratories/pl-model-common';
-import { Optional } from 'utility-types';
+import type { BlockOutputsBase, BlockState } from '@milaboratories/pl-model-common';
+import type { Optional } from 'utility-types';
 
 export type BlockStateInternal<
   Args = unknown,
   Outputs extends BlockOutputsBase = BlockOutputsBase,
   UiState = unknown,
-  Href extends `/${string}` = `/${string}`
+  Href extends `/${string}` = `/${string}`,
 > = Optional<Readonly<BlockState<Args, Outputs, UiState, Href>>, 'outputs'>;

package/src/pframe/internal_api/api_factory.ts

@@ -1,4 +1,4 @@
-import {
+import type {
   BinaryPartitionedDataInfo,
   Branded,
   JsonDataInfo,
@@ -8,10 +8,7 @@ import {
   PColumnSpec,
   PObjectId,
 } from '@milaboratories/pl-model-common';
-import {
-  HttpHelpers,
-  HttpServerInfo,
-} from './http_helpers';
+import type { HttpServerInfo } from './http_helpers';
 
 /** PColumn spec file extension */
 export const SpecExtension = '.spec' as const;
@@ -74,7 +71,7 @@ export interface PFrameFactoryAPIV4 extends Disposable {
     columnId: PObjectId,
     dataInfo: DataInfo<PFrameBlobId>,
     options?: {
-      signal?: AbortSignal,
+      signal?: AbortSignal;
     }
   ): Promise<void>;
 

package/src/pframe/internal_api/api_read.ts

@@ -1,14 +1,17 @@
-import { FindColumnsRequest, FindColumnsResponse } from './find_columns';
-import { DeleteColumnFromColumnsRequest, DeleteColumnFromColumnsResponse } from './delete_column';
-import {
+import type { FindColumnsRequest, FindColumnsResponse } from './find_columns';
+import type { DeleteColumnFromColumnsRequest, DeleteColumnFromColumnsResponse } from './delete_column';
+import type {
   PColumnInfo,
   PColumnSpec,
   PObjectId,
   UniqueValuesRequest,
-  UniqueValuesResponse
+  UniqueValuesResponse,
 } from '@milaboratories/pl-model-common';
-import { CreateTableRequestV4 } from './create_table';
-import { PTableV7 } from './table';
+import type { CreateTableRequestV4 } from './create_table';
+import type { PTableV7, PTableV8 } from './table';
+import type { PTableId } from './common';
+
+// REMOVE AFTER MIGRATION BEGIN...
 
 /** Read interface exposed by PFrames library */
 export interface PFrameReadAPIV10 {
@@ -29,7 +32,7 @@ export interface PFrameReadAPIV10 {
   deleteColumn(request: DeleteColumnFromColumnsRequest): Promise<DeleteColumnFromColumnsResponse>;
 
   /** Retrieve single column spec */
-  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec>;
+  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec | null>;
 
   /** Retrieve information about all columns currently added to the PFrame */
   listColumns(): Promise<PColumnInfo[]>;
@@ -41,7 +44,45 @@ export interface PFrameReadAPIV10 {
   getUniqueValues(
     request: UniqueValuesRequest,
     ops?: {
-      signal?: AbortSignal,
+      signal?: AbortSignal;
+    }
+  ): Promise<UniqueValuesResponse>;
+}
+
+// ...REMOVE AFTER MIGRATION END
+
+/** Read interface exposed by PFrames library */
+export interface PFrameReadAPIV11 {
+  /**
+   * Finds columns given filtering criteria on column name, annotations etc.
+   * and a set of qualified axes specs to find only columns with compatible
+   * axes spec.
+   *
+   * Only column specs are used, this method will work even for columns
+   * with no assigned data.
+   * */
+  findColumns(request: FindColumnsRequest): Promise<FindColumnsResponse>;
+
+  /**
+   * Construct new axes integration with some entry removed but integration qualificaitons preserved.
+   * Removes more then one entry in case the removal will create several disjoint sets of axes.
+   */
+  deleteColumn(request: DeleteColumnFromColumnsRequest): Promise<DeleteColumnFromColumnsResponse>;
+
+  /** Retrieve single column spec */
+  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec | null>;
+
+  /** Retrieve information about all columns currently added to the PFrame */
+  listColumns(): Promise<PColumnInfo[]>;
+
+  /** Calculates data for the table and returns an object to access it */
+  createTable(tableId: PTableId, request: CreateTableRequestV4): PTableV8;
+
+  /** Calculate set of unique values for a specific axis for the filtered set of records */
+  getUniqueValues(
+    request: UniqueValuesRequest,
+    ops?: {
+      signal?: AbortSignal;
     }
   ): Promise<UniqueValuesResponse>;
 }

package/src/pframe/internal_api/common.ts

@@ -1,10 +1,19 @@
-import { AxisSpec, ValueType } from '@milaboratories/pl-model-common';
+import type {
+  AxisSpec,
+  PFrameHandle,
+  PTableHandle,
+  ValueType,
+} from '@milaboratories/pl-model-common';
 
 export type Logger = (
   level: 'info' | 'warn' | 'error',
   message: string
 ) => void;
 
+export type PFrameId = PFrameHandle;
+
+export type PTableId = PTableHandle;
+
 export interface SingleAxisSelector {
   name: string;
   type?: ValueType;
@@ -25,4 +34,4 @@ export type ConstantAxisFilter = {
   type: 'constant';
   axisIndex: number;
   constant: string | number;
-}
+};

package/src/pframe/internal_api/create_table.ts

@@ -1,5 +1,5 @@
-import { ConstantAxisFilter } from './common';
-import { JsonDataInfo, PColumnSpec, PObjectId, PTableRecordFilter } from '@milaboratories/pl-model-common';
+import type { ConstantAxisFilter } from './common';
+import type { JsonDataInfo, PColumnSpec, PObjectId, PTableRecordFilter } from '@milaboratories/pl-model-common';
 
 export interface ColumnJoinEntry {
   type: 'column';
@@ -43,7 +43,7 @@ export interface OuterJoinV4 {
   secondary: JoinEntryV4[];
 }
 
-export type JoinEntryV4 = 
+export type JoinEntryV4 =
   | ColumnJoinEntry
   | SlicedColumnJoinEntry
   | ArtificialColumnJoinEntry

package/src/pframe/internal_api/delete_column.ts

@@ -1,4 +1,4 @@
-import { ColumnAxesWithQualifications } from './common';
+import type { ColumnAxesWithQualifications } from './common';
 
 export interface DeleteColumnFromColumnsRequest {
   columns: ColumnAxesWithQualifications[];

package/src/pframe/internal_api/find_columns.ts

@@ -1,5 +1,5 @@
-import { ColumnFilter, PColumnIdAndSpec } from '@milaboratories/pl-model-common';
-import { AxisQualification, ColumnAxesWithQualifications } from './common';
+import type { ColumnFilter, PColumnIdAndSpec } from '@milaboratories/pl-model-common';
+import type { AxisQualification, ColumnAxesWithQualifications } from './common';
 
 export interface FindColumnsRequest {
   columnFilter: ColumnFilter;

package/src/pframe/internal_api/http_helpers.ts

@@ -14,54 +14,54 @@ export type FileRange = {
   start: number;
   /** End byte position (inclusive) */
   end: number;
-}
+};
 
 /** HTTP range as of RFC 9110 <https://datatracker.ietf.org/doc/html/rfc9110#name-range> */
 export type HttpRange =
   | {
-      /**
+    /**
        * Get file content in the specified byte range
-       * 
+       *
        * @example
        * ```
        * GET /file.parquet HTTP/1.1
        * Range: bytes=0-1023
        * ```
        */
-      type: 'bounded';
-      /** Start byte position (inclusive) */
-      start: number;
-      /** End byte position (inclusive) */
-      end: number;
-    }
+    type: 'bounded';
+    /** Start byte position (inclusive) */
+    start: number;
+    /** End byte position (inclusive) */
+    end: number;
+  }
   | {
-      /**
+    /**
        * Get byte range starting from the specified offset
-       * 
+       *
        * @example
        * ```
        * GET /file.parquet HTTP/1.1
        * Range: bytes=1024-
        * ```
        */
-      type: 'offset';
-      /** Start byte position (inclusive) */
-      offset: number;
-    }
+    type: 'offset';
+    /** Start byte position (inclusive) */
+    offset: number;
+  }
   | {
-      /**
+    /**
        * Get byte range starting from the specified suffix
-       * 
+       *
        * @example
        * ```
        * GET /file.parquet HTTP/1.1
        * Range: bytes=-1024
        * ```
        */
-      type: 'suffix';
-      /** End byte position (inclusive) */
-      suffix: number;
-    };
+    type: 'suffix';
+    /** End byte position (inclusive) */
+    suffix: number;
+  };
 
 /** HTTP method passed to object store */
 export type HttpMethod = 'GET' | 'HEAD';
@@ -69,32 +69,32 @@ export type HttpMethod = 'GET' | 'HEAD';
 /** HTTP response from object store */
 export type ObjectStoreResponse =
   | {
-      /**
+    /**
        * Will be translated to 500 Internal Server Error by the handler
        * or 408 Request Timeout if the request was aborted
        */
-      type: 'InternalError';
-    }
+    type: 'InternalError';
+  }
   | {
-      /** Will be translated to 404 Not Found by the handler */
-      type: 'NotFound';
-    }
+    /** Will be translated to 404 Not Found by the handler */
+    type: 'NotFound';
+  }
   | {
-      /** Will be translated to 416 Range Not Satisfiable by the handler */
-      type: 'RangeNotSatisfiable';
-      /** Total file size in bytes */
-      size: number;
-    }
+    /** Will be translated to 416 Range Not Satisfiable by the handler */
+    type: 'RangeNotSatisfiable';
+    /** Total file size in bytes */
+    size: number;
+  }
   | {
-      /** Will be translated to 200 OK or 206 Partial Content by the handler */
-      type: 'Ok';
-      /** Total file size in bytes */
-      size: number;
-      /** File range translated from HTTP range */
-      range: FileRange;
-      /** Stream of file content, undefined for HEAD requests */
-      data?: Readable;
-    }
+    /** Will be translated to 200 OK or 206 Partial Content by the handler */
+    type: 'Ok';
+    /** Total file size in bytes */
+    size: number;
+    /** File range translated from HTTP range */
+    range: FileRange;
+    /** Stream of file content, undefined for HEAD requests */
+    data?: Readable;
+  };
 
 /** Common options for object store creation */
 export interface ObjectStoreOptions {
@@ -173,7 +173,7 @@ export type RequestHandlerOptions = {
   /** Object store to serve files from, @see HttpHelpers.createFsStore */
   store: ObjectStore;
   /** Here will go caching options... */
-}
+};
 
 /** Server configuration options */
 export type HttpServerOptions = {
@@ -189,7 +189,7 @@ export type HttpServerOptions = {
 
 /**
  * Long unique opaque string for use in Bearer authorization header
- * 
+ *
  * @example
  * ```ts
  * request.setHeader('Authorization', `Bearer ${authToken}`);
@@ -199,7 +199,7 @@ export type HttpAuthorizationToken = Branded<string, 'PFrameInternal.HttpAuthori
 
 /**
  * TLS certificate in PEM format
- * 
+ *
  * @example
  * ```txt
  * -----BEGIN CERTIFICATE-----

package/src/pframe/internal_api/pframe.ts

@@ -1,6 +1,9 @@
 import type { PFrameFactoryAPIV4 } from './api_factory';
-import type { PFrameReadAPIV10 } from './api_read';
+import type { PFrameReadAPIV10, PFrameReadAPIV11 } from './api_read';
 import type { Logger } from './common';
+import type { PFrameId } from './common';
+
+// REMOVE AFTER MIGRATION BEGIN...
 
 export interface PFrameV12 extends PFrameFactoryAPIV4, PFrameReadAPIV10 {}
 
@@ -9,7 +12,7 @@ export type PFrameOptions = {
   spillPath: string;
   /** Logger instance, no logging is performed when not provided */
   logger?: Logger;
-}
+};
 
 /** List of PFrame management functions exposed by PFrame module */
 export interface PFrameFactoryV3 {
@@ -28,3 +31,34 @@ export interface PFrameFactoryV3 {
    */
   pprofDump: () => Promise<Uint8Array>;
 };
+
+// ...REMOVE AFTER MIGRATION END
+
+export interface PFrameV13 extends PFrameFactoryAPIV4, PFrameReadAPIV11 {}
+
+export type PFrameOptionsV2 = {
+  /** PFrame ID for logging purposes */
+  frameId: PFrameId;
+  /** Path to directory where PFrame can create temporary files */
+  spillPath: string;
+  /** Logger instance, no logging is performed when not provided */
+  logger?: Logger;
+};
+
+/** List of PFrame management functions exposed by PFrame module */
+export interface PFrameFactoryV4 {
+  /**
+   * Create a new PFrame instance.
+   * @warning Use concurrency limiting to avoid OOM crashes when multiple instances are simultaneously in use.
+   */
+  createPFrame(options: PFrameOptionsV2): PFrameV13;
+
+  /**
+   * Dump active allocations from all PFrames instances in pprof format.
+   * The result of this function should be saved as `profile.pb.gz`.
+   * Use {@link https://pprof.me/} or {@link https://www.speedscope.app/}
+   * to view the allocation flamechart.
+   * @warning This method will always reject on Windows!
+   */
+  pprofDump: () => Promise<Uint8Array>;
+};

package/src/pframe/internal_api/table.ts

@@ -1,12 +1,15 @@
-import {
+import type {
   PTableColumnId,
   PTableColumnSpec,
   PTableRecordFilter,
   PTableShape,
   PTableSorting,
   PTableVector,
-  TableRange
+  TableRange,
 } from '@milaboratories/pl-model-common';
+import type { PTableId } from './common';
+
+// REMOVE AFTER MIGRATION BEGIN...
 
 /**
  * Table view returned as a result of create table operation.
@@ -34,8 +37,7 @@ export interface PTableV7 extends Disposable {
    * Warning: This call materializes the join.
    */
   getFootprint(ops?: {
-    withPredecessors?: boolean,
-    signal?: AbortSignal,
+    signal?: AbortSignal;
   }): Promise<number>;
 
   /**
@@ -43,7 +45,7 @@ export interface PTableV7 extends Disposable {
    * Warning: This call materializes the join.
    */
   getShape(ops?: {
-    signal?: AbortSignal,
+    signal?: AbortSignal;
   }): Promise<PTableShape>;
 
   /**
@@ -58,8 +60,8 @@ export interface PTableV7 extends Disposable {
   getData(
     columnIndices: number[],
     ops?: {
-      range?: TableRange,
-      signal?: AbortSignal,
+      range?: TableRange;
+      signal?: AbortSignal;
     },
   ): Promise<PTableVector[]>;
 
@@ -72,3 +74,69 @@ export interface PTableV7 extends Disposable {
   /** Deallocates all underlying resources */
   dispose(): void;
 }
+
+// ...REMOVE AFTER MIGRATION END
+
+/**
+ * Table view returned as a result of create table operation.
+ *
+ * PTable can be thought as having a composite primary key, consisting of axes,
+ * and a set of data columns derived from PColumn values.
+ * */
+export interface PTableV8 extends Disposable {
+  /**
+   * Returns ordered array of table axes specs (primary key "columns" in SQL
+   * terms) and data column specs (regular "columns" in SQL terms).
+   *
+   * Data for a specific table column can be retrieved using unified indexing
+   * corresponding to elements in this array.
+   *
+   * Axes are always listed first.
+   * */
+  getSpec(): PTableColumnSpec[];
+
+  /** Transforms unified column identifiers into unified indices of columns. */
+  getColumnIndices(columnIds: PTableColumnId[]): number[];
+
+  /**
+   * Get PTable disk footprint in bytes
+   * Warning: This call materializes the join.
+   */
+  getFootprint(ops?: {
+    signal?: AbortSignal;
+  }): Promise<number>;
+
+  /**
+   * Unified table shape
+   * Warning: This call materializes the join.
+   */
+  getShape(ops?: {
+    signal?: AbortSignal;
+  }): Promise<PTableShape>;
+
+  /**
+   * Retrieve the data from the table. To retrieve only data required, it can be
+   * sliced both horizontally ({@link columnIndices}) and vertically
+   * ({@link range}).
+   * This call materializes the join.
+   *
+   * @param columnIndices unified indices of columns to be retrieved
+   * @param range optionally limit the range of records to retrieve
+   * */
+  getData(
+    columnIndices: number[],
+    ops?: {
+      range?: TableRange;
+      signal?: AbortSignal;
+    },
+  ): Promise<PTableVector[]>;
+
+  /** Filters the table and returns new PTable instance */
+  filter(tableId: PTableId, request: PTableRecordFilter[]): PTableV8;
+
+  /** Sorts the table and returns new PTable instance. */
+  sort(tableId: PTableId, request: PTableSorting[]): PTableV8;
+
+  /** Deallocates all underlying resources */
+  dispose(): void;
+}

package/src/project_list.ts

@@ -1,4 +1,4 @@
-import { ProjectMeta } from './project';
+import type { ProjectMeta } from './project';
 
 /** Represents single entry in the list of projects owned by current user */
 export interface ProjectListEntry {

package/src/project_overview.ts

@@ -1,8 +1,8 @@
-import { ProjectMeta } from './project';
-import { BlockPackSpec } from './block_registry/block_pack_spec';
-import { BlockCodeFeatureFlags, BlockRenderingMode, BlockSection, NavigationState, AuthorMarker } from '@milaboratories/pl-model-common';
-import { UpdateSuggestions } from './update_info';
-import { BlockSettings } from './block_settings';
+import type { ProjectMeta } from './project';
+import type { BlockPackSpec } from './block_registry/block_pack_spec';
+import type { BlockCodeFeatureFlags, BlockRenderingMode, BlockSection, NavigationState, AuthorMarker } from '@milaboratories/pl-model-common';
+import type { UpdateSuggestions } from './update_info';
+import type { BlockSettings } from './block_settings';
 
 /** Generalized block status, to be used in block item "styling". */
 export type BlockCalculationStatus = 'NotCalculated' | 'Running' | 'Done' | 'Limbo';

package/src/update_info.ts

@@ -1,4 +1,4 @@
-import { BlockPackSpec } from './block_registry';
+import type { BlockPackSpec } from './block_registry';
 
 /** Communicates possible block update option */
 export type UpdateSuggestion<V = BlockPackSpec> = {