@milaboratories/pl-model-common 1.3.9

package/src/drivers/log.ts

@@ -0,0 +1,79 @@
+/** Handle of logs. This handle should be passed
+ * to the driver for retrieving logs. */
+export type AnyLogHandle = LiveLogHandle | ReadyLogHandle;
+
+/** Handle of the live logs of a program.
+ * The resource that represents a log can be deleted,
+ * in this case the handle should be refreshed. */
+export type LiveLogHandle = `log+live://log/${string}`;
+
+/** Handle of the ready logs of a program. */
+export type ReadyLogHandle = `log+ready://log/${string}`;
+
+/** Driver to retrieve logs given log handle */
+export interface LogsDriver {
+  lastLines(
+    /** A handle that was issued previously. */
+    handle: AnyLogHandle,
+
+    /** Allows client to limit total data sent from server. */
+    lineCount: number,
+
+    /** Makes streamer to perform seek operation to given offset before sending the contents.
+     * Client can just use the <new_offset> value of the last response from server to continue streaming after reconnection.
+     * If undefined, then starts from the end. */
+    offsetBytes?: number,
+
+    /** Is substring for line search pattern.
+     * This option makes controller to send to the client only lines, that
+     * have given substring. */
+    searchStr?: string
+  ): Promise<StreamingApiResponse>;
+
+  readText(
+    /** A handle that was issued previously. */
+    handle: AnyLogHandle,
+
+    /** Allows client to limit total data sent from server. */
+    lineCount: number,
+
+    /** Makes streamer to perform seek operation to given offset before sending the contents.
+     * Client can just use the <new_offset> value of the last response from server to continue streaming after reconnection.
+     * If undefined of 0, then starts from the beginning. */
+    offsetBytes?: number,
+
+    /** Is substring for line search pattern.
+     * This option makes controller to send to the client only lines, that
+     * have given substring. */
+    searchStr?: string
+  ): Promise<StreamingApiResponse>;
+}
+
+/** Response of the driver.
+ * The caller should give a handle to retrieve it.
+ * It can be OK or outdated, in which case the handle
+ * should be issued again. */
+export type StreamingApiResponse =
+  | StreamingApiResponseOk
+  | StreamingApiResponseHandleOutdated;
+
+export type StreamingApiResponseOk = {
+  /** The handle don't have to be updated,
+   * the response is OK. */
+  shouldUpdateHandle: false;
+
+  /** Whether the log can still grow or it's in a final state. */
+  live: boolean;
+
+  /** Data of the response, in bytes. */
+  data: Uint8Array;
+  /** Current size of the file. It can grow if it's still live. */
+  size: number;
+  /** Offset in bytes from the beginning of a file. */
+  newOffset: number;
+};
+
+/** The handle should be issued again, this one is done. */
+export type StreamingApiResponseHandleOutdated = {
+  shouldUpdateHandle: true;
+};

package/src/drivers/ls.ts

@@ -0,0 +1,80 @@
+import { assertNever } from '../util';
+
+const uploadPrefix = 'upload://upload/';
+const indexPrefix = 'index://index/';
+
+export type ImportFileHandleUpload = `upload://upload/${string}`;
+export type ImportFileHandleIndex = `index://index/${string}`;
+
+export type ImportFileHandle = ImportFileHandleUpload | ImportFileHandleIndex;
+
+export function isImportFileHandleUpload(
+  handle: ImportFileHandle
+): handle is ImportFileHandleUpload {
+  return handle.startsWith(uploadPrefix);
+}
+
+export function isImportFileHandleIndex(
+  handle: ImportFileHandle
+): handle is ImportFileHandleIndex {
+  return handle.startsWith(indexPrefix);
+}
+
+/** Results in upload */
+export type StorageHandleLocal = `local://${string}`;
+
+/** Results in index */
+export type StorageHandleRemote = `remote://${string}`;
+
+export type StorageHandle = StorageHandleLocal | StorageHandleRemote;
+
+export type StorageEntry = {
+  name: string;
+  handle: StorageHandle;
+  initialFullPath: string;
+
+  // TODO
+  // pathStartsWithDisk
+};
+
+export type ListFilesResult = {
+  parent?: string;
+  entries: LsEntry[];
+};
+
+export type LsEntry =
+  | {
+      type: 'dir';
+      name: string;
+      fullPath: string;
+    }
+  | {
+      type: 'file';
+      name: string;
+      fullPath: string;
+
+      /** This handle should be set to args... */
+      handle: ImportFileHandle;
+    };
+
+export interface LsDriver {
+  /** remote and local storages */
+  getStorageList(): Promise<StorageEntry[]>;
+
+  listFiles(storage: StorageHandle, fullPath: string): Promise<ListFilesResult>;
+}
+
+/** Gets a file path from an import handle. */
+export function getFilePathFromHandle(handle: ImportFileHandle): string {
+  if (isImportFileHandleIndex(handle)) {
+    const trimmed = handle.slice(indexPrefix.length);
+    const data = JSON.parse(decodeURIComponent(trimmed));
+    return data.path;
+  } else if (isImportFileHandleUpload(handle)) {
+    const trimmed = handle.slice(uploadPrefix.length);
+    const data = JSON.parse(decodeURIComponent(trimmed));
+    return data.localPath;
+  }
+
+  assertNever(handle);
+}

package/src/drivers/pframe/column_filter.ts

@@ -0,0 +1,19 @@
+import { ValueType } from './spec';
+
+/** Allows to search multiple columns in different contexts. */
+export interface ColumnFilter {
+  /** Match any of the types listed here. If undefined, will be ignored during
+   * matching. */
+  readonly type?: ValueType[];
+
+  /** Match any of the names listed here. If undefined, will be ignored during
+   * matching. */
+  readonly name?: string[];
+
+  /** Match requires all the annotations listed here to have corresponding values. */
+  readonly annotationValue?: Record<string, string>;
+
+  /** Match requires all the annotations listed here to match corresponding regex
+   * pattern. */
+  readonly annotationPattern?: Record<string, string>;
+}

package/src/drivers/pframe/data.ts

@@ -0,0 +1,67 @@
+import { ValueType } from './spec';
+
+export const PValueIntNA = -2147483648;
+export const PValueLongNA = -9007199254740991n;
+export const PValueFloatNA = NaN;
+export const PValueDoubleNA = NaN;
+export const PValueStringNA = null;
+export const PValueBytesNA = null;
+
+export type PValueInt = number;
+export type PValueLong = number; // use bigint only if extra integer precision is needed
+export type PValueFloat = number;
+export type PValueDouble = number;
+export type PValueString = string | null;
+export type PValueBytes = Uint8Array | null;
+
+export type PVectorDataInt = Int32Array;
+export type PVectorDataLong = BigInt64Array;
+export type PVectorDataFloat = Float32Array;
+export type PVectorDataDouble = Float64Array;
+export type PVectorDataString = PValueString[];
+export type PVectorDataBytes = PValueBytes[];
+
+export type PVectorData =
+  | PVectorDataInt
+  | PVectorDataLong
+  | PVectorDataFloat
+  | PVectorDataDouble
+  | PVectorDataString
+  | PVectorDataBytes;
+
+/** Table column data in comparison to the data stored in a separate PColumn
+ * may have some of the values "absent", i.e. as a result of missing record in
+ * outer join operation. This information is encoded in {@link absent} field. */
+export interface PTableVector {
+  /** Stored data type */
+  readonly type: ValueType;
+
+  /** Values for present positions, absent positions have NA values */
+  readonly data: PVectorData;
+
+  /**
+   * Encoded bit array marking some elements of this vector as absent,
+   *
+   * Encoding described here:
+   * https://gist.github.com/vadimpiven/45f8279d84f47b9857df845126694c39
+   * */
+  readonly absent: Uint8Array;
+}
+
+/** Used in requests to partially retrieve table's data */
+export type TableRange = {
+  /** Index of the first record to retrieve */
+  readonly offset: number;
+
+  /** Block length */
+  readonly length: number;
+};
+
+/** Unified information about table shape */
+export type PTableShape = {
+  /** Number of unified table columns, including all axes and PColumn values */
+  columns: number;
+
+  /** Number of rows */
+  rows: number;
+};

package/src/drivers/pframe/driver.ts

@@ -0,0 +1,110 @@
+import { Branded } from '../../branding';
+import { PTable } from './table';
+import { PFrame } from './pframe';
+import { AddParameterToAllMethods } from './type_util';
+import { PTableShape, PTableVector, TableRange } from './data';
+import { FindColumnsRequest, FindColumnsResponse } from './find_columns';
+import { PObjectId } from '../../pool';
+import { PColumnIdAndSpec, PColumnSpec } from './spec';
+import {
+  CalculateTableDataRequest,
+  CalculateTableDataResponse
+} from './table_calculate';
+import { UniqueValuesRequest, UniqueValuesResponse } from './unique_values';
+import { PTableColumnSpec } from './table_common';
+
+/** PFrame handle */
+export type PFrameHandle = Branded<string, 'PFrame'>;
+
+/** PFrame handle */
+export type PTableHandle = Branded<string, 'PTable'>;
+
+/** Allows to access main data layer features of platforma */
+export interface PFrameDriver {
+  //
+  // PFrame methods
+  //
+
+  /**
+   * Finds columns given filtering criteria on column name, annotations etc.
+   * and a set of axes ids to find only columns with compatible specs.
+   * */
+  findColumns(
+    handle: PFrameHandle,
+    request: FindColumnsRequest
+  ): Promise<FindColumnsResponse>;
+
+  /** Retrieve single column spec */
+  getColumnSpec(
+    handle: PFrameHandle,
+    columnId: PObjectId
+  ): Promise<PColumnSpec>;
+
+  /** Retrieve information about all columns currently added to the PFrame */
+  listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]>;
+
+  /** Calculates data for the table and returns complete data representation of it */
+  calculateTableData(
+    handle: PFrameHandle,
+    request: CalculateTableDataRequest<PObjectId>
+  ): Promise<CalculateTableDataResponse>;
+
+  /** Calculate set of unique values for a specific axis for the filtered set of records */
+  getUniqueValues(
+    handle: PFrameHandle,
+    request: UniqueValuesRequest
+  ): Promise<UniqueValuesResponse>;
+
+  //
+  // PTable methods
+  //
+
+  /** Unified table shape */
+  getShape(handle: PTableHandle): Promise<PTableShape>;
+
+  /**
+   * Returns ordered array of table axes specs (primary key "columns" in SQL
+   * terms) and data column specs (regular "columns" in SQL terms).
+   *
+   * Data for a specific table column can be retrieved using unified indexing
+   * corresponding to elements in this array.
+   *
+   * Axes are always listed first.
+   * */
+  getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]>;
+
+  /**
+   * Retrieve the data from the table. To retrieve only data required, it can be
+   * sliced both horizontally ({@link columnIndices}) and vertically
+   * ({@link range}).
+   *
+   * @param columnIndices unified indices of columns to be retrieved
+   * @param range optionally limit the range of records to retrieve
+   * */
+  getData(
+    handle: PTableHandle,
+    columnIndices: number[],
+    range?: TableRange | undefined
+  ): Promise<PTableVector[]>;
+}
+
+//
+// The following keeps the PFrame driver from above to be in sync with
+// PFrame and PTable interfaces.
+//
+
+type ExpectedPFrameDriverTypeF = AddParameterToAllMethods<
+  PFrame,
+  [handle: PFrameHandle]
+>;
+type ExpectedPFrameDriverTypeT = AddParameterToAllMethods<
+  PTable,
+  [handle: PTableHandle]
+>;
+type ExpectedPFrameDriverType = ExpectedPFrameDriverTypeF &
+  ExpectedPFrameDriverTypeT;
+
+type TypeEqualityGuard<A, B> = Exclude<A, B> | Exclude<B, A>;
+function assert<T extends never>() {}
+
+assert<TypeEqualityGuard<PFrameDriver, ExpectedPFrameDriverType>>();

package/src/drivers/pframe/find_columns.ts

@@ -0,0 +1,30 @@
+import { ColumnFilter } from './column_filter';
+import { AxisId, PColumnIdAndSpec } from './spec';
+
+/**
+ * Request to search among existing columns in the PFrame. Two filtering
+ * criteria can be used: (1) column ашдеук, to search for columns with
+ * specific properties like name, annotations and domains, and (2) being
+ * compatible with the given list of axis ids.
+ * */
+export interface FindColumnsRequest {
+  /** Basic column filter */
+  readonly columnFilter: ColumnFilter;
+
+  /** Will only search for columns compatible with these list of axis ids */
+  readonly compatibleWith: AxisId[];
+
+  /**
+   * Defines what level of compatibility with provided list of axis ids is required.
+   *
+   * If true will search only for such columns which axes completely maps onto the
+   * axes listed in the {@link compatibleWith} list.
+   * */
+  readonly strictlyCompatible: boolean;
+}
+
+/** Response for {@link FindColumnsRequest} */
+export interface FindColumnsResponse {
+  /** Array of column ids found using request criteria. */
+  readonly hits: PColumnIdAndSpec[];
+}

package/src/drivers/pframe/index.ts

@@ -0,0 +1,11 @@
+export * from './column_filter';
+export * from './data';
+export * from './find_columns';
+export * from './pframe';
+export * from './spec';
+export * from './table';
+export * from './table_calculate';
+export * from './table_common';
+export * from './unique_values';
+
+export * from './driver';

package/src/drivers/pframe/pframe.ts

@@ -0,0 +1,34 @@
+import { PObjectId } from '../../pool';
+import { FindColumnsRequest, FindColumnsResponse } from './find_columns';
+import { PColumn, PColumnIdAndSpec, PColumnSpec } from './spec';
+import {
+  CalculateTableDataRequest,
+  CalculateTableDataResponse
+} from './table_calculate';
+import { UniqueValuesRequest, UniqueValuesResponse } from './unique_values';
+
+/** Read interface exposed by PFrames library */
+export interface PFrame {
+  /**
+   * Finds columns given filtering criteria on column name, annotations etc.
+   * and a set of axes ids to find only columns with compatible specs.
+   * */
+  findColumns(request: FindColumnsRequest): Promise<FindColumnsResponse>;
+
+  /** Retrieve single column spec */
+  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec>;
+
+  /** Retrieve information about all columns currently added to the PFrame */
+  listColumns(): Promise<PColumnIdAndSpec[]>;
+
+  /** Calculates data for the table and returns complete data representation of it */
+  calculateTableData(
+    request: CalculateTableDataRequest<PObjectId>
+  ): Promise<CalculateTableDataResponse>;
+
+  /** Calculate set of unique values for a specific axis for the filtered set of records */
+  getUniqueValues(request: UniqueValuesRequest): Promise<UniqueValuesResponse>;
+}
+
+/** Information required to instantiate a PFrame. */
+export type PFrameDef<Data> = PColumn<Data>[];

package/src/drivers/pframe/spec.ts

@@ -0,0 +1,139 @@
+import { PObject, PObjectId, PObjectSpec } from '../../pool';
+
+/** PFrame columns and axes within them may store one of these types. */
+export type ValueType =
+  | 'Int'
+  | 'Long'
+  | 'Float'
+  | 'Double'
+  | 'String'
+  | 'Bytes';
+
+/**
+ * Specification of an individual axis.
+ *
+ * Each axis is a part of a composite key that addresses data inside the PColumn.
+ *
+ * Each record inside a PColumn is addressed by a unique tuple of values set for
+ * all the axes specified in the column spec.
+ * */
+export interface AxisSpec {
+  /** Type of the axis value. Should not use non-key types like float or double. */
+  readonly type: ValueType;
+
+  /** Name of the axis */
+  readonly name: string;
+
+  /** Adds auxiliary information to the axis name, type and parents to form a
+   * unique identifier */
+  readonly domain?: Record<string, string>;
+
+  /** Any additional information attached to the axis that does not affect its
+   * identifier */
+  readonly annotations?: Record<string, string>;
+
+  /**
+   * Parent axes provide contextual grouping for the axis in question, establishing
+   * a hierarchy where the current axis is dependent on one or more axes for its
+   * full definition and meaning. For instance, in a data structure where each
+   * "container" axis may contain multiple "item" axes, the `item` axis would
+   * list the index of the `container` axis in this field to denote its dependency.
+   *
+   * This means that the identity or significance of the `item` axis is only
+   * interpretable when combined with its parent `container` axis. An `item` axis
+   * index by itself may be non-unique and only gains uniqueness within the context
+   * of its parent `container`. Therefore, the `parentAxes` field is essential for
+   * mapping these relationships and ensuring data coherence across nested or
+   * multi-level data models.
+   *
+   * A list of zero-based indices of parent axes in the overall axes specification
+   * from the column spec. Each index corresponds to the position of a parent axis
+   * in the list that defines the structure of the data model.
+   */
+  readonly parentAxes?: number[];
+}
+
+/** Common type representing spec for all the axes in a column */
+export type AxesSpec = AxisSpec[];
+
+/**
+ * Full column specification including all axes specs and specs of the column
+ * itself.
+ *
+ * A PColumn in its essence represents a mapping from a fixed size, explicitly
+ * typed tuple to an explicitly typed value.
+ *
+ * (axis1Value1, axis2Value1, ...) -> columnValue
+ *
+ * Each element in tuple correspond to the axis having the same index in axesSpec.
+ * */
+export interface PColumnSpec extends PObjectSpec {
+  /** Defines specific type of BObject, the most generic type of unit of
+   * information in Platforma Project. */
+  readonly kind: 'PColumn';
+
+  /** Type of column values */
+  readonly valueType: ValueType;
+
+  /** Column name */
+  readonly name: string;
+
+  /** Adds auxiliary information to the axis name, type and parents to form a
+   * unique identifier */
+  readonly domain?: Record<string, string>;
+
+  /** Any additional information attached to the column that does not affect its
+   * identifier */
+  readonly annotations?: Record<string, string>;
+
+  /** A list of zero-based indices of parent axes from the {@link axesSpec} array. */
+  readonly parentAxes?: number[];
+
+  /** Axes specifications */
+  readonly axesSpec: AxesSpec;
+}
+
+export interface PColumn<Data> extends PObject<Data> {
+  /** PColumn spec, allowing it to be found among other PObjects */
+  readonly spec: PColumnSpec;
+}
+
+/** Columns in a PFrame also have internal identifier, this object represents
+ * combination of specs and such id */
+export interface PColumnIdAndSpec {
+  /** Internal column id within the PFrame */
+  readonly columnId: PObjectId;
+
+  /** Column spec */
+  readonly spec: PColumnSpec;
+}
+
+/** Information returned by {@link PFrame.listColumns} method */
+export interface PColumnInfo extends PColumnIdAndSpec {
+  /** True if data was associated with this PColumn */
+  readonly hasData: boolean;
+}
+
+export interface AxisId {
+  /** Type of the axis or column value. For an axis should not use non-key
+   * types like float or double. */
+  readonly type: ValueType;
+
+  /** Name of the axis or column */
+  readonly name: string;
+
+  /** Adds auxiliary information to the axis or column name and type to form a
+   * unique identifier */
+  readonly domain?: Record<string, string>;
+}
+
+/** Array of axis ids */
+export type AxesId = AxisId[];
+
+/** Extracts axes ids from axes spec array from column spec */
+export function getAxesId(spec: AxesSpec): AxesId {
+  return spec.map((s) => {
+    const { type, name, domain } = s;
+    return { type, name, domain };
+  });
+}

package/src/drivers/pframe/table.ts

@@ -0,0 +1,31 @@
+import { PTableColumnSpec } from './table_common';
+import { PTableShape, PTableVector, TableRange } from './data';
+
+/**
+ * Table view.
+ * */
+export interface PTable {
+  /** Unified table shape */
+  getShape(): Promise<PTableShape>;
+
+  /**
+   * Returns ordered array of table axes specs (primary key "columns" in SQL
+   * terms) and data column specs (regular "columns" in SQL terms).
+   *
+   * Data for a specific table column can be retrieved using unified indexing
+   * corresponding to elements in this array.
+   *
+   * Axes are always listed first.
+   * */
+  getSpec(): Promise<PTableColumnSpec[]>;
+
+  /**
+   * Retrieve the data from the table. To retrieve only data required, it can be
+   * sliced both horizontally ({@link columnIndices}) and vertically
+   * ({@link range}).
+   *
+   * @param columnIndices unified indices of columns to be retrieved
+   * @param range optionally limit the range of records to retrieve
+   * */
+  getData(columnIndices: number[], range?: TableRange): Promise<PTableVector[]>;
+}