@milaboratories/pl-model-common 1.3.9

package/src/drivers/pframe/table_calculate.ts

@@ -0,0 +1,295 @@
+import { PTableColumnId, PTableColumnSpec } from './table_common';
+import { PTableVector } from './data';
+import { assertNever } from '../../util';
+
+/** Defines a terminal column node in the join request tree */
+export interface ColumnJoinEntry<Col> {
+  /** Node type discriminator */
+  readonly type: 'column';
+
+  /** Local column */
+  readonly column: Col;
+}
+
+/**
+ * Defines a join request tree node that will output only records present in
+ * all child nodes ({@link entries}).
+ * */
+export interface InnerJoin<Col> {
+  /** Node type discriminator */
+  readonly type: 'inner';
+
+  /** Child nodes to be inner joined */
+  readonly entries: JoinEntry<Col>[];
+}
+
+/**
+ * Defines a join request tree node that will output all records present at
+ * least in one of the child nodes ({@link entries}), values for those PColumns
+ * that lacks corresponding combinations of axis values will be marked as absent,
+ * see {@link PTableVector.absent}.
+ * */
+export interface FullJoin<Col> {
+  /** Node type discriminator */
+  readonly type: 'full';
+
+  /** Child nodes to be fully outer joined */
+  readonly entries: JoinEntry<Col>[];
+}
+
+/**
+ * Defines a join request tree node that will output all records present in
+ * {@link primary} child node, and records from the {@link secondary} nodes will
+ * be added to the output only if present, values for those PColumns from the
+ * {@link secondary} list, that lacks corresponding combinations of axis values
+ * will be marked as absent, see {@link PTableVector.absent}.
+ *
+ * This node can be thought as a chain of SQL LEFT JOIN operations starting from
+ * the {@link primary} node and adding {@link secondary} nodes one by one.
+ * */
+export interface OuterJoin<Col> {
+  /** Node type discriminator */
+  readonly type: 'outer';
+
+  /** Primes the join operation. Left part of LEFT JOIN. */
+  readonly primary: JoinEntry<Col>;
+
+  /** Driven nodes, giving their values only if primary node have corresponding
+   * nodes. Right parts of LEFT JOIN chain. */
+  readonly secondary: JoinEntry<Col>[];
+}
+
+/**
+ * Base type of all join request tree nodes. Join request tree allows to combine
+ * information from multiple PColumns into a PTable. Correlation between records
+ * is performed by looking for records with the same values in common axis between
+ * the PColumns. Common axis are those axis which have equal {@link AxisId} derived
+ * from the columns axes spec.
+ * */
+export type JoinEntry<Col> =
+  | ColumnJoinEntry<Col>
+  | InnerJoin<Col>
+  | FullJoin<Col>
+  | OuterJoin<Col>;
+
+/** Container representing whole data stored in specific PTable column. */
+export interface FullPTableColumnData {
+  /** Unified spec */
+  readonly spec: PTableColumnSpec;
+
+  /** Data */
+  readonly data: PTableVector;
+}
+
+export interface SingleValueIsNAPredicate {
+  /** Comparison operator */
+  readonly operator: 'IsNA';
+}
+
+export interface SingleValueEqualPredicate {
+  /** Comparison operator */
+  readonly operator: 'Equal';
+
+  /** Reference value, NA values will not match */
+  readonly reference: string | number;
+}
+
+export interface SingleValueLessPredicate {
+  /** Comparison operator */
+  readonly operator: 'Less';
+
+  /** Reference value, NA values will not match */
+  readonly reference: string | number;
+}
+
+export interface SingleValueLessOrEqualPredicate {
+  /** Comparison operator */
+  readonly operator: 'LessOrEqual';
+
+  /** Reference value, NA values will not match */
+  readonly reference: string | number;
+}
+
+export interface SingleValueGreaterPredicate {
+  /** Comparison operator */
+  readonly operator: 'Greater';
+
+  /** Reference value, NA values will not match */
+  readonly reference: string | number;
+}
+
+export interface SingleValueGreaterOrEqualPredicate {
+  /** Comparison operator */
+  readonly operator: 'GreaterOrEqual';
+
+  /** Reference value, NA values will not match */
+  readonly reference: string | number;
+}
+
+export interface SingleValueStringContainsPredicate {
+  /** Comparison operator */
+  readonly operator: 'StringContains';
+
+  /** Reference substring, NA values are skipped */
+  readonly substring: string;
+}
+
+export interface SingleValueMatchesPredicate {
+  /** Comparison operator */
+  readonly operator: 'Matches';
+
+  /** Regular expression, NA values are skipped */
+  readonly regex: string;
+}
+
+export interface SingleValueStringContainsFuzzyPredicate {
+  /** Comparison operator */
+  readonly operator: 'StringContainsFuzzy';
+
+  /** Reference value, NA values are skipped */
+  readonly reference: string;
+
+  /**
+   * Integer specifying the upper bound of edit distance between
+   * reference and actual value.
+   * When {@link substitutionsOnly} is not defined or set to false
+   * Levenshtein distance is used (substitutions and indels)
+   * @see https://en.wikipedia.org/wiki/Levenshtein_distance
+   * When {@link substitutionsOnly} is set to true
+   * Hamming distance is used (substitutions only)
+   * @see https://en.wikipedia.org/wiki/Hamming_distance
+   */
+  readonly maxEdits: number;
+
+  /** Changes the type of edit distance in {@link maxEdits} */
+  readonly substitutionsOnly?: boolean;
+
+  /**
+   * Some character in {@link reference} that will match any
+   * single character in searched text.
+   */
+  readonly wildcard?: string;
+}
+
+export interface SingleValueNotPredicate {
+  /** Comparison operator */
+  readonly operator: 'Not';
+
+  /** Operand to negate */
+  readonly operand: SingleValuePredicate;
+}
+
+export interface SingleValueAndPredicate {
+  /** Comparison operator */
+  readonly operator: 'And';
+
+  /** Operands to combine */
+  readonly operands: SingleValuePredicate[];
+}
+
+export interface SingleValueOrPredicate {
+  /** Comparison operator */
+  readonly operator: 'Or';
+
+  /** Operands to combine */
+  readonly operands: SingleValuePredicate[];
+}
+
+/** Filtering predicate for a single axis or column value */
+export type SingleValuePredicate =
+  | SingleValueIsNAPredicate
+  | SingleValueEqualPredicate
+  | SingleValueLessPredicate
+  | SingleValueLessOrEqualPredicate
+  | SingleValueGreaterPredicate
+  | SingleValueGreaterOrEqualPredicate
+  | SingleValueStringContainsPredicate
+  | SingleValueMatchesPredicate
+  | SingleValueStringContainsFuzzyPredicate
+  | SingleValueNotPredicate
+  | SingleValueAndPredicate
+  | SingleValueOrPredicate;
+
+/**
+ * Filter PTable records based on specific axis or column value. If this is an
+ * axis value filter and the axis is part of a partitioning key in some of the
+ * source PColumns, the filter will be pushed down to those columns, so only
+ * specific partitions will be retrieved from the remote storage.
+ * */
+export interface PTableRecordSingleValueFilter {
+  /** Filter type discriminator */
+  readonly type: 'bySingleColumn';
+
+  /** Target axis selector to examine values from */
+  readonly column: PTableColumnId;
+
+  /** Value predicate */
+  readonly predicate: SingleValuePredicate;
+}
+
+/** Generic PTable records filter */
+export type PTableRecordFilter = PTableRecordSingleValueFilter;
+
+/** Sorting parameters for a PTable.  */
+export type PTableSorting = {
+  /** Unified column identifier */
+  readonly column: PTableColumnId;
+
+  /** Sorting order */
+  readonly ascending: boolean;
+
+  /** Sorting in respect to NA and absent values */
+  readonly naAndAbsentAreLeastValues: boolean;
+};
+
+/** Information required to instantiate a PTable. */
+export interface PTableDef<Col> {
+  /** Join tree to populate the PTable */
+  readonly src: JoinEntry<Col>;
+
+  /** Record filters */
+  readonly filters: PTableRecordFilter[];
+
+  /** Table sorting */
+  readonly sorting: PTableSorting[];
+}
+
+/** Request to create and retrieve entirety of data of PTable. */
+export type CalculateTableDataRequest<Col> = PTableDef<Col>;
+
+/** Response for {@link CalculateTableDataRequest} */
+export type CalculateTableDataResponse = FullPTableColumnData[];
+
+export function mapPTableDef<C1, C2>(
+  def: PTableDef<C1>,
+  cb: (c: C1) => C2
+): PTableDef<C2> {
+  return { ...def, src: mapJoinEntry(def.src, cb) };
+}
+
+export function mapJoinEntry<C1, C2>(
+  entry: JoinEntry<C1>,
+  cb: (c: C1) => C2
+): JoinEntry<C2> {
+  switch (entry.type) {
+    case 'column':
+      return {
+        type: 'column',
+        column: cb(entry.column)
+      };
+    case 'inner':
+    case 'full':
+      return {
+        type: entry.type,
+        entries: entry.entries.map((col) => mapJoinEntry(col, cb))
+      };
+    case 'outer':
+      return {
+        type: 'outer',
+        primary: mapJoinEntry(entry.primary, cb),
+        secondary: entry.secondary.map((col) => mapJoinEntry(col, cb))
+      };
+    default:
+      assertNever(entry);
+  }
+}

package/src/drivers/pframe/table_common.ts

@@ -0,0 +1,28 @@
+import { PObjectId } from '../../pool';
+import { AxisId, AxisSpec, PColumnSpec } from './spec';
+
+/** Unified spec object for axes and columns */
+export type PTableColumnSpec =
+  | {
+      type: 'axis';
+      id: AxisId;
+      spec: AxisSpec;
+    }
+  | {
+      type: 'column';
+      id: PObjectId;
+      spec: PColumnSpec;
+    };
+
+export type PTableColumnIdAxis = {
+  type: 'axis';
+  id: AxisId;
+};
+
+export type PTableColumnIdColumn = {
+  type: 'column';
+  id: PObjectId;
+};
+
+/** Unified PTable column identifier */
+export type PTableColumnId = PTableColumnIdAxis | PTableColumnIdColumn;

package/src/drivers/pframe/type_util.ts

@@ -0,0 +1,15 @@
+export type AddParameters<
+  TParameters extends [...args: any],
+  TFunction extends (...args: any) => any
+> = (
+  ...args: [...TParameters, ...Parameters<TFunction>]
+) => ReturnType<TFunction>;
+
+export type AddParameterToAllMethods<
+  Interface,
+  TParameters extends [...args: any]
+> = {
+  [Field in keyof Interface]: Interface[Field] extends (...args: any) => any
+    ? AddParameters<TParameters, Interface[Field]>
+    : Interface[Field];
+};

package/src/drivers/pframe/unique_values.ts

@@ -0,0 +1,27 @@
+import { AxisId } from './spec';
+import { PTableRecordFilter } from './table_calculate';
+import { PTableVector } from './data';
+import { PObjectId } from '../../pool';
+
+/** Calculate set of unique values for a specific axis for the filtered set of records */
+export interface UniqueValuesRequest {
+  /** Target axis id */
+  readonly columnId: PObjectId;
+
+  /** Target axis id, if not specified calculates unique column values */
+  readonly axis?: AxisId;
+
+  /** Filters to apply before calculating unique values */
+  readonly filters: PTableRecordFilter[];
+
+  /** Max number of values to return, if reached response will contain overflow flag */
+  readonly limit: number;
+}
+
+export interface UniqueValuesResponse {
+  /** Unique values */
+  readonly values: PTableVector;
+
+  /** True if limit was reached and response contain non-exhaustive list of values. */
+  readonly overflow: boolean;
+}

package/src/drivers/upload.ts

@@ -0,0 +1,22 @@
+export interface ImportProgress {
+  done: boolean;
+  /** Status of indexing/uploading got from platforma gRPC. */
+  status?: ImportStatus;
+  /** True if BlobUpload, false if BlobIndex. */
+  readonly isUpload: boolean;
+  /** True if signature matched. */
+  isUploadSignMatch?: boolean;
+  /** Exists when an upload failed and was restarted,
+   * but the error was recoverable.
+   * If the error was non-recoverable,
+   * the driver's computable will throw an error instead. */
+  lastError?: string;
+}
+
+/** Almost direct mapping from proto struct got from gRPC. */
+export interface ImportStatus {
+  /** A float from 0 to 1 and is equal to bytesProcessed / bytesTotal. */
+  readonly progress: number;
+  readonly bytesProcessed?: number;
+  readonly bytesTotal?: number;
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export * from './block';
+export * from './block_state';
+export * from './utag';
+export * from './drivers';
+export * from './driver_kit';
+export * from './navigation';
+export * from './ref';
+export * from './common_types';
+export * from './pool';
+export * from './value_or_error';

package/src/navigation.ts

@@ -0,0 +1,31 @@
+/** Block section visualized as items in left panel block overview */
+export type BlockSection = BlockSectionLink | BlockSectionDelimiter;
+
+/** Tells the system that specific section from the main UI of this block should
+ * be opened */
+export type BlockSectionLink = {
+  /** Potentially there may be multiple section types, i.e. for "+" rows and for
+   * sections directly opening html from the outputs. */
+  readonly type: 'link';
+
+  /** Internal block section identifier */
+  readonly href: `/${string}`;
+
+  /** Visible section title, can also be used in the window header. */
+  readonly label: string;
+};
+
+/** Create a horisontal line between sections */
+export type BlockSectionDelimiter = {
+  readonly type: 'delimiter';
+};
+
+/**
+ * Part of the block state, representing current navigation information
+ * (i.e. currently selected section)
+ * */
+export type NavigationState<Href extends `/${string}` = `/${string}`> = {
+  readonly href: Href;
+};
+
+export const DefaultNavigationState: NavigationState = { href: '/' };

package/src/pool/entry.ts

@@ -0,0 +1,20 @@
+import { Ref } from '../ref';
+import { PObject } from './spec';
+
+/** Result pool entry */
+export type ResultPoolEntry<O> = {
+  /** Reference that can be passed to args to import the object in workflow */
+  readonly ref: Ref;
+
+  /** Object. Normally spec or PObject. */
+  readonly obj: O;
+};
+
+/** Collection of results from the result pool */
+export type ResultCollection<O> = {
+  /** List of results from the pool */
+  readonly entries: ResultPoolEntry<O>[];
+
+  /** False means that current collection is not fully loaded due to some computations still taking place */
+  readonly isComplete: boolean;
+};

package/src/pool/index.ts

@@ -0,0 +1,3 @@
+export * from './entry';
+export * from './spec';
+export * from './query';

package/src/pool/query.ts

@@ -0,0 +1,93 @@
+import { AxisId } from '../drivers';
+import { PObjectSpec, isPColumnSpec } from './spec';
+
+export type PSpecPredicate =
+  | {
+      type: 'and' | 'or';
+      operands: PSpecPredicate[];
+    }
+  | {
+      type: 'not';
+      operand: PSpecPredicate;
+    }
+  | {
+      type: 'name';
+      name: string;
+    }
+  | {
+      type: 'name_pattern';
+      pattern: string;
+    }
+  | {
+      type: 'annotation';
+      annotation: string;
+      value: string;
+    }
+  | {
+      type: 'annotation_pattern';
+      annotation: string;
+      pattern: string;
+    }
+  | {
+      type: 'has_axes';
+      axes: Partial<AxisId>[];
+    };
+
+function assertNever(x: never): never {
+  throw new Error('Unexpected object: ' + x);
+}
+
+export function executePSpecPredicate(
+  predicate: PSpecPredicate,
+  spec: PObjectSpec
+): boolean {
+  switch (predicate.type) {
+    case 'and':
+      for (const operator of predicate.operands)
+        if (!executePSpecPredicate(operator, spec)) return false;
+      return true;
+    case 'or':
+      for (const operator of predicate.operands)
+        if (executePSpecPredicate(operator, spec)) return true;
+      return false;
+    case 'not':
+      return !executePSpecPredicate(predicate.operand, spec);
+    case 'name':
+      return isPColumnSpec(spec) && spec.name === predicate.name;
+    case 'name_pattern':
+      return isPColumnSpec(spec) && Boolean(spec.name.match(predicate.pattern));
+    case 'annotation':
+      return (
+        isPColumnSpec(spec) &&
+        spec.annotations !== undefined &&
+        spec.annotations[predicate.annotation] === predicate.value
+      );
+    case 'annotation_pattern':
+      return (
+        isPColumnSpec(spec) &&
+        spec.annotations !== undefined &&
+        spec.annotations[predicate.annotation] !== undefined &&
+        Boolean(spec.annotations[predicate.annotation].match(predicate.pattern))
+      );
+    case 'has_axes':
+      return (
+        isPColumnSpec(spec) &&
+        predicate.axes.every((matcher) =>
+          spec.axesSpec.some(
+            (axisSpec) =>
+              (matcher.type === undefined || matcher.type === axisSpec.type) &&
+              (matcher.name === undefined || matcher.name === axisSpec.name) &&
+              (matcher.domain === undefined ||
+                Object.keys(matcher.domain).length === 0 ||
+                (axisSpec.domain !== undefined &&
+                  Object.entries(matcher.domain).every(
+                    ([domain, domainValue]) =>
+                      axisSpec.domain![domain] === domainValue
+                  )))
+          )
+        )
+      );
+    default:
+      assertNever(predicate);
+  }
+}

package/src/pool/spec.ts

@@ -0,0 +1,62 @@
+import { Branded } from '../branding';
+import { PColumn, PColumnSpec } from '../drivers';
+
+/** Any object exported into the result pool by the block always have spec attached to it */
+export interface PObjectSpec {
+  /** PObject kind discriminator */
+  readonly kind: string;
+
+  /** Additional information attached to the object */
+  readonly annotations?: Record<string, string>;
+}
+
+/** Stable PObject id */
+export type PObjectId = Branded<string, 'PColumnId'>;
+
+/**
+ * Full PObject representation.
+ *
+ * @template Data type of the object referencing or describing the "data" part of the PObject
+ * */
+export interface PObject<Data> {
+  /** Fully rendered PObjects are assigned a stable identifier. */
+  readonly id: PObjectId;
+
+  /** PObject spec, allowing it to be found among other PObjects */
+  readonly spec: PObjectSpec;
+
+  /** A handle to data object */
+  readonly data: Data;
+}
+
+export function isPColumnSpec(spec: PObjectSpec): spec is PColumnSpec {
+  return spec.kind === 'PColumn';
+}
+
+export function isPColumn<T>(obj: PObject<T>): obj is PColumn<T> {
+  return isPColumnSpec(obj.spec);
+}
+
+export function ensurePColumn<T>(obj: PObject<T>): PColumn<T> {
+  if (!isPColumn(obj))
+    throw new Error(`not a PColumn (kind = ${obj.spec.kind})`);
+  return obj;
+}
+
+export function mapPObjectData<D1, D2>(
+  pObj: PColumn<D1>,
+  cb: (d: D1) => D2
+): PColumn<D2>;
+export function mapPObjectData<D1, D2>(
+  pObj: PObject<D1>,
+  cb: (d: D1) => D2
+): PObject<D2>;
+export function mapPObjectData<D1, D2>(
+  pObj: PObject<D1>,
+  cb: (d: D1) => D2
+): PObject<D2> {
+  return {
+    ...pObj,
+    data: cb(pObj.data)
+  };
+}

package/src/ref.ts

@@ -0,0 +1,33 @@
+import { z } from 'zod';
+
+export const Ref = z
+  .object({
+    __isRef: z
+      .literal(true)
+      .describe('Crucial marker for the block dependency tree reconstruction'),
+    blockId: z.string().describe('Upstream block id'),
+    name: z
+      .string()
+      .describe(
+        "Name of the output provided to the upstream block's output context"
+      )
+  })
+  .describe(
+    'Universal reference type, allowing to set block connections. It is crucial that ' +
+      '{@link __isRef} is present and equal to true, internal logic relies on this marker ' +
+      'to build block dependency trees.'
+  )
+  .strict()
+  .readonly();
+export type Ref = z.infer<typeof Ref>;
+
+/** Standard way how to communicate possible connections given specific
+ * requirements for incoming data. */
+export type Option = {
+  /** Fully rendered reference to be assigned for the intended field in block's
+   * args */
+  readonly ref: Ref;
+
+  /** Label to be present for the user in i.e. drop-down list */
+  readonly label: string;
+};

package/src/utag.ts

@@ -0,0 +1,14 @@
+/** Value returned for changing states supporting reactive listening for changes */
+export interface ValueWithUTag<V> {
+  /** Value snapshot. */
+  readonly value: V;
+
+  /**
+   * Unique tag for the value snapshot.
+   *
+   * It can be used to synchronously detect if changes happened after current
+   * snapshot was retrieved, or asynchronously await next value snapshot,
+   * generated on underlying data changes.
+   * */
+  readonly uTag: string;
+}

package/src/util.ts

@@ -0,0 +1,3 @@
+export function assertNever(x: never): never {
+  throw new Error('Unexpected object: ' + x);
+}

package/src/value_or_error.ts

@@ -0,0 +1,16 @@
+export type ValueOrError<V, E> =
+  | {
+      ok: true;
+      value: V;
+    }
+  | {
+      ok: false;
+      error: E;
+    };
+
+export function mapValueInVOE<V1, V2, E>(
+  voe: ValueOrError<V1, E>,
+  cb: (value: V1) => V2
+): ValueOrError<V2, E> {
+  return voe.ok ? { ok: true, value: cb(voe.value) } : voe;
+}