@milaboratories/pl-model-common 1.11.4 → 1.13.0

package/src/drivers/pframe/data_info.ts

@@ -0,0 +1,430 @@
+/**
+ * Represents a JavaScript representation of a value in a PColumn. Can be null, a number, or a string.
+ * These are the primitive types that can be stored directly in PColumns.
+ *
+ * Note: Actual columns can hold more value types, which are converted to these JavaScript types
+ * once they enter the JavaScript runtime.
+ */
+export type PColumnValue = null | number | string;
+
+/**
+ * Represents a key for a PColumn value.
+ * Can be an array of strings or numbers.
+ */
+export type PColumnKey = (number | string)[];
+
+/**
+ * Represents a single entry in a PColumn's data structure.
+ * Contains a key and a value.
+ */
+export type PColumnDataEntry<T> = {
+  /** Key for the value */
+  key: PColumnKey;
+
+  /** Value / blob at the given key */
+  value: T;
+};
+
+/**
+ * Represents column data stored as a simple JSON structure.
+ * Used for small datasets that can be efficiently stored directly in memory.
+ */
+export type JsonDataInfo = {
+  /** Identifier for this data format ('Json') */
+  type: 'Json';
+
+  /** Number of axes that make up the complete key (tuple length) */
+  keyLength: number;
+
+  /**
+   * Key-value pairs where keys are stringified tuples of axis values
+   * and values are the column values for those coordinates
+   */
+  data: Record<string, PColumnValue>;
+};
+
+/**
+ * Represents column data partitioned across multiple JSON blobs.
+ * Used for larger datasets that need to be split into manageable chunks.
+ */
+export type JsonPartitionedDataInfo<Blob> = {
+  /** Identifier for this data format ('JsonPartitioned') */
+  type: 'JsonPartitioned';
+
+  /** Number of leading axes used for partitioning */
+  partitionKeyLength: number;
+
+  /** Map of stringified partition keys to blob references */
+  parts: Record<string, Blob>;
+};
+
+/**
+ * Represents a binary format chunk containing index and values as separate blobs.
+ * Used for efficient storage and retrieval of column data in binary format.
+ */
+export type BinaryChunk<Blob> = {
+  /** Binary blob containing structured index information */
+  index: Blob;
+
+  /** Binary blob containing the actual values */
+  values: Blob;
+};
+
+/**
+ * Represents column data partitioned across multiple binary chunks.
+ * Optimized for efficient storage and retrieval of large datasets.
+ */
+export type BinaryPartitionedDataInfo<Blob> = {
+  /** Identifier for this data format ('BinaryPartitioned') */
+  type: 'BinaryPartitioned';
+
+  /** Number of leading axes used for partitioning */
+  partitionKeyLength: number;
+
+  /** Map of stringified partition keys to binary chunks */
+  parts: Record<string, BinaryChunk<Blob>>;
+};
+
+/**
+ * Union type representing all possible data storage formats for PColumn data.
+ * The specific format used depends on data size, access patterns, and performance requirements.
+ *
+ * @template Blob - Type parameter representing the storage reference type (could be ResourceInfo, PFrameBlobId, etc.)
+ */
+export type DataInfo<Blob> =
+  | JsonDataInfo
+  | JsonPartitionedDataInfo<Blob>
+  | BinaryPartitionedDataInfo<Blob>;
+
+/**
+ * Type guard function that checks if the given value is a valid DataInfo.
+ *
+ * @param value - The value to check
+ * @returns True if the value is a valid DataInfo, false otherwise
+ */
+export function isDataInfo<Blob>(value: unknown): value is DataInfo<Blob> {
+  if (!value || typeof value !== 'object') {
+    return false;
+  }
+
+  const data = value as Record<string, unknown>;
+  if (!('type' in data)) {
+    return false;
+  }
+
+  switch (data.type) {
+    case 'Json':
+      return (
+        typeof data.keyLength === 'number'
+        && data.data !== undefined
+        && typeof data.data === 'object'
+      );
+    case 'JsonPartitioned':
+      return (
+        typeof data.partitionKeyLength === 'number'
+        && data.parts !== undefined
+        && typeof data.parts === 'object'
+      );
+    case 'BinaryPartitioned':
+      return (
+        typeof data.partitionKeyLength === 'number'
+        && data.parts !== undefined
+        && typeof data.parts === 'object'
+      );
+    default:
+      return false;
+  }
+}
+
+/**
+ * Maps blob references in a DataInfo object from one type to another using a mapping function.
+ *
+ * @template B1 - Source blob type
+ * @template B2 - Target blob type
+ * @param dataInfo - The source DataInfo object
+ * @param mapFn - Function to transform blobs from type B1 to type B2
+ * @returns A new DataInfo object with transformed blob references
+ */
+export function mapDataInfo<B1, B2>(
+  dataInfo: DataInfo<B1>,
+  mapFn: (blob: B1) => B2,
+): DataInfo<B2>;
+export function mapDataInfo<B1, B2>(
+  dataInfo: DataInfo<B1> | undefined,
+  mapFn: (blob: B1) => B2,
+): DataInfo<B2> | undefined {
+  if (dataInfo === undefined) {
+    return undefined;
+  }
+
+  switch (dataInfo.type) {
+    case 'Json':
+      // Json type doesn't contain blobs, so return as is
+      return dataInfo;
+    case 'JsonPartitioned': {
+      // Map each blob in parts
+      const newParts: Record<string, B2> = {};
+      for (const [key, blob] of Object.entries(dataInfo.parts)) {
+        newParts[key] = mapFn(blob);
+      }
+      return {
+        ...dataInfo,
+        parts: newParts,
+      };
+    }
+    case 'BinaryPartitioned': {
+      // Map each index and values blob in parts
+      const newParts: Record<string, BinaryChunk<B2>> = {};
+      for (const [key, chunk] of Object.entries(dataInfo.parts)) {
+        newParts[key] = {
+          index: mapFn(chunk.index),
+          values: mapFn(chunk.values),
+        };
+      }
+      return {
+        ...dataInfo,
+        parts: newParts,
+      };
+    }
+  }
+}
+
+//
+// Lightway representation for ExplicitJsonData
+//
+
+/**
+ * Represents a single key-value entry in a column's explicit data structure.
+ * Used when directly instantiating PColumns with explicit data.
+ */
+export type PColumnValuesEntry = {
+  key: PColumnKey;
+  val: PColumnValue;
+};
+
+/**
+ * Array of key-value entries representing explicit column data.
+ * Used for lightweight explicit instantiation of PColumns.
+ */
+export type PColumnValues = PColumnValuesEntry[];
+
+/**
+ * Entry-based representation of JsonDataInfo
+ */
+export interface JsonDataInfoEntries {
+  type: 'Json';
+  keyLength: number;
+  data: PColumnDataEntry<PColumnValue>[];
+}
+
+/**
+ * Entry-based representation of JsonPartitionedDataInfo
+ */
+export interface JsonPartitionedDataInfoEntries<Blob> {
+  type: 'JsonPartitioned';
+  partitionKeyLength: number;
+  parts: PColumnDataEntry<Blob>[];
+}
+
+/**
+ * Entry-based representation of BinaryPartitionedDataInfo
+ */
+export interface BinaryPartitionedDataInfoEntries<Blob> {
+  type: 'BinaryPartitioned';
+  partitionKeyLength: number;
+  parts: PColumnDataEntry<BinaryChunk<Blob>>[];
+}
+
+/**
+ * Union type representing all possible entry-based data storage formats
+ */
+export type DataInfoEntries<Blob> =
+  | JsonDataInfoEntries
+  | JsonPartitionedDataInfoEntries<Blob>
+  | BinaryPartitionedDataInfoEntries<Blob>;
+
+/**
+ * Type guard function that checks if the given value is a valid DataInfoEntries.
+ *
+ * @param value - The value to check
+ * @returns True if the value is a valid DataInfoEntries, false otherwise
+ */
+export function isDataInfoEntries<Blob>(value: unknown): value is DataInfoEntries<Blob> {
+  if (!value || typeof value !== 'object') {
+    return false;
+  }
+
+  const data = value as Record<string, unknown>;
+  if (!('type' in data)) {
+    return false;
+  }
+
+  switch (data.type) {
+    case 'Json':
+      return (
+        typeof data.keyLength === 'number'
+        && Array.isArray(data.data)
+      );
+    case 'JsonPartitioned':
+      return (
+        typeof data.partitionKeyLength === 'number'
+        && Array.isArray(data.parts)
+      );
+    case 'BinaryPartitioned':
+      return (
+        typeof data.partitionKeyLength === 'number'
+        && Array.isArray(data.parts)
+      );
+    default:
+      return false;
+  }
+}
+
+/**
+ * Converts DataInfo to DataInfoEntries
+ *
+ * @param dataInfo - The record-based DataInfo object
+ * @returns The equivalent entry-based DataInfoEntries object
+ */
+export function dataInfoToEntries<Blob>(dataInfo: DataInfo<Blob>): DataInfoEntries<Blob> {
+  switch (dataInfo.type) {
+    case 'Json': {
+      const entries: PColumnDataEntry<PColumnValue>[] = Object.entries(dataInfo.data).map(([keyStr, value]) => {
+        const key = JSON.parse(keyStr) as PColumnKey;
+        return { key, value };
+      });
+
+      return {
+        type: 'Json',
+        keyLength: dataInfo.keyLength,
+        data: entries,
+      };
+    }
+    case 'JsonPartitioned': {
+      const parts: PColumnDataEntry<Blob>[] = Object.entries(dataInfo.parts).map(([keyStr, blob]) => {
+        const key = JSON.parse(keyStr) as PColumnKey;
+        return { key, value: blob };
+      });
+
+      return {
+        type: 'JsonPartitioned',
+        partitionKeyLength: dataInfo.partitionKeyLength,
+        parts,
+      };
+    }
+    case 'BinaryPartitioned': {
+      const parts: PColumnDataEntry<BinaryChunk<Blob>>[] = Object.entries(dataInfo.parts).map(([keyStr, chunk]) => {
+        const key = JSON.parse(keyStr) as PColumnKey;
+        return { key, value: chunk };
+      });
+
+      return {
+        type: 'BinaryPartitioned',
+        partitionKeyLength: dataInfo.partitionKeyLength,
+        parts,
+      };
+    }
+  }
+}
+
+/**
+ * Converts DataInfoEntries to DataInfo
+ *
+ * @param dataInfoEntries - The entry-based DataInfoEntries object
+ * @returns The equivalent record-based DataInfo object
+ */
+export function entriesToDataInfo<Blob>(dataInfoEntries: DataInfoEntries<Blob>): DataInfo<Blob> {
+  switch (dataInfoEntries.type) {
+    case 'Json': {
+      const data: Record<string, PColumnValue> = {};
+      for (const entry of dataInfoEntries.data) {
+        data[JSON.stringify(entry.key)] = entry.value;
+      }
+
+      return {
+        type: 'Json',
+        keyLength: dataInfoEntries.keyLength,
+        data,
+      };
+    }
+    case 'JsonPartitioned': {
+      const parts: Record<string, Blob> = {};
+      for (const entry of dataInfoEntries.parts) {
+        parts[JSON.stringify(entry.key)] = entry.value;
+      }
+
+      return {
+        type: 'JsonPartitioned',
+        partitionKeyLength: dataInfoEntries.partitionKeyLength,
+        parts,
+      };
+    }
+    case 'BinaryPartitioned': {
+      const parts: Record<string, BinaryChunk<Blob>> = {};
+      for (const entry of dataInfoEntries.parts) {
+        parts[JSON.stringify(entry.key)] = entry.value;
+      }
+
+      return {
+        type: 'BinaryPartitioned',
+        partitionKeyLength: dataInfoEntries.partitionKeyLength,
+        parts,
+      };
+    }
+  }
+}
+
+/**
+ * Maps blob references in a DataInfoEntries object from one type to another using a mapping function.
+ *
+ * @template B1 - Source blob type
+ * @template B2 - Target blob type
+ * @param dataInfoEntries - The source DataInfoEntries object
+ * @param mapFn - Function to transform blobs from type B1 to type B2
+ * @returns A new DataInfoEntries object with transformed blob references
+ */
+export function mapDataInfoEntries<B1, B2>(
+  dataInfoEntries: DataInfoEntries<B1>,
+  mapFn: (blob: B1) => B2,
+): DataInfoEntries<B2>;
+export function mapDataInfoEntries<B1, B2>(
+  dataInfoEntries: DataInfoEntries<B1> | undefined,
+  mapFn: (blob: B1) => B2,
+): DataInfoEntries<B2> | undefined {
+  if (dataInfoEntries === undefined) {
+    return undefined;
+  }
+
+  switch (dataInfoEntries.type) {
+    case 'Json':
+      // Json type doesn't contain blobs, so return as is
+      return dataInfoEntries;
+    case 'JsonPartitioned': {
+      // Map each blob in parts
+      const newParts = dataInfoEntries.parts.map((entry) => ({
+        key: entry.key,
+        value: mapFn(entry.value),
+      }));
+
+      return {
+        ...dataInfoEntries,
+        parts: newParts,
+      };
+    }
+    case 'BinaryPartitioned': {
+      // Map each index and values blob in parts
+      const newParts = dataInfoEntries.parts.map((entry) => ({
+        key: entry.key,
+        value: {
+          index: mapFn(entry.value.index),
+          values: mapFn(entry.value.values),
+        },
+      }));
+
+      return {
+        ...dataInfoEntries,
+        parts: newParts,
+      };
+    }
+  }
+}

package/src/drivers/pframe/{data.ts → data_types.ts}

@@ -242,13 +242,6 @@ export function isValueAbsent(absent: Uint8Array, index: number): boolean {
   return (absent[chunkIndex] & mask) > 0;
 }
 
-export type PColumnValue = null | number | string;
-export type PColumnValuesEntry = {
-  key: PColumnValue[];
-  val: PColumnValue;
-};
-export type PColumnValues = PColumnValuesEntry[];
-
 export const PTableAbsent = { type: 'absent' };
 export type PTableAbsent = typeof PTableAbsent;
 export const PTableNA = null;

package/src/drivers/pframe/driver.ts

@@ -2,7 +2,7 @@ import type { Branded } from '../../branding';
 import type { PTable } from './table';
 import type { PFrame } from './pframe';
 import type { AddParameterToAllMethods } from './type_util';
-import type { PTableShape, PTableVector, TableRange } from './data';
+import type { PTableShape, PTableVector, TableRange } from './data_types';
 import type { FindColumnsRequest, FindColumnsResponse } from './find_columns';
 import type { PObjectId } from '../../pool';
 import type { PColumnIdAndSpec, PColumnSpec } from './spec/spec';

package/src/drivers/pframe/index.ts

@@ -1,5 +1,6 @@
 export * from './column_filter';
-export * from './data';
+export * from './data_info';
+export * from './data_types';
 export * from './find_columns';
 export * from './pframe';
 export * from './spec/spec';

package/src/drivers/pframe/spec/{anchored_id.ts → anchored.ts}

@@ -1,8 +1,11 @@
 import canonicalize from 'canonicalize';
 import type { AxisId, PColumnSpec } from './spec';
 import { getAxisId, matchAxisId } from './spec';
-import type { AAxisSelector, AnchorAxisRef, AnchorAxisRefByIdx, APColumnId, APColumnSelector, AxisSelector, PColumnSelector } from './selectors';
-import type { Branded } from '../../../branding';
+import type { AAxisSelector, AnchorAxisRef, AnchorAxisRefByIdx, AnchoredPColumnId, AnchoredPColumnSelector, AxisSelector, PColumnSelector } from './selectors';
+import type { AxisFilter } from './filtered_column';
+import type { PValue } from '../data_types';
+import type { SUniversalPColumnId, UniversalPColumnId } from './ids';
+import { stringifyColumnId } from './ids';
 
 //
 // Helper functions
@@ -16,18 +19,11 @@ function domainKey(key: string, value: string): string {
   return JSON.stringify([key, value]);
 }
 
-//
-// Branded types
-//
-
-/** Canonicalized string representation of an anchored column identifier, either anchored or absolute. */
-export type CanonicalPColumnId = Branded<string, 'CanonicalPColumnId'>;
-
 /**
  * Context for resolving and generating anchored references to columns and axes
  * Maintains maps of known domain values and axes that can be referenced by anchors
  */
-export class AnchorIdDeriver {
+export class AnchoredIdDeriver {
   private readonly domains = new Map<string, string>();
   private readonly axes = new Map<string, AnchorAxisRefByIdx>();
   /**
@@ -70,12 +66,24 @@ export class AnchorIdDeriver {
 
   /**
    * Derives an anchored column identifier from a column specification
-   * Replaces domain values and axes with anchored references when possible
    * @param spec Column specification to anchor
    * @returns An anchored column identifier that can be used to identify columns similar to the input specification
    */
-  derive(spec: PColumnSpec): APColumnId {
-    const result: APColumnId = {
+  derive(spec: PColumnSpec): AnchoredPColumnId;
+
+  /**
+   * Derives an anchored column identifier from a column specification
+   * @param spec Column specification to anchor
+   * @param axisFilters Axis filters to apply to the column
+   * @returns An anchored and sliced column identifier that can be used to identify columns similar to the input specification
+   */
+  derive(spec: PColumnSpec, axisFilters?: AxisFilter[]): UniversalPColumnId;
+
+  /**
+   * Implementation of derive method
+   */
+  derive(spec: PColumnSpec, axisFilters?: AxisFilter[]): UniversalPColumnId {
+    const result: AnchoredPColumnId = {
       name: spec.name,
       axes: [],
     };
@@ -116,28 +124,62 @@ export class AnchorIdDeriver {
       return anchorAxisRef ?? axis;
     });
 
-    return result;
+    // If no axis filters are provided, return the anchored ID as is
+    if (!axisFilters || axisFilters.length === 0) {
+      return result;
+    }
+
+    // Process axis filters and create a sliced column ID
+    const resolvedFilters: [number, PValue][] = [];
+
+    for (const filter of axisFilters) {
+      const [axisIdOrIndex, value] = filter;
+
+      // If it's already a numeric index, validate it
+      if (typeof axisIdOrIndex === 'number') {
+        if (axisIdOrIndex < 0 || axisIdOrIndex >= spec.axesSpec.length) {
+          throw new Error(`Axis index ${axisIdOrIndex} is out of bounds (0-${spec.axesSpec.length - 1})`);
+        }
+        resolvedFilters.push([axisIdOrIndex, value]);
+      } else {
+        // If it's a string (axis name), resolve it to an index
+        const axisIndex = spec.axesSpec.findIndex((axis) => axis.name === axisIdOrIndex);
+        if (axisIndex === -1) {
+          throw new Error(`Axis with name "${axisIdOrIndex}" not found in the column specification`);
+        }
+        resolvedFilters.push([axisIndex, value]);
+      }
+    }
+
+    // Sort filters by axis index to ensure consistency
+    resolvedFilters.sort((a, b) => a[0] - b[0]);
+
+    return {
+      source: result,
+      axisFilters: resolvedFilters,
+    };
   }
 
   /**
    * Derives a canonicalized string representation of an anchored column identifier, can be used as a unique identifier for the column
    * @param spec Column specification to anchor
+   * @param axisFilters Optional axis filters to apply to the column
    * @returns A canonicalized string representation of the anchored column identifier
    */
-  deriveCanonical(spec: PColumnSpec): CanonicalPColumnId {
-    const aId = this.derive(spec);
-    return canonicalize(aId)! as CanonicalPColumnId;
+  deriveS(spec: PColumnSpec, axisFilters?: AxisFilter[]): SUniversalPColumnId {
+    return stringifyColumnId(this.derive(spec, axisFilters));
   }
 }
 
 /**
- * Resolves anchored references in a column matcher to create a non-anchored matcher
+ * Resolves anchored references in a column matcher to create a non-anchored matcher.
+ * Doing an opposite operation to {@link AnchorIdDeriver.derive()}.
  *
- * @param anchors - Record of anchor column specifications indexed by anchor ID
- * @param matcher - An anchored column matcher containing references that need to be resolved
+ * @param anchors - Record of anchor column specifications indexed by anchor id
+ * @param matcher - An anchored column matcher (or id, which is subtype of it) containing references that need to be resolved
  * @returns A non-anchored column matcher with all references resolved to actual values
  */
-export function resolveAnchors(anchors: Record<string, PColumnSpec>, matcher: APColumnSelector): PColumnSelector {
+export function resolveAnchors(anchors: Record<string, PColumnSpec>, matcher: AnchoredPColumnSelector): PColumnSelector {
   const result = { ...matcher };
 
   if (result.domainAnchor !== undefined) {

package/src/drivers/pframe/spec/filtered_column.ts

@@ -0,0 +1,39 @@
+import type { PValue } from '../data_types';
+import type { AnchoredPColumnId } from './selectors';
+
+/** Axis filter by index */
+export type AxisFilterByIdx = [number, PValue];
+
+/** Axis filter by name */
+export type AxisFilterByName = [string, PValue];
+
+/** Axis filter */
+export type AxisFilter = AxisFilterByIdx | AxisFilterByName;
+
+/**
+ * Identifies a column derived from a CanonicalPColumnId by fixing values on certain axes,
+ * thus effectively reducing the dimensionality of the original column.
+ */
+export type FilteredPColumn<CID = AnchoredPColumnId, AFI = AxisFilter> = {
+  /** The original column identifier */
+  source: CID;
+
+  /**
+   * List of fixed axes and their corresponding values.
+   * Each entry fixes one axis to a specific value, creating a slice of the multidimensional data.
+   * This effectively reduces the dimensionality by one for each fixed axis.
+   * Ordered by the axis index in canonical case.
+   */
+  axisFilters: AFI[];
+};
+
+export type FilteredPColumnId = FilteredPColumn<AnchoredPColumnId, AxisFilterByIdx>;
+
+/**
+ * Checks if a given value is a FilteredPColumn
+ * @param id - The value to check
+ * @returns True if the value is a FilteredPColumn, false otherwise
+ */
+export function isFilteredPColumn(id: unknown): id is FilteredPColumn {
+  return typeof id === 'object' && id !== null && 'source' in id && 'axisFilters' in id;
+}

package/src/drivers/pframe/spec/ids.ts

@@ -0,0 +1,31 @@
+import type { Branded } from '../../../branding';
+import type { AnchoredPColumnId } from './selectors';
+import type { FilteredPColumnId } from './filtered_column';
+import canonicalize from 'canonicalize';
+/**
+ * Universal column identifier optionally anchored and optionally filtered.
+ */
+export type UniversalPColumnId = AnchoredPColumnId | FilteredPColumnId;
+
+/**
+ * Canonically serialized {@link UniversalPColumnId}.
+ */
+export type SUniversalPColumnId = Branded<string, 'SUniversalPColumnId'>;
+
+/**
+ * Canonically serializes a {@link UniversalPColumnId} to a string.
+ * @param id - The column identifier to serialize
+ * @returns The canonically serialized string
+ */
+export function stringifyColumnId(id: UniversalPColumnId): SUniversalPColumnId {
+  return canonicalize(id)! as SUniversalPColumnId;
+}
+
+/**
+ * Parses a canonically serialized {@link UniversalPColumnId} from a string.
+ * @param str - The string to parse
+ * @returns The parsed column identifier
+ */
+export function parseColumnId(str: SUniversalPColumnId): UniversalPColumnId {
+  return JSON.parse(str) as UniversalPColumnId;
+}

package/src/drivers/pframe/spec/index.ts

@@ -1,3 +1,5 @@
-export * from './anchored_id';
+export * from './anchored';
+export * from './ids';
+export * from './filtered_column';
 export * from './spec';
 export * from './selectors';