@platforma-sdk/model 1.59.0 → 1.60.0

package/src/columns/column_snapshot.ts

@@ -0,0 +1,55 @@
+import type { PColumnSpec, PObjectId } from "@milaboratories/pl-model-common";
+import type { PColumnDataUniversal } from "../render/internal";
+
+// --- ColumnSnapshot ---
+
+/** Data status of a column snapshot. */
+export type ColumnDataStatus = "ready" | "computing" | "absent";
+
+/**
+ * Immutable snapshot of a column: spec, data status, and lazy data accessor.
+ *
+ * - `dataStatus` is readable without marking the render context unstable.
+ * - `data` holds an active object when data exists (ready or computing),
+ *   or `undefined` when data is permanently absent.
+ */
+export interface ColumnSnapshot<Id extends PObjectId = PObjectId> {
+  readonly id: Id;
+  readonly spec: PColumnSpec;
+  readonly dataStatus: ColumnDataStatus;
+
+  /**
+   * Lazy data accessor.
+   * - `'ready'`: `data.get()` returns column data, context stays stable.
+   * - `'computing'`: `data.get()` returns `undefined`, marks context unstable.
+   * - `'absent'`: `data` is `undefined` — no active object, no instability.
+   */
+  readonly data: ColumnData | undefined;
+}
+
+// --- ColumnData ---
+
+/**
+ * Active object wrapping lazy column data access.
+ * Accessing data on a computing column marks the render context unstable.
+ */
+export interface ColumnData {
+  get(): PColumnDataUniversal | undefined;
+}
+
+/** Creates a ColumnData active object for a ready column. */
+export function createReadyColumnData(getData: () => PColumnDataUniversal | undefined): ColumnData {
+  return { get: getData };
+}
+
+// --- Snapshot construction helpers ---
+
+/** Creates a ColumnSnapshot from parts. */
+export function createColumnSnapshot<Id extends PObjectId = PObjectId>(
+  id: Id,
+  spec: PColumnSpec,
+  dataStatus: ColumnDataStatus,
+  data: ColumnData | undefined,
+): ColumnSnapshot<Id> {
+  return { id, spec, dataStatus, data };
+}

package/src/columns/column_snapshot_provider.ts

@@ -0,0 +1,177 @@
+import { PColumn } from "@milaboratories/pl-model-common";
+import { TreeNodeAccessor } from "../render/accessor";
+import type { PColumnDataUniversal } from "../render/internal";
+import type { ColumnDataStatus, ColumnSnapshot } from "./column_snapshot";
+
+// --- ColumnProvider ---
+
+/**
+ * Data source interface for column enumeration.
+ *
+ * Knows nothing about the render framework, stability tracking, labels,
+ * anchoring, or splitting. All that complexity lives in the collection layer.
+ */
+export interface ColumnSnapshotProvider {
+  /** Returns all currently known columns. */
+  getAllColumns(): ColumnSnapshot[];
+
+  /** Whether the provider has finished enumerating all its columns.
+   *  Calling this may mark the render context unstable — it touches
+   *  the reactive tree to check field resolution state. */
+  isColumnListComplete(): boolean;
+}
+
+// --- ColumnSource ---
+
+/**
+ * Union of types that can serve as column sources for helpers and builders.
+ * Does NOT include TreeNodeAccessor — call `.toColumnSource()` on it first.
+ */
+export type ColumnSource =
+  | ColumnSnapshotProvider
+  | ColumnSnapshot[]
+  | PColumn<PColumnDataUniversal | undefined>[];
+
+// --- ArrayColumnProvider ---
+
+/**
+ * Simple provider wrapping an array of PColumns.
+ * Always complete, data status always 'ready'.
+ */
+export class ArrayColumnProvider implements ColumnSnapshotProvider {
+  private readonly columns: ColumnSnapshot[];
+
+  constructor(columns: PColumn<PColumnDataUniversal | undefined>[]) {
+    this.columns = columns.map((col) => ({
+      id: col.id,
+      spec: col.spec,
+      dataStatus: "ready" as const,
+      data: { get: () => col.data },
+    }));
+  }
+
+  getAllColumns(): ColumnSnapshot[] {
+    return this.columns;
+  }
+
+  isColumnListComplete(): boolean {
+    return true;
+  }
+}
+
+// --- SnapshotColumnProvider ---
+
+/**
+ * Provider wrapping an array of ColumnSnapshots.
+ * Always complete. Data status taken from each snapshot.
+ */
+export class SnapshotColumnProvider implements ColumnSnapshotProvider {
+  constructor(private readonly snapshots: ColumnSnapshot[]) {}
+
+  getAllColumns(): ColumnSnapshot[] {
+    return this.snapshots;
+  }
+
+  isColumnListComplete(): boolean {
+    return true;
+  }
+}
+
+// --- OutputColumnProvider ---
+
+export interface OutputColumnProviderOpts {
+  /** When true and the accessor is final, columns with no ready data get status 'absent'. */
+  allowPermanentAbsence?: boolean;
+}
+
+/**
+ * Provider wrapping a TreeNodeAccessor (output/prerun resolve result).
+ * Detects data status from accessor readiness state.
+ */
+export class OutputColumnProvider implements ColumnSnapshotProvider {
+  constructor(
+    private readonly accessor: TreeNodeAccessor,
+    private readonly opts?: OutputColumnProviderOpts,
+  ) {}
+
+  getAllColumns(): ColumnSnapshot[] {
+    return this.getColumns();
+  }
+
+  isColumnListComplete(): boolean {
+    return this.accessor.getInputsLocked();
+  }
+
+  private getColumns(): ColumnSnapshot[] {
+    const pColumns = this.accessor.getPColumns();
+    if (pColumns === undefined) return [];
+
+    const isFinal = this.accessor.getIsFinal();
+    const allowAbsence = this.opts?.allowPermanentAbsence === true;
+
+    return pColumns.map((col) => {
+      const dataAccessor = col.data;
+      const isReady = dataAccessor.getIsReadyOrError();
+
+      let dataStatus: ColumnDataStatus;
+      if (isReady) {
+        dataStatus = "ready";
+      } else if (allowAbsence && isFinal) {
+        dataStatus = "absent";
+      } else {
+        dataStatus = "computing";
+      }
+
+      return {
+        id: col.id,
+        spec: col.spec,
+        dataStatus,
+        data: { get: () => (isReady ? dataAccessor : undefined) },
+      };
+    });
+  }
+}
+
+// --- Source normalization ---
+
+/** Checks if a value is a ColumnSnapshotProvider (duck-typing). */
+export function isColumnSnapshotProvider(source: unknown): source is ColumnSnapshotProvider {
+  return (
+    typeof source === "object" &&
+    source !== null &&
+    "getAllColumns" in source &&
+    "isColumnListComplete" in source &&
+    typeof (source as ColumnSnapshotProvider).getAllColumns === "function" &&
+    typeof (source as ColumnSnapshotProvider).isColumnListComplete === "function"
+  );
+}
+
+/** Checks if a value looks like a PColumn (has id, spec, data). */
+function isPColumnArray(source: unknown): source is PColumn<PColumnDataUniversal | undefined>[] {
+  if (!Array.isArray(source)) return false;
+  if (source.length === 0) return true; // empty array — treat as PColumn[]
+  const first = source[0];
+  return "id" in first && "spec" in first && "data" in first && !("dataStatus" in first);
+}
+
+/** Checks if a value looks like a ColumnSnapshot array. */
+function isColumnSnapshotArray(source: unknown): source is ColumnSnapshot[] {
+  if (!Array.isArray(source)) return false;
+  if (source.length === 0) return true; // empty array — treat as snapshots
+  const first = source[0];
+  return "id" in first && "spec" in first && "dataStatus" in first;
+}
+
+/**
+ * Normalize any ColumnSource into a ColumnSnapshotProvider.
+ * - ColumnSnapshotProvider → returned as-is
+ * - ColumnSnapshot[] → wrapped in SnapshotColumnProvider
+ * - PColumn[] → wrapped in ArrayColumnProvider
+ */
+export function toColumnSnapshotProvider(source: ColumnSource): ColumnSnapshotProvider {
+  if (isColumnSnapshotProvider(source)) return source;
+  if (isColumnSnapshotArray(source)) return new SnapshotColumnProvider(source);
+  if (isPColumnArray(source)) return new ArrayColumnProvider(source);
+  // Should not reach here given the type union, but be safe
+  throw new Error("Unknown ColumnSource type");
+}

package/src/columns/ctx_column_sources.ts

@@ -0,0 +1,107 @@
+import type { PColumnSpec, PObjectId } from "@milaboratories/pl-model-common";
+import { TreeNodeAccessor } from "../render/accessor";
+import type { RenderCtxBase, ResultPool } from "../render";
+import type { ColumnSnapshot } from "./column_snapshot";
+import type { ColumnDataStatus } from "./column_snapshot";
+import type { ColumnSnapshotProvider } from "./column_snapshot_provider";
+import { OutputColumnProvider } from "./column_snapshot_provider";
+import { ResourceTypeName } from "@milaboratories/pl-model-common";
+import type { ValueOf } from "@milaboratories/helpers";
+
+/**
+ * Collect ColumnSnapshotProviders from all render context sources:
+ *
+ * - **resultPool** — all upstream columns (always included)
+ * - **outputs** — PFrame fields from block execution outputs
+ * - **prerun** — PFrame fields from prerun/staging results
+ *
+ * Returns an array of providers suitable for `ColumnCollectionBuilder.addSource()`.
+ */
+export function collectCtxColumnSnapshotProviders<A, U>(
+  ctx: RenderCtxBase<A, U>,
+): ColumnSnapshotProvider[] {
+  const providers: ColumnSnapshotProvider[] = [];
+
+  // ResultPool — all upstream columns
+  providers.push(new ResultPoolColumnSnapshotProvider(ctx.resultPool));
+
+  // Outputs — each PFrame-like output field becomes a provider
+  const outputs = ctx.outputs;
+  if (outputs) {
+    providers.push(...collectPFrameProviders(outputs));
+  }
+
+  // Prerun — same treatment as outputs
+  const prerun = ctx.prerun;
+  if (prerun) {
+    providers.push(...collectPFrameProviders(prerun));
+  }
+
+  return providers;
+}
+
+/**
+ * Adapter wrapping ResultPool into the new ColumnSnapshotProvider interface.
+ *
+ * - `isColumnListComplete()` always returns true — the result pool
+ *   is a stable snapshot within a single render cycle.
+ * - Data status is derived from the underlying TreeNodeAccessor:
+ *   ready (getIsReadyOrError), computing, or absent (no data resource).
+ */
+export class ResultPoolColumnSnapshotProvider implements ColumnSnapshotProvider {
+  constructor(private readonly pool: ResultPool) {}
+
+  getAllColumns(): ColumnSnapshot[] {
+    const pColumns = this.pool.selectColumns(() => true);
+    return pColumns.map((col) => toSnapshot(col.id, col.spec, col.data));
+  }
+
+  isColumnListComplete(): boolean {
+    return true;
+  }
+}
+
+function toSnapshot(
+  id: PObjectId,
+  spec: PColumnSpec,
+  accessor: TreeNodeAccessor | undefined,
+): ColumnSnapshot {
+  if (accessor === undefined) {
+    return { id, spec, dataStatus: "absent" as ColumnDataStatus, data: undefined };
+  }
+  const isReady = accessor.getIsReadyOrError();
+  return {
+    id,
+    spec,
+    dataStatus: (isReady ? "ready" : "computing") as ColumnDataStatus,
+    data: { get: () => (isReady ? accessor : undefined) },
+  };
+}
+
+/**
+ * Recursively walk the output tree starting from `accessor`.
+ * - If a node's resourceType is PFrame → wrap it as OutputColumnProvider.
+ * - If a node's resourceType is StdMap/std/map → recurse into its output fields.
+ * - Otherwise → skip (leaf of unknown type).
+ */
+function collectPFrameProviders(accessor: TreeNodeAccessor): ColumnSnapshotProvider[] {
+  const out: ColumnSnapshotProvider[] = [];
+  walkTree(accessor, out);
+  return out;
+}
+
+function walkTree(node: TreeNodeAccessor, out: ColumnSnapshotProvider[]): void {
+  const typeName = node.resourceType.name as ValueOf<typeof ResourceTypeName>;
+
+  if (typeName === ResourceTypeName.PFrame) {
+    out.push(new OutputColumnProvider(node));
+    return;
+  }
+
+  if (typeName === ResourceTypeName.StdMap || typeName === ResourceTypeName.StdMapSlash) {
+    for (const field of node.listInputFields()) {
+      const child = node.resolve(field);
+      if (child) walkTree(child, out);
+    }
+  }
+}

package/src/columns/expand_by_partition.test.ts

@@ -0,0 +1,289 @@
+import type {
+  AxisSpec,
+  JsonPartitionedDataInfoEntries,
+  PColumnSpec,
+  PObjectId,
+} from "@milaboratories/pl-model-common";
+import { canonicalizeAxisId, getAxisId } from "@milaboratories/pl-model-common";
+import { describe, expect, test } from "vitest";
+import type { PColumnDataUniversal } from "../render/internal";
+import type { ColumnSnapshot } from "./column_snapshot";
+import { expandByPartition } from "./expand_by_partition";
+
+// --- Helpers ---
+
+function createAxis(name: string, type = "String"): AxisSpec {
+  return { name, type } as AxisSpec;
+}
+
+function createSpec(name: string, axes: AxisSpec[]): PColumnSpec {
+  return {
+    kind: "PColumn",
+    name,
+    valueType: "Int",
+    axesSpec: axes,
+    annotations: {},
+  } as PColumnSpec;
+}
+
+/** Create a ready snapshot whose data.get() returns a JsonPartitioned DataInfoEntries. */
+function createReadySnapshot(
+  id: string,
+  columnSpec: PColumnSpec,
+  partitionKeyLength: number,
+  parts: { key: (string | number)[]; value: unknown }[],
+): ColumnSnapshot {
+  const dataEntries: JsonPartitionedDataInfoEntries<unknown> = {
+    type: "JsonPartitioned",
+    partitionKeyLength,
+    parts,
+  };
+  return {
+    id: id as PObjectId,
+    spec: columnSpec,
+    dataStatus: "ready",
+    // convertOrParsePColumnData checks isDataInfoEntries first (duck-type),
+    // so this works at runtime despite the PColumnDataUniversal type
+    data: { get: () => dataEntries as unknown as PColumnDataUniversal },
+  };
+}
+
+function createComputingSnapshot(id: string, columnSpec: PColumnSpec): ColumnSnapshot {
+  return {
+    id: id as PObjectId,
+    spec: columnSpec,
+    dataStatus: "computing",
+    data: {
+      get: () => undefined,
+    },
+  };
+}
+
+function createAbsentSnapshot(id: string, columnSpec: PColumnSpec): ColumnSnapshot {
+  return {
+    id: id as PObjectId,
+    spec: columnSpec,
+    dataStatus: "absent",
+    data: undefined,
+  };
+}
+
+interface Trace {
+  type: string;
+  label: string;
+  importance: number;
+}
+
+function extractTrace(snapshot: ColumnSnapshot): Trace[] {
+  const raw = snapshot.spec.annotations?.["pl7.app/trace"];
+  return raw ? (JSON.parse(raw) as Trace[]) : [];
+}
+
+// --- Tests ---
+
+describe("expandByPartition", () => {
+  test("no split axes returns snapshots as-is", () => {
+    const s = createReadySnapshot("col1", createSpec("c", [createAxis("a")]), 1, [
+      { key: ["x"], value: {} },
+    ]);
+    const result = expandByPartition([s], []);
+    expect(result.complete).toBe(true);
+    expect(result.items).toHaveLength(1);
+    expect(result.items[0]).toBe(s); // same reference
+  });
+
+  test("single axis split produces K snapshots", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("sample"), createAxis("gene")]),
+      2,
+      [
+        { key: ["s1", "g1"], value: {} },
+        { key: ["s1", "g2"], value: {} },
+        { key: ["s2", "g1"], value: {} },
+      ],
+    );
+
+    const result = expandByPartition([s], [{ idx: 0 }]);
+
+    expect(result.complete).toBe(true);
+    // unique values on axis 0: s1, s2 → 2 snapshots
+    expect(result.items).toHaveLength(2);
+
+    // split axis removed from axesSpec
+    for (const snap of result.items) {
+      expect(snap.spec.axesSpec).toHaveLength(1);
+      expect(snap.spec.axesSpec[0].name).toBe("gene");
+    }
+  });
+
+  test("multi-axis split produces K1 x K2 snapshots", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("a"), createAxis("b"), createAxis("value")]),
+      2,
+      [
+        { key: ["a1", "b1"], value: {} },
+        { key: ["a1", "b2"], value: {} },
+        { key: ["a2", "b1"], value: {} },
+      ],
+    );
+
+    const result = expandByPartition([s], [{ idx: 0 }, { idx: 1 }]);
+
+    expect(result.complete).toBe(true);
+    // a: a1, a2 (2) × b: b1, b2 (2) = 4
+    expect(result.items).toHaveLength(4);
+
+    // both split axes removed, only "value" remains
+    for (const snap of result.items) {
+      expect(snap.spec.axesSpec).toHaveLength(1);
+      expect(snap.spec.axesSpec[0].name).toBe("value");
+    }
+  });
+
+  test("trace annotations include split info", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("sample"), createAxis("gene")]),
+      1,
+      [
+        { key: ["s1"], value: {} },
+        { key: ["s2"], value: {} },
+      ],
+    );
+
+    const result = expandByPartition([s], [{ idx: 0 }]);
+    expect(result.complete).toBe(true);
+    expect(result.items).toHaveLength(2);
+
+    const trace0 = extractTrace(result.items[0]);
+    expect(trace0).toEqual([
+      {
+        type: `split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`,
+        label: "s1",
+        importance: 1_000_000,
+      },
+    ]);
+
+    const trace1 = extractTrace(result.items[1]);
+    expect(trace1).toEqual([
+      {
+        type: `split:${canonicalizeAxisId(getAxisId(createAxis("sample")))}`,
+        label: "s2",
+        importance: 1_000_000,
+      },
+    ]);
+  });
+
+  test("axisLabels option resolves labels", () => {
+    const sampleAxis = createAxis("sample");
+    const s = createReadySnapshot("col1", createSpec("c", [sampleAxis, createAxis("gene")]), 1, [
+      { key: ["s1"], value: {} },
+      { key: ["s2"], value: {} },
+    ]);
+
+    const labels: Record<string | number, string> = {
+      s1: "Sample One",
+      s2: "Sample Two",
+    };
+
+    const result = expandByPartition([s], [{ idx: 0 }], {
+      axisLabels: () => labels,
+    });
+
+    expect(result.complete).toBe(true);
+    const trace0 = extractTrace(result.items[0]);
+    expect(trace0[0].label).toBe("Sample One");
+    const trace1 = extractTrace(result.items[1]);
+    expect(trace1[0].label).toBe("Sample Two");
+  });
+
+  test("computing snapshot returns incomplete", () => {
+    const s = createComputingSnapshot("col1", createSpec("c", [createAxis("a")]));
+    const result = expandByPartition([s], [{ idx: 0 }]);
+    expect(result.complete).toBe(false);
+    expect(result.items).toHaveLength(0);
+  });
+
+  test("absent snapshot returns incomplete", () => {
+    const s = createAbsentSnapshot("col1", createSpec("c", [createAxis("a")]));
+    const result = expandByPartition([s], [{ idx: 0 }]);
+    expect(result.complete).toBe(false);
+    expect(result.items).toHaveLength(0);
+  });
+
+  test("empty unique keys for an axis produces no snapshots for that column", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("a"), createAxis("b")]),
+      2,
+      [], // no parts → no unique keys
+    );
+
+    const result = expandByPartition([s], [{ idx: 0 }]);
+    expect(result.complete).toBe(true);
+    expect(result.items).toHaveLength(0);
+  });
+
+  test("filtered data is accessible on expanded snapshots", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("sample"), createAxis("gene")]),
+      1,
+      [
+        { key: ["s1"], value: { payload: "data-s1" } },
+        { key: ["s2"], value: { payload: "data-s2" } },
+      ],
+    );
+
+    const result = expandByPartition([s], [{ idx: 0 }]);
+    expect(result.complete).toBe(true);
+    expect(result.items).toHaveLength(2);
+
+    // Each expanded snapshot should have accessible data
+    for (const snap of result.items) {
+      expect(snap.data).toBeDefined();
+      const data = snap.data!.get();
+      expect(data).toBeDefined();
+    }
+  });
+
+  test("multiple input snapshots are all expanded", () => {
+    const s1 = createReadySnapshot(
+      "col1",
+      createSpec("c1", [createAxis("sample"), createAxis("gene")]),
+      1,
+      [
+        { key: ["s1"], value: {} },
+        { key: ["s2"], value: {} },
+      ],
+    );
+    const s2 = createReadySnapshot(
+      "col2",
+      createSpec("c2", [createAxis("sample"), createAxis("gene")]),
+      1,
+      [
+        { key: ["x1"], value: {} },
+        { key: ["x2"], value: {} },
+        { key: ["x3"], value: {} },
+      ],
+    );
+
+    const result = expandByPartition([s1, s2], [{ idx: 0 }]);
+    expect(result.complete).toBe(true);
+    // s1: 2 unique + s2: 3 unique = 5 total
+    expect(result.items).toHaveLength(5);
+  });
+
+  test("throws when split axis exceeds partition key length", () => {
+    const s = createReadySnapshot(
+      "col1",
+      createSpec("c", [createAxis("a"), createAxis("b"), createAxis("c")]),
+      1, // only 1 partition key
+      [{ key: ["x"], value: {} }],
+    );
+
+    expect(() => expandByPartition([s], [{ idx: 1 }])).toThrow(/Not enough partition keys/);
+  });
+});