@platforma-sdk/model 1.69.0 → 1.71.0

package/src/columns/column_snapshot_provider.ts

@@ -1,11 +1,9 @@
 import type { PObjectId } from "@milaboratories/pl-model-common";
-import { PColumn } from "@milaboratories/pl-model-common";
+import { isDataInfo, PColumn, visitDataInfo } from "@milaboratories/pl-model-common";
 import { TreeNodeAccessor } from "../render/accessor";
 import type { PColumnDataUniversal } from "../render/internal";
 import type { ColumnDataStatus, ColumnSnapshot } from "./column_snapshot";
 
-// --- ColumnProvider ---
-
 /**
  * Data source interface for column enumeration.
  *
@@ -22,8 +20,6 @@ export interface ColumnSnapshotProvider {
   isColumnListComplete(): boolean;
 }
 
-// --- ColumnSource ---
-
 /**
  * Union of types that can serve as column sources for helpers and builders.
  * Does NOT include TreeNodeAccessor — call `.toColumnSource()` on it first.
@@ -33,8 +29,6 @@ export type ColumnSource =
   | ColumnSnapshot<PObjectId>[]
   | PColumn<PColumnDataUniversal | undefined>[];
 
-// --- ArrayColumnProvider ---
-
 /**
  * Simple provider wrapping an array of PColumns.
  * Always complete, data status always 'ready'.
@@ -46,8 +40,8 @@ export class ArrayColumnProvider implements ColumnSnapshotProvider {
     this.columns = columns.map((col) => ({
       id: col.id,
       spec: col.spec,
-      dataStatus: "ready" as const,
       data: { get: () => col.data },
+      dataStatus: this.getStatus(col.data),
     }));
   }
 
@@ -58,9 +52,35 @@ export class ArrayColumnProvider implements ColumnSnapshotProvider {
   isColumnListComplete(): boolean {
     return true;
   }
-}
 
-// --- SnapshotColumnProvider ---
+  protected getStatus(
+    d: undefined | PColumnDataUniversal | (() => undefined | PColumnDataUniversal),
+  ): ColumnDataStatus {
+    if (d == null) {
+      return "absent";
+    }
+    if (typeof d === "function") {
+      return this.getStatus(d());
+    }
+    if (d instanceof TreeNodeAccessor) {
+      if (d.getIsReadyOrError()) return "ready";
+      if (d.getIsFinal()) return "absent";
+      return "computing";
+    }
+    if (isDataInfo(d)) {
+      let ready = true;
+      let final = true;
+      visitDataInfo(d, (v) => {
+        ready &&= v.getIsReadyOrError();
+        final &&= v.getIsFinal();
+      });
+      if (ready) return "ready";
+      if (final) return "absent";
+      return "computing";
+    }
+    return "ready";
+  }
+}
 
 /**
  * Provider wrapping an array of ColumnSnapshots.
@@ -78,8 +98,6 @@ export class SnapshotColumnProvider implements ColumnSnapshotProvider {
   }
 }
 
-// --- OutputColumnProvider ---
-
 export interface OutputColumnProviderOpts {
   /** When true and the accessor is final, columns with no ready data get status 'absent'. */
   allowPermanentAbsence?: boolean;
@@ -133,8 +151,6 @@ export class OutputColumnProvider implements ColumnSnapshotProvider {
   }
 }
 
-// --- Source normalization ---
-
 /** Checks if a value is a ColumnSnapshotProvider (duck-typing). */
 export function isColumnSnapshotProvider(source: unknown): source is ColumnSnapshotProvider {
   return (

package/src/columns/ctx_column_sources.ts

@@ -9,32 +9,26 @@ import { ResourceTypeName } from "@milaboratories/pl-model-common";
 import type { ValueOf } from "@milaboratories/helpers";
 
 /**
- * Collect ColumnSnapshotProviders from all render context sources:
- *
- * - **resultPool** — all upstream columns (always included)
- * - **outputs** — PFrame fields from block execution outputs
- * - **prerun** — PFrame fields from prerun/staging results
- *
- * Returns an array of providers suitable for `ColumnCollectionBuilder.addSource()`.
+ * Collect ColumnSnapshotProviders from `outputs`, `prerun`, and
+ * `resultPool` in that order. Dedup keeps the first occurrence per
+ * `NativePObjectId`, so a block re-publishing its own columns keeps
+ * the `outputs`-rooted canonical id instead of the result-pool variant.
  */
 export function collectCtxColumnSnapshotProviders(ctx: RenderCtxBase): ColumnSnapshotProvider[] {
   const providers: ColumnSnapshotProvider[] = [];
 
-  // ResultPool — all upstream columns
-  providers.push(new ResultPoolColumnSnapshotProvider(ctx.resultPool));
-
-  // Outputs — each PFrame-like output field becomes a provider
   const outputs = ctx.outputs;
   if (outputs) {
     providers.push(...collectPFrameProviders(outputs));
   }
 
-  // Prerun — same treatment as outputs
   const prerun = ctx.prerun;
   if (prerun) {
     providers.push(...collectPFrameProviders(prerun));
   }
 
+  providers.push(new ResultPoolColumnSnapshotProvider(ctx.resultPool));
+
   return providers;
 }
 

package/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts

@@ -32,6 +32,7 @@ import {
   withLabelAnnotations,
   withTableVisualAnnotations,
   withInfoAnnotations,
+  withDataStatusAnnotations,
 } from "./utils";
 import type { PrimaryEntry, SecondaryGroup } from "./createPTableDefV3";
 import { createPTableDefV3 } from "./createPTableDefV3";
@@ -79,8 +80,6 @@ export type ColumnVisibilityRule = {
 
 export type ColumnMatcher = (spec: PColumnSpec) => boolean;
 
-// Main Function
-
 export function createPlDataTableV3<A, U>(
   ctx: RenderCtxBase<A, U>,
   options: createPlDataTableOptionsV3,
@@ -138,17 +137,18 @@ export function createPlDataTableV3<A, U>(
   ]);
 
   const remapedDefaultFilters = remapFilterColumnIds(options.filters, discovered);
-  const filters = concatFilters(
-    state.pTableParams.filters,
-    state.pTableParams.defaultFilters ?? remapedDefaultFilters,
+  const filters = filterFilters(
+    concatFilters(
+      state.pTableParams.filters,
+      state.pTableParams.defaultFilters ?? remapedDefaultFilters,
+    ),
+    columnIsAvailable,
   );
-  validateFilters(filters, columnIsAvailable);
 
-  const sorting = resolveSorting(
-    state.pTableParams.sorting,
-    remapSortingColumnIds(options.sorting, discovered),
+  const sorting = filterSorting(
+    resolveSorting(state.pTableParams.sorting, remapSortingColumnIds(options.sorting, discovered)),
+    columnIsAvailable,
   );
-  validateSorting(sorting, columnIsAvailable);
 
   const primaryEntries: PrimaryEntry<undefined | PColumnDataUniversal>[] = primarySnapshots.map(
     (v) => ({ column: resolveSnapshot(v.column) }),
@@ -274,6 +274,7 @@ function annotateColumnGroups(params: {
   const directAnnotated = liftToVariantColumns(
     direct,
     flow(
+      (cols) => withDataStatusAnnotations(cols),
       (cols) => withLabelAnnotations(derivedLabels, cols),
       (cols) => withInfoAnnotations(derivedTooltips, cols),
       (cols) => withTableVisualAnnotations(visibilityByColId, orderByColId, cols),
@@ -283,6 +284,7 @@ function annotateColumnGroups(params: {
   const linkedAnnotated = liftToVariantColumns(
     linked,
     flow(
+      (cols) => withDataStatusAnnotations(cols),
       (cols) => withHidenAxesAnnotations(cols),
       (cols) => withLabelAnnotations(derivedLabels, cols),
       (cols) => withInfoAnnotations(derivedTooltips, cols),
@@ -344,19 +346,35 @@ function createColumnValidationById(
   };
 }
 
-/** Validate that all column references in filters exist in the table. */
-function validateFilters(
+/** Drop filter leaves whose column references are not available in the table. */
+function filterFilters(
   filters: Nil | PlDataTableFilters,
   isValidColumnId: (id: string) => boolean,
-): void {
-  if (filters == null) return;
-  const filterColumns = collectFilterSpecColumns(filters);
-  const firstInvalid = filterColumns.find((col) => !isValidColumnId(col));
-  if (firstInvalid !== undefined) {
-    throw new Error(
-      `Invalid filter column ${firstInvalid}: column reference does not match the table columns`,
-    );
-  }
+): Nil | PlDataTableFilters {
+  if (isNil(filters)) return filters;
+
+  const isLeafValid = (leaf: PlDataTableFilterSpecLeaf): boolean => {
+    if (leaf.type === undefined) return true;
+    if ("column" in leaf && !isValidColumnId(leaf.column)) return false;
+    if ("rhs" in leaf && !isValidColumnId(leaf.rhs)) return false;
+    return true;
+  };
+
+  const prune = (node: PlDataTableFilterNode): Nil | PlDataTableFilterNode => {
+    if (node.type === "and" || node.type === "or") {
+      const kept = node.filters
+        .map((f) => prune(f))
+        .filter((f): f is PlDataTableFilterNode => !isNil(f));
+      return { type: node.type, filters: kept };
+    }
+    if (node.type === "not") {
+      const inner = prune(node.filter);
+      return isNil(inner) ? undefined : { type: "not", filter: inner };
+    }
+    return isLeafValid(node) ? node : undefined;
+  };
+
+  return prune(filters) as Nil | PlDataTableFilters;
 }
 
 /** Merge two filter trees into one AND-combined tree. Returns the non-nil one if the other is nil. */
@@ -377,16 +395,12 @@ function resolveSorting(
   return (isEmpty(userSorting) ? defaultSorting : userSorting) ?? [];
 }
 
-/** Validate that all column references in sorting exist in the table. */
-function validateSorting(sorting: PTableSorting[], isValidColumnId: (id: string) => boolean): void {
-  const firstInvalid = sorting.find(
-    (s) => !isValidColumnId(canonicalizeJson<PTableColumnId>(s.column)),
-  );
-  if (firstInvalid !== undefined) {
-    throw new Error(
-      `Invalid sorting column ${JSON.stringify(firstInvalid.column)}: column reference does not match the table columns`,
-    );
-  }
+/** Drop sorting entries whose column is not available in the table. */
+function filterSorting(
+  sorting: PTableSorting[],
+  isValidColumnId: (id: string) => boolean,
+): PTableSorting[] {
+  return sorting.filter((s) => isValidColumnId(canonicalizeJson<PTableColumnId>(s.column)));
 }
 
 function buildSecondaryGroups(
@@ -405,7 +419,6 @@ function buildSecondaryGroups(
         entries: [
           ...lc.path.map((s) => ({
             column: resolveSnapshot(s.linker),
-            qualifications: s.qualifications,
           })),
           { column: resolveSnapshot(lc.column), qualifications: lc.qualifications.forHit },
         ],
@@ -474,21 +487,22 @@ function remapSortingColumnIds(
   sorting: Nil | PTableSorting[],
   columns: TableColumnVariant[],
 ): Nil | PTableSorting[] {
-  return sorting?.map((s) => {
-    if (s.column.type === "axis") return s; // Axis references are unaffected by column ID remapping
+  return sorting?.flatMap((s) => {
+    if (s.column.type === "axis") return [s]; // Axis references are unaffected by column ID remapping
 
     const id = s.column.id;
-    const column =
-      columns.find((c) => (c.originalId ?? c.column.id) === id) ??
-      throwError(`Column ID "${id}" in sorting does not match any discovered column`);
-
-    return {
-      ...s,
-      column: {
-        type: "column",
-        id: column.column.id,
+    const column = columns.find((c) => (c.originalId ?? c.column.id) === id);
+    if (column === undefined) return [];
+
+    return [
+      {
+        ...s,
+        column: {
+          type: "column" as const,
+          id: column.column.id,
+        },
       },
-    };
+    ];
   });
 }
 

package/src/components/PlDataTable/createPlDataTable/discoverColumns.ts

@@ -102,7 +102,6 @@ function mapToTableColumnVariants(
       path: variant.path.map((p) => ({
         type: "linker",
         column: p.linker.id,
-        qualifications: p.qualifications,
       })),
       columnQualifications: variant.qualifications.forHit,
       queriesQualifications: variant.qualifications.forQueries,

package/src/components/PlDataTable/createPlDataTable/utils.ts

@@ -17,7 +17,7 @@ import {
   deriveDistinctTooltips,
   type TooltipEntry,
 } from "../../../labels/derive_distinct_tooltips";
-import type { MatchQualifications, MatchVariant } from "../../../columns";
+import type { ColumnDataStatus, MatchQualifications, MatchVariant } from "../../../columns";
 import type { ColumnMatcher, ColumnOrderRule, ColumnVisibilityRule } from "./createPlDataTableV3";
 import type { ColumnSelector } from "../../../columns";
 import { ArrayColumnProvider, ColumnCollectionBuilder } from "../../../columns";
@@ -130,7 +130,10 @@ function dedupeById(columns: RuleColumn[]): RuleColumn[] {
  * For each axis in column specs: writes derived axis label into AxisSpec annotations.
  */
 export function withLabelAnnotations<
-  T extends { readonly id: PObjectId; readonly spec: PColumnSpec },
+  T extends {
+    readonly id: PObjectId;
+    readonly spec: PColumnSpec;
+  },
 >(derivedLabels: undefined | Record<string, string>, columns: T[]): T[] {
   if (derivedLabels === undefined) return columns;
   return columns.map((col) => {
@@ -153,6 +156,26 @@ export function withLabelAnnotations<
   });
 }
 
+export function withDataStatusAnnotations<
+  T extends {
+    readonly spec: PColumnSpec;
+    readonly dataStatus: ColumnDataStatus;
+  },
+>(columns: T[]): T[] {
+  return columns.map((col) => {
+    return {
+      ...col,
+      spec: {
+        ...col.spec,
+        annotations: {
+          ...col.spec.annotations,
+          [Annotation.DataStatus]: col.dataStatus,
+        },
+      },
+    } as T;
+  });
+}
+
 /**
  * Writes effective display properties (OrderPriority, Visibility) from precomputed rule maps
  * into column annotations. Returns new column objects — originals are not mutated.
@@ -241,7 +264,6 @@ export function deriveAllLabels(options: {
     spec: c.spec,
     linkerPath: c.linkerPath?.map((step) => ({
       spec: step.linker.spec,
-      qualifications: step.qualifications,
     })),
     qualifications: c.qualifications,
   }));

package/src/components/PlDatasetSelector/build_dataset_options.ts

@@ -1,20 +1,25 @@
-import type {
-  DatasetOption,
-  Option,
-  PColumnSelector,
-  PObjectSpec,
-} from "@milaboratories/pl-model-common";
+import type { MultiColumnSelector, Option, PObjectSpec } from "@milaboratories/pl-model-common";
+import { multiColumnSelectorsToPredicate } from "@milaboratories/pl-model-common";
 import type { DeriveLabelsOptions } from "../../labels/derive_distinct_labels";
 import type { RenderCtxBase } from "../../render";
 import { ColumnCollectionBuilder } from "../../columns/column_collection_builder";
 import { collectCtxColumnSnapshotProviders } from "../../columns/ctx_column_sources";
+import type { DatasetOption } from "./dataset_selection";
 import { buildRefMap, filterMatchesToOptions, findFilterColumns } from "./filter_discovery";
+import { enrichmentVariantsToRefs, findEnrichmentColumns } from "./enrichment_discovery";
 
 export type BuildDatasetOptions = {
   /** Which result pool columns qualify as datasets. Defaults to all. */
-  selector?: PColumnSelector | PColumnSelector[] | ((spec: PObjectSpec) => boolean);
+  primary?: MultiColumnSelector | MultiColumnSelector[] | ((spec: PObjectSpec) => boolean);
   /** Formatting options for filter labels. */
   labelOptions?: DeriveLabelsOptions;
+  /**
+   * Enables enrichment discovery and filters hits attached to
+   * `DatasetOption.enrichments`. Use `() => true` to accept all; omit to disable.
+   */
+  withEnrichments?: MultiColumnSelector | MultiColumnSelector[] | ((spec: PObjectSpec) => boolean);
+  /** Maximum linker hops considered. Only used when `withEnrichments` is set. */
+  enrichmentMaxHops?: number;
 };
 
 /**
@@ -27,28 +32,54 @@ export function buildDatasetOptions(
   ctx: RenderCtxBase,
   opts?: BuildDatasetOptions,
 ): DatasetOption[] | undefined {
-  const predicate = opts?.selector ?? (() => true);
-  const options = ctx.resultPool.getOptions(predicate, { refsWithEnrichments: true });
+  const primary = opts?.primary;
+  const primaryPredicate =
+    primary === undefined
+      ? () => true
+      : typeof primary === "function"
+        ? primary
+        : multiColumnSelectorsToPredicate(primary);
+  const options = ctx.resultPool.getOptions(primaryPredicate, { refsWithEnrichments: true });
   if (options.length === 0) return [];
 
   const columnSources = collectCtxColumnSnapshotProviders(ctx);
   const refMap = buildRefMap(ctx.resultPool.getSpecs().entries);
   const pframeSpec = ctx.getService("pframeSpec");
 
-  return options.map((o: Option): DatasetOption => {
-    const datasetSpec = ctx.resultPool.getPColumnSpecByRef(o.ref);
-    if (!datasetSpec) return o;
+  return options.map((primary: Option): DatasetOption => {
+    const datasetSpec = ctx.resultPool.getPColumnSpecByRef(primary.ref);
+    if (!datasetSpec) return { primary };
 
     const builder = new ColumnCollectionBuilder(pframeSpec);
     for (const src of columnSources) builder.addSource(src);
     const collection = builder.build({ anchors: { main: datasetSpec } });
-    if (!collection) return o;
+    if (!collection) return { primary };
 
     try {
-      const matches = findFilterColumns(collection);
-      if (matches.length === 0) return o;
-      const filters = filterMatchesToOptions(matches, refMap, opts?.labelOptions);
-      return { ...o, filters };
+      const filterMatches = findFilterColumns(collection);
+      const filters =
+        filterMatches.length === 0
+          ? undefined
+          : filterMatchesToOptions(filterMatches, refMap, opts?.labelOptions);
+
+      let enrichments;
+      if (opts?.withEnrichments !== undefined) {
+        const enrichmentVariants = findEnrichmentColumns(collection, {
+          maxHops: opts.enrichmentMaxHops,
+          ...(typeof opts.withEnrichments === "function"
+            ? { predicate: opts.withEnrichments }
+            : { include: opts.withEnrichments }),
+        });
+        if (enrichmentVariants.length > 0) {
+          enrichments = enrichmentVariantsToRefs(enrichmentVariants, opts.labelOptions);
+        }
+      }
+
+      return {
+        primary,
+        ...(filters !== undefined && filters.length > 0 ? { filters } : {}),
+        ...(enrichments !== undefined && enrichments.length > 0 ? { enrichments } : {}),
+      };
     } finally {
       collection.dispose();
     }

package/src/components/PlDatasetSelector/dataset_selection.ts

@@ -0,0 +1,37 @@
+import type { LabeledEnrichmentRefs, Option, PrimaryRef } from "@milaboratories/pl-model-common";
+
+/** Dataset picker entry: user picks {@link primary}, gets {@link enrichments} attached. */
+export type DatasetOption = {
+  readonly primary: Option;
+  readonly filters?: readonly Option[];
+  readonly enrichments?: LabeledEnrichmentRefs;
+};
+
+/**
+ * Picked dataset bundle emitted by `PlDatasetSelector`. Stored opaquely in
+ * block data; block authors unbundle inside their args resolver.
+ */
+export type DatasetSelection = {
+  readonly __isDatasetSelection: "v1";
+  readonly primary: PrimaryRef;
+  readonly enrichments?: LabeledEnrichmentRefs;
+};
+
+export function isDatasetSelection(value: unknown): value is DatasetSelection {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    (value as { __isDatasetSelection?: unknown }).__isDatasetSelection === "v1" &&
+    "primary" in value
+  );
+}
+
+export function createDatasetSelection(
+  primary: PrimaryRef,
+  enrichments?: LabeledEnrichmentRefs,
+): DatasetSelection {
+  if (enrichments !== undefined && enrichments.length > 0) {
+    return { __isDatasetSelection: "v1", primary, enrichments };
+  }
+  return { __isDatasetSelection: "v1", primary };
+}

package/src/components/PlDatasetSelector/enrichment_discovery.ts

@@ -0,0 +1,111 @@
+import { Annotation, createEnrichmentRef } from "@milaboratories/pl-model-common";
+import type {
+  EnrichmentStep,
+  LabeledEnrichmentRef,
+  LabeledEnrichmentRefs,
+  MultiColumnSelector,
+  PObjectId,
+  PObjectSpec,
+} from "@milaboratories/pl-model-common";
+import type {
+  AnchoredColumnCollection,
+  ColumnVariant,
+} from "../../columns/column_collection_builder";
+import {
+  deriveDistinctLabels,
+  type DeriveLabelsOptions,
+  type Entry,
+} from "../../labels/derive_distinct_labels";
+
+/**
+ * True for global-form ids — `canonicalize({__isRef: true, blockId, name})` —
+ * which the workflow can resolve via bquery. Local-form ids (`resolvePath`)
+ * fail this check and are excluded from auto-discovery; prerun/outputs hops
+ * must be supplied as resolved `{spec, data}` instead.
+ */
+function isGloballyAddressable(id: PObjectId): boolean {
+  try {
+    const decoded = JSON.parse(id);
+    return (
+      typeof decoded === "object" &&
+      decoded !== null &&
+      decoded.__isRef === true &&
+      typeof decoded.blockId === "string" &&
+      typeof decoded.name === "string"
+    );
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Linker-reached hits attached to the anchor primary. Drops zero-hop variants
+ * (filters / the primary itself) and structural hits (subset, linker, label
+ * columns). Narrow further with `include` selectors or a `predicate`.
+ */
+export function findEnrichmentColumns(
+  collection: AnchoredColumnCollection,
+  options?: {
+    maxHops?: number;
+    include?: MultiColumnSelector | MultiColumnSelector[];
+    predicate?: (spec: PObjectSpec) => boolean;
+  },
+): ColumnVariant[] {
+  const include =
+    options?.include === undefined
+      ? undefined
+      : Array.isArray(options.include)
+        ? options.include
+        : [options.include];
+  const variants = collection.findColumnVariants({
+    mode: "enrichment",
+    maxHops: options?.maxHops ?? 4,
+    include,
+    exclude: [
+      { annotations: { [Annotation.IsSubset]: "true" } },
+      { annotations: { [Annotation.IsLinkerColumn]: "true" } },
+      { name: Annotation.Label },
+    ],
+  });
+  const predicate = options?.predicate;
+  return variants.filter((v) => {
+    if (v.path.length === 0) return false;
+    if (predicate !== undefined && !predicate(v.column.spec)) return false;
+    if (!isGloballyAddressable(v.column.id)) return false;
+    if (v.path.some((p) => !isGloballyAddressable(p.linker.id))) return false;
+    return true;
+  });
+}
+
+/**
+ * Pair each variant with a path-disambiguated label (so export headers stay
+ * unique) and carry hit/linker `PObjectId`s through verbatim. Propagates
+ * `qualifications.forHit`; `forQueries` is re-derived by the table builder.
+ */
+export function enrichmentVariantsToRefs(
+  variants: ColumnVariant[],
+  labelOptions?: DeriveLabelsOptions,
+): LabeledEnrichmentRefs {
+  if (variants.length === 0) return [];
+
+  const entries: Entry[] = variants.map((variant) => ({
+    spec: variant.column.spec,
+    linkerPath: variant.path.map((p) => ({ spec: p.linker.spec })),
+    qualifications: variant.qualifications,
+  }));
+  const labels = deriveDistinctLabels(entries, labelOptions);
+
+  return variants.map((variant, i): LabeledEnrichmentRef => {
+    const path: EnrichmentStep[] = variant.path.map((step) => ({
+      type: "linker",
+      linker: step.linker.id,
+    }));
+    return {
+      ref: createEnrichmentRef(variant.column.id, {
+        path,
+        qualifications: variant.qualifications.forHit,
+      }),
+      label: labels[i],
+    };
+  });
+}

package/src/components/PlDatasetSelector/index.ts

@@ -1,2 +1,3 @@
+export * from "./dataset_selection";
 export * from "./filter_discovery";
 export * from "./build_dataset_options";