@platforma-sdk/model 1.79.27 → 1.80.2

package/src/render/api.ts

@@ -6,16 +6,15 @@ import type {
   ModelServices,
   Option,
   PColumn,
-  PColumnLazy,
   PColumnSelector,
   PColumnSpec,
   PColumnValues,
   PFrameDef,
   PFrameHandle,
+  ColumnUniversalId,
   PObject,
   PObjectId,
   PObjectSpec,
-  PSpecPredicate,
   PTableDef,
   PTableDefV2,
   PTableHandle,
@@ -45,6 +44,7 @@ import {
   withEnrichments,
   legacyColumnSelectorsToPredicate,
   extractAllColumns,
+  visitDataInfo,
 } from "@milaboratories/pl-model-common";
 import canonicalize from "canonicalize";
 import type { Optional } from "utility-types";
@@ -72,6 +72,8 @@ import { patchInSetFilters } from "./util/pframe_upgraders";
 import type { PColumnDataUniversal } from "./internal";
 import { getService } from "../services";
 import { allPColumnsReady } from "./util";
+import { throwError } from "@milaboratories/helpers";
+import { isString } from "es-toolkit";
 
 /**
  * Helper function to match domain objects
@@ -92,19 +94,25 @@ export type UniversalColumnOption = { label: string; value: SUniversalPColumnId
 
 /**
  * Transforms PColumn data into the internal representation expected by the platform
- * @param data Data from a PColumn to transform
+ * @param column Data from a PColumn to transform
  * @returns Transformed data compatible with platform API
  */
-function transformPColumnData(
-  data: PColumn<undefined | PColumnDataUniversal> | PColumnLazy<undefined | PColumnDataUniversal>,
+export function finalizePColumnData(
+  column: PColumn<PColumnDataUniversal<undefined | TreeNodeAccessor>>,
 ): PColumn<PColumnValues | AccessorHandle | DataInfo<AccessorHandle>> {
-  return mapPObjectData(data, (d) => {
-    if (d instanceof TreeNodeAccessor) {
-      return d.handle;
-    } else if (isDataInfo(d)) {
-      return mapDataInfo(d, (accessor) => accessor.handle);
+  return mapPObjectData(column, (data) => {
+    if (data instanceof TreeNodeAccessor) {
+      return data.handle;
+    } else if (isDataInfo(data)) {
+      let ready = true;
+      visitDataInfo(data, (accessor) => (ready &&= accessor?.getIsReadyOrError() ?? false));
+      if (!ready) return [];
+      return mapDataInfo(
+        data,
+        (accessor) => accessor?.handle ?? throwError("Accessor handle is undefined"),
+      );
     } else {
-      return d ?? [];
+      return data ?? [];
     }
   });
 }
@@ -130,16 +138,22 @@ type GetOptionsOpts = {
   label?: ((spec: PObjectSpec, ref: PlRef) => string) | LabelDerivationOps;
 };
 
+/**
+ * @deprecated use the direct `ctx.*` methods on `RenderCtxBase` instead.
+ *
+ * Migration map:
+ * - `getSpecs` → `ctx.getSpecs`
+ * - `getSpecByRef` / `getPColumnSpecByRef` → `ctx.getSpecByRef` / `ctx.getPColumnSpecByRef`
+ * - `getDataByRef` / `getPColumnByRef` → `ctx.getDataByRef` / `ctx.getPColumnByRef`
+ * - `getOptions` → `ctx.getOptions`
+ *
+ * Eager methods `selectColumns` / `getData` / `getDataWithErrors` should be
+ * replaced by `ctx.getSpecs` + on-demand `ctx.getStatusByRef` /
+ * `ctx.getPColumnByRef` so that the data resource is not subscribed up-front.
+ */
 export class ResultPool implements ColumnProvider, AxisLabelProvider {
   private readonly ctx: GlobalCfgRenderCtx = getCfgRenderCtx();
 
-  /**
-   * @deprecated use getOptions()
-   */
-  public calculateOptions(predicate: PSpecPredicate): Option[] {
-    return this.ctx.calculateOptions(predicate);
-  }
-
   public getOptions(
     predicateOrSelector: ((spec: PObjectSpec) => boolean) | PColumnSelector | PColumnSelector[],
     opts?: GetOptionsOpts,
@@ -221,7 +235,7 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider {
       | APColumnSelectorWithSplit
       | APColumnSelectorWithSplit[],
     opts?: UniversalPColumnOpts,
-  ): PColumn<PColumnDataUniversal>[] | undefined {
+  ): undefined | PColumn<undefined | PColumnDataUniversal>[] {
     const anchorCtx = this.resolveAnchorCtx(anchorsOrCtx);
     if (!anchorCtx) return undefined;
     return new PColumnCollection()
@@ -351,11 +365,6 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider {
    * @returns data associated with the ref
    */
   public getDataByRef(ref: PlRef): PObject<TreeNodeAccessor> | undefined {
-    // @TODO remove after 1 Jan 2025; forward compatibility
-    if (typeof this.ctx.getDataFromResultPoolByRef === "undefined")
-      return this.getData().entries.find(
-        (f) => f.ref.blockId === ref.blockId && f.ref.name === ref.name,
-      )?.obj;
     const data = this.ctx.getDataFromResultPoolByRef(ref.blockId, ref.name); // Keep original call
     // Need to handle undefined case before mapping
     if (!data) return undefined;
@@ -491,6 +500,9 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider {
    *
    * @param selectors - A predicate function, a single selector, or an array of selectors.
    * @returns An array of PColumn objects matching the selectors. Data is loaded on first access.
+   * @deprecated use `RenderCtxBase.rawResultPool` + `ResultPoolColumnsProvider` +
+   * `discoverColumns` instead. This eager path materializes column data
+   * unnecessarily for callers that only filter by spec.
    */
   public selectColumns(
     selectors: ((spec: PColumnSpec) => boolean) | PColumnSelector | PColumnSelector[],
@@ -552,7 +564,7 @@ export class ResultPool implements ColumnProvider, AxisLabelProvider {
 
 /** Main entry point to the API available within model lambdas (like outputs, sections, etc..) */
 export abstract class RenderCtxBase<Args = unknown, Data = unknown> {
-  protected readonly ctx: GlobalCfgRenderCtx;
+  public readonly ctx: GlobalCfgRenderCtx;
 
   constructor() {
     this.ctx = getCfgRenderCtx();
@@ -611,30 +623,9 @@ export abstract class RenderCtxBase<Args = unknown, Data = unknown> {
     return this.getNamedAccessor(MainAccessorName);
   }
 
+  /** @deprecated use `rawResultPool` + `ResultPoolColumnsProvider` instead. */
   public readonly resultPool = new ResultPool();
 
-  /**
-   * Find labels data for a given axis id. It will search for a label column and return its data as a map.
-   * @returns a map of axis value => label
-   * @deprecated Use resultPool.findLabels instead
-   */
-  public findLabels(axis: AxisId): Record<string | number, string> | undefined {
-    return this.resultPool.findLabels(axis);
-  }
-
-  private verifyInlineAndExplicitColumnsSupport(
-    columns: (PColumn<PColumnDataUniversal> | PColumnLazy<undefined | PColumnDataUniversal>)[],
-  ) {
-    const hasInlineColumns = columns.some(
-      (c) => !(c.data instanceof TreeNodeAccessor) || isDataInfo(c.data),
-    ); // Updated check for DataInfo
-    const inlineColumnsSupport = this.ctx.featureFlags?.inlineColumnsSupport === true;
-    if (hasInlineColumns && !inlineColumnsSupport)
-      throw Error(`Inline or explicit columns not supported`); // Combined check
-
-    // Removed redundant explicitColumns check
-  }
-
   private patchPTableDef(
     def: PTableDef<PColumn<PColumnDataUniversal>>,
   ): PTableDef<PColumn<PColumnDataUniversal>> {
@@ -656,13 +647,10 @@ export abstract class RenderCtxBase<Args = unknown, Data = unknown> {
     return def;
   }
 
-  // TODO remove all non-PColumn fields
   public createPFrame(
-    def: PFrameDef<
-      PColumn<undefined | PColumnDataUniversal> | PColumnLazy<undefined | PColumnDataUniversal>
-    >,
+    def: PFrameDef<PObjectId | SUniversalPColumnId | PColumn<undefined | PColumnDataUniversal>>,
   ): PFrameHandle | undefined {
-    return this.ctx.createPFrame(def.map((c) => transformPColumnData(c)));
+    return this.ctx.createPFrame(def.map((v) => (isString(v) ? v : finalizePColumnData(v))));
   }
 
   // TODO remove all non-PColumn fields
@@ -698,20 +686,16 @@ export abstract class RenderCtxBase<Args = unknown, Data = unknown> {
       rawDef = this.patchPTableDef(def);
     }
     const columns = extractAllColumns(rawDef.src);
-    this.verifyInlineAndExplicitColumnsSupport(columns);
     if (!allPColumnsReady(columns)) return undefined;
-    return this.ctx.createPTable(mapPTableDef(rawDef, (po) => transformPColumnData(po)));
+    return this.ctx.createPTable(mapPTableDef(rawDef, finalizePColumnData));
   }
 
   public createPTableV2(
-    def: PTableDefV2<PColumn<undefined | PColumnDataUniversal>>,
+    def: PTableDefV2<ColumnUniversalId | PColumn<undefined | PColumnDataUniversal>>,
   ): PTableHandle | undefined {
-    return this.ctx.createPTableV2(mapPTableDefV2(def, (po) => transformPColumnData(po)));
-  }
-
-  /** @deprecated scheduled for removal from SDK */
-  public getBlockLabel(blockId: string): string {
-    return this.ctx.getBlockLabel(blockId);
+    return this.ctx.createPTableV2(
+      mapPTableDefV2(def, { column: (v) => (isString(v) ? v : finalizePColumnData(v)) }),
+    );
   }
 
   public getCurrentUnstableMarker(): string | undefined {

package/src/render/index.ts

@@ -3,5 +3,6 @@ export * from "./traversal_ops";
 export * from "./accessor";
 export * from "./util";
 export { FutureRef } from "./future";
-export type { PColumnDataUniversal } from "./internal";
+export type { AccessorHandle, PColumnDataUniversal } from "./internal";
+export { MainAccessorName, StagingAccessorName } from "./internal";
 export type { ExtractFutureRefType } from "./future";

package/src/render/internal.ts

@@ -1,17 +1,22 @@
 import type { Optional } from "utility-types";
-import type { Branded, StringifiedJson } from "@milaboratories/pl-model-common";
+import type {
+  AccessorHandle,
+  Branded,
+  PObjectId,
+  StringifiedJson,
+  SUniversalPColumnId,
+} from "@milaboratories/pl-model-common";
+export type { AccessorHandle };
 import type { CommonFieldTraverseOps, FieldTraversalStep, ResourceType } from "./traversal_ops";
 import type {
   ArchiveFormat,
   AnyFunction,
-  Option,
   PColumn,
   PColumnValues,
   PFrameDef,
   PFrameHandle,
   PObject,
   PObjectSpec,
-  PSpecPredicate,
   PTableDef,
   PTableDefV2,
   PTableHandle,
@@ -23,10 +28,9 @@ import type {
 import type { TreeNodeAccessor } from "./accessor";
 import type { ServiceDispatch } from "@milaboratories/pl-model-common";
 
-export const StagingAccessorName = "staging";
 export const MainAccessorName = "main";
+export const StagingAccessorName = "staging";
 
-export type AccessorHandle = Branded<string, "AccessorHandle">;
 export type FutureHandle = Branded<string, "FutureHandle">;
 
 export type PColumnDataUniversal<TreeEntry = TreeNodeAccessor> =
@@ -34,6 +38,10 @@ export type PColumnDataUniversal<TreeEntry = TreeNodeAccessor> =
   | DataInfo<TreeEntry>
   | PColumnValues;
 
+// Raw upstream-block ctx shape lives in `@milaboratories/pl-model-common`
+// (`UpstreamBlockCtx`). Import directly from there at use sites.
+import type { UpstreamBlockCtx } from "@milaboratories/pl-model-common";
+
 export interface GlobalCfgRenderCtxMethods<AHandle = AccessorHandle, FHandle = FutureHandle> {
   //
   // Root accessor creation
@@ -73,11 +81,13 @@ export interface GlobalCfgRenderCtxMethods<AHandle = AccessorHandle, FHandle = F
 
   getKeyValueAsString(handle: AHandle, key: string): string | undefined;
 
+  hasData(handle: AHandle): boolean;
+
   getDataBase64(handle: AHandle): string | undefined;
 
   getDataAsString(handle: AHandle): string | undefined;
 
-  /** If not final returns undefined */
+  /** @deprecated */
   parsePObjectCollection(
     handle: AHandle,
     errorOnUnknownField: boolean,
@@ -144,18 +154,40 @@ export interface GlobalCfgRenderCtxMethods<AHandle = AccessorHandle, FHandle = F
 
   getDataFromResultPoolByRef(blockId: string, exportName: string): PObject<AHandle> | undefined;
 
-  calculateOptions(predicate: PSpecPredicate): Option[];
+  //
+  // Raw result pool — list of upstream block ctx accessor handles.
+  // SDK-side providers compose enumerate/status/data themselves on top of these.
+  //
+
+  /**
+   * For each upstream block in the staging graph, returns its prod/staging ctx
+   * accessor handles (when present). Sandbox uses these handles with
+   * `getPObjectEntryDataHandle` to build
+   * column snapshots; spec/status/data merge with prod-vs-staging precedence
+   * lives sandbox-side.
+   */
+  getUpstreamBlockCtx(): ReadonlyArray<UpstreamBlockCtx<AHandle>>;
 
   //
   // PFrame / PTable
   //
 
-  createPFrame(def: PFrameDef<PColumn<AHandle | PColumnValues | DataInfo<AHandle>>>): PFrameHandle;
+  createPFrame(
+    def: PFrameDef<
+      PObjectId | SUniversalPColumnId | PColumn<AHandle | PColumnValues | DataInfo<AHandle>>
+    >,
+  ): PFrameHandle;
 
-  createPTable(def: PTableDef<PColumn<AHandle | PColumnValues | DataInfo<AHandle>>>): PTableHandle;
+  createPTable(
+    def: PTableDef<
+      PObjectId | SUniversalPColumnId | PColumn<AHandle | PColumnValues | DataInfo<AHandle>>
+    >,
+  ): PTableHandle;
 
   createPTableV2(
-    def: PTableDefV2<PColumn<AHandle | PColumnValues | DataInfo<AHandle>>>,
+    def: PTableDefV2<
+      PObjectId | SUniversalPColumnId | PColumn<AHandle | PColumnValues | DataInfo<AHandle>>
+    >,
   ): PTableHandle;
 
   //
@@ -181,6 +213,7 @@ export const GlobalCfgRenderCtxFeatureFlags = {
   activeArgs: true as const,
   pTablePartitionFiltersSupport: true as const,
   pFrameInSetFilterSupport: true as const,
+  lazyColumnStatusSupport: true as const,
 };
 
 export interface GlobalCfgRenderCtx extends GlobalCfgRenderCtxMethods, ServiceDispatch {

package/src/render/util/axis_filtering.ts

@@ -69,7 +69,6 @@ export function filterDataInfoEntries<Blob>(
       break;
     }
     default:
-      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
       throw new Error(`Unsupported data info type: ${type satisfies never}`);
   }
 

package/src/render/util/column_collection.ts

@@ -7,7 +7,6 @@ import type {
   NativePObjectId,
   PartitionedDataInfoEntries,
   PColumn,
-  PColumnLazy,
   PColumnSelector,
   PColumnSpec,
   PColumnValues,
@@ -60,7 +59,7 @@ export interface AxisLabelProvider {
 /**
  * A simple implementation of {@link ColumnProvider} backed by a pre-defined array of columns.
  */
-class ArrayColumnProvider implements ColumnProvider {
+class ArrayColumnsProvider implements ColumnProvider {
   constructor(private readonly columns: PColumn<PColumnDataUniversal | undefined>[]) {}
 
   selectColumns(
@@ -76,21 +75,21 @@ class ArrayColumnProvider implements ColumnProvider {
 }
 
 /** Lazy calculates the data, returns undefined if data is not ready. */
-export type PColumnLazyWithLabel<T> = PColumnLazy<T> & {
+export type PColumnWithLabel<T> = PColumn<T> & {
   label: string;
 };
 
 /** Universal column is a column that uses a universal column id, and always have label. */
-export type PColumnLazyUniversal<T> = PColumnLazyWithLabel<T> & {
+export type PColumnUniversal<T> = PColumnWithLabel<T> & {
   id: SUniversalPColumnId;
 };
 
-/** @deprecated Use PColumnLazyWithLabel instead. */
-export type PColumnEntryWithLabel = PColumnLazy<undefined | PColumnDataUniversal> & {
+/** @deprecated Use PColumnWithLabel instead. */
+export type PColumnEntryWithLabel = PColumn<undefined | PColumnDataUniversal> & {
   label: string;
 };
 
-/** @deprecated Use PColumnLazyUniversal instead. */
+/** @deprecated Use PColumnUniversal instead. */
 export type PColumnEntryUniversal = PColumnEntryWithLabel & {
   id: SUniversalPColumnId;
 };
@@ -220,7 +219,7 @@ type UniversalPColumnOpts = UniversalPColumnOptsNoDeriver & {
 export class PColumnCollection {
   private readonly defaultProviderStore: PColumn<PColumnDataUniversal | undefined>[] = [];
   private readonly providers: ColumnProvider[] = [
-    new ArrayColumnProvider(this.defaultProviderStore),
+    new ArrayColumnsProvider(this.defaultProviderStore),
   ];
   private readonly axisLabelProviders: AxisLabelProvider[] = [];
 
@@ -498,12 +497,13 @@ export class PColumnCollection {
       }
 
       result.push({
-        id: finalId,
+        id: finalId as PObjectId,
         spec: finalSpec,
-        data: () =>
-          entry.type === "split"
+        get data() {
+          return entry.type === "split"
             ? entriesToDataInfo(filterDataInfoEntries(entry.dataEntries, axisFiltersTuple!))
-            : entry.originalColumn.data,
+            : entry.originalColumn.data;
+        },
         label: label,
       });
     }
@@ -555,30 +555,30 @@ export class PColumnCollection {
       | APColumnSelectorWithSplit
       | APColumnSelectorWithSplit[],
     opts: UniversalPColumnOpts,
-  ): PColumn<PColumnDataUniversal>[] | undefined;
+  ): PColumn<undefined | PColumnDataUniversal>[] | undefined;
   public getColumns(
     predicateOrSelectors:
       | ((spec: PColumnSpec) => boolean)
       | PColumnSelectorWithSplit
       | PColumnSelectorWithSplit[],
     opts?: UniversalPColumnOptsNoDeriver,
-  ): PColumn<PColumnDataUniversal>[] | undefined;
+  ): PColumn<undefined | PColumnDataUniversal>[] | undefined;
   public getColumns(
     predicateOrSelectors:
       | ((spec: PColumnSpec) => boolean)
       | APColumnSelectorWithSplit
       | APColumnSelectorWithSplit[],
     opts?: Optional<UniversalPColumnOpts, "anchorCtx">,
-  ): PColumn<PColumnDataUniversal>[] | undefined {
+  ): PColumn<undefined | PColumnDataUniversal>[] | undefined {
     const entries = this.getUniversalEntries(predicateOrSelectors, {
       overrideLabelAnnotation: true, // default for getColumns
       ...opts,
     } as UniversalPColumnOpts);
     if (!entries) return undefined;
 
-    const columns: PColumn<PColumnDataUniversal>[] = [];
+    const columns: PColumn<undefined | PColumnDataUniversal>[] = [];
     for (const entry of entries) {
-      const data = entry.data();
+      const data = entry.data;
       if (!data) {
         if (opts?.dontWaitAllData) continue;
         return undefined;

package/src/render/util/pcolumn_data.ts

@@ -3,7 +3,6 @@ import type {
   ParquetChunkMetadata,
   PartitionedDataInfoEntries,
   PColumn,
-  PColumnLazy,
   PColumnValues,
 } from "@milaboratories/pl-model-common";
 import {
@@ -606,14 +605,14 @@ export function getNumberOfRows(data: PColumnDataUniversal | undefined): number
 }
 
 export function isPColumnReady(
-  c: PColumn<undefined | PColumnDataUniversal> | PColumnLazy<undefined | PColumnDataUniversal>,
-): c is PColumn<PColumnDataUniversal> | PColumnLazy<PColumnDataUniversal> {
+  c: PColumn<undefined | PColumnDataUniversal>,
+): c is PColumn<PColumnDataUniversal> {
   const isValues = (d: PColumnDataUniversal): d is PColumnValues => Array.isArray(d);
   const isAccessor = (d: PColumnDataUniversal): d is TreeNodeAccessor =>
     d instanceof TreeNodeAccessor;
 
   let ready = true;
-  const data = typeof c.data === "function" ? c.data() : c.data;
+  const data = c.data;
   if (data == null) {
     return false;
   } else if (isAccessor(data)) {
@@ -627,10 +626,7 @@ export function isPColumnReady(
 }
 
 export function allPColumnsReady(
-  columns: (
-    | PColumn<undefined | PColumnDataUniversal>
-    | PColumnLazy<undefined | PColumnDataUniversal>
-  )[],
-): columns is (PColumn<PColumnDataUniversal> | PColumnLazy<PColumnDataUniversal>)[] {
+  columns: PColumn<undefined | PColumnDataUniversal>[],
+): columns is PColumn<PColumnDataUniversal>[] {
   return columns.every(isPColumnReady);
 }

package/src/services/block_services.ts

@@ -12,6 +12,7 @@ export const BLOCK_SERVICE_FLAGS = {
   requiresPFrameSpec: true,
   requiresPFrame: true,
   requiresDialog: true,
+  requiresColumnsCollection: true,
 } as const satisfies Partial<ServiceRequireFlags>;
 
 export type BlockServiceFlags = typeof BLOCK_SERVICE_FLAGS;

package/src/services/get_services.ts

@@ -9,8 +9,11 @@ const cachedServices = new WeakMap<
   Map<keyof ModelServices, ValueOf<ModelServices>>
 >();
 
-export function getService<T extends keyof ModelServices>(name: T): ModelServices[T] {
-  const ctx = getCfgRenderCtx();
+export function getService<T extends keyof ModelServices>(
+  name: T,
+  deps?: { ctx?: GlobalCfgRenderCtx },
+): ModelServices[T] {
+  const ctx = deps?.ctx ?? getCfgRenderCtx();
 
   const map = cachedServices.has(ctx)
     ? cachedServices.get(ctx)!