@milaboratories/pl-middle-layer 1.65.8 → 1.66.0

package/src/js_render/column_registry.ts

@@ -0,0 +1,95 @@
+import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import {
+  AccessorEntriesProvider,
+  applySpecOverrides,
+  ColumnRegistry,
+  extractPObjectId,
+  PColumnValues,
+  ResultPoolEntriesProvider,
+  SpecOverrides,
+  type ColumnEntriesProvider,
+  type PColumn,
+  type PColumnSpec,
+  type PObjectId,
+  type UpstreamBlockCtx,
+} from "@milaboratories/pl-model-common";
+
+export interface ColumnRegistryRoots {
+  /** Main `outputs` accessor (if available) — usually `MainAccessorName`. */
+  readonly outputs?: PlTreeNodeAccessor;
+  /** Prerun `staging` accessor (if available) — usually `StagingAccessorName`. */
+  readonly prerun?: PlTreeNodeAccessor;
+  /** Upstream-block ctx accessors (as `collectUpstreamBlockCtxes` returns). */
+  readonly upstreamBlockCtxes: ReadonlyArray<UpstreamBlockCtx<PlTreeNodeAccessor>>;
+}
+
+/**
+ * Build a host-side {@link ColumnRegistry} mirroring the sandbox layout:
+ * `outputs` precedes `prerun` precedes `rawResultPool`, first-wins on conflict.
+ *
+ * Uses raw {@link PlTreeNodeAccessor}s directly — both `TreeNodeAccessor`
+ * (sandbox) and `PlTreeNodeAccessor` (host) satisfy the `AccessorLike`
+ * traversal surface, so no host-side adapter is required.
+ */
+export function buildColumnRegistry(
+  roots: ColumnRegistryRoots,
+): ColumnRegistry<PlTreeNodeAccessor> {
+  const providers: ColumnEntriesProvider<PlTreeNodeAccessor>[] = [];
+
+  if (roots.outputs !== undefined) {
+    providers.push(new AccessorEntriesProvider(roots.outputs, ["main"]));
+  }
+  if (roots.prerun !== undefined) {
+    providers.push(new AccessorEntriesProvider(roots.prerun, ["staging"]));
+  }
+
+  providers.push(new ResultPoolEntriesProvider<PlTreeNodeAccessor>(roots.upstreamBlockCtxes));
+
+  return new ColumnRegistry<PlTreeNodeAccessor>(providers);
+}
+
+/**
+ * Resolve a column id to a materialised {@link PColumn} backed by the
+ * underlying {@link PlTreeNodeAccessor} for the `.data` resource. Returns the
+ * bare leaf spec (via {@link extractPObjectId}): filters / overrides encoded in
+ * a rich id are NOT reconstructed here — they are applied by the query nodes
+ * (`sliceAxes` / `specOverride`) that wrap this leaf in the def, so reconstructing
+ * would double-apply them. Only the explicit `overrides` argument is folded in.
+ *
+ * Throws when the id is not in the registry or the column has no resolved spec.
+ * A `.data` field that has not yet materialised yields an empty value list — the
+ * column refills on a later recomputation, mirroring the sandbox-side
+ * `finalizePColumnData` behaviour rather than blocking the whole PFrame.
+ */
+export function resolvePColumnById(
+  registry: ColumnRegistry<PlTreeNodeAccessor>,
+  id: PObjectId,
+  overrides?: SpecOverrides,
+): PColumn<PColumnValues | PlTreeNodeAccessor> {
+  const pid = extractPObjectId(id);
+  const leaf = registry.resolve(pid);
+  if (leaf === undefined) {
+    throw new Error(`column id ${String(pid)} not found in host column registry`);
+  }
+  const specNode = leaf.accessor.traverse({
+    field: `${leaf.name}.spec`,
+    assertFieldType: "Input",
+    ignoreError: true,
+  });
+  const spec = specNode?.getDataAsJson<PColumnSpec>();
+  if (spec === undefined) {
+    throw new Error(`column ${String(pid)} has no resolved spec`);
+  }
+  const data =
+    leaf.accessor.traverse({
+      field: `${leaf.name}.data`,
+      assertFieldType: "Input",
+      ignoreError: true,
+    }) ?? [];
+
+  return {
+    id: id as PObjectId,
+    spec: applySpecOverrides(spec, overrides),
+    data,
+  };
+}

package/src/js_render/computable_context.ts

@@ -7,14 +7,12 @@ import type {
   CommonFieldTraverseOps as CommonFieldTraverseOpsFromSDK,
   DataInfo,
   FieldTraversalStep as FieldTraversalStepFromSDK,
-  Option,
   PColumn,
   PColumnValues,
   PFrameDef,
   PFrameHandle,
   PObject,
   PObjectSpec,
-  PSpecPredicate,
   PTableDef,
   PTableDefV2,
   PTableHandle,
@@ -42,6 +40,7 @@ import type { MiddleLayerEnvironment } from "../middle_layer/middle_layer";
 import type { Block } from "../model/project_model";
 import { parseFinalPObjectCollection } from "../pool/p_object_collection";
 import type { ResultPool } from "../pool/result_pool";
+import { collectUpstreamBlockCtx, type UpstreamBlockCtx } from "../pool/upstream_block_ctx";
 import type { JsExecutionContext } from "./context";
 import type { VmFunctionImplementation } from "quickjs-emscripten";
 import { Scope, type QuickJSHandle } from "quickjs-emscripten";
@@ -54,10 +53,10 @@ import {
   ServiceMethodNotFoundError,
 } from "@milaboratories/pl-model-common";
 import { getServiceInjectors } from "./service_injectors";
-
-function bytesToBase64(data: Uint8Array | undefined): string | undefined {
-  return data !== undefined ? Buffer.from(data).toString("base64") : undefined;
-}
+import { MainAccessorName, StagingAccessorName } from "@platforma-sdk/model";
+import { buildColumnRegistry, resolvePColumnById } from "./column_registry";
+import type { ColumnRegistry, PObjectId } from "@milaboratories/pl-model-common";
+import { collapseSpecOverrideNode } from "./spec_override_collapse";
 
 export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRenderCtxMethods<
   string,
@@ -92,6 +91,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
   public resetComputableCtx() {
     this.computableCtx = undefined;
     this.accessors.clear();
+    this._resultPool = undefined;
+    this._upstreamBlockCtx = undefined;
+    this._rawUpstreamBlockCtx = undefined;
+    this._columnRegistry = undefined;
   }
 
   private get requireComputableCtx(): ComputableCtx {
@@ -153,6 +156,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
     return this.getAccessor(handle).getIsReadyOrError();
   }
 
+  hasData(handle: string): boolean {
+    return this.getAccessor(handle).hasData();
+  }
+
   getIsFinal(handle: string): boolean {
     return this.getAccessor(handle).getIsFinal();
   }
@@ -350,10 +357,6 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
     return this._resultPool;
   }
 
-  public calculateOptions(predicate: PSpecPredicate): Option[] {
-    return this.resultPool.calculateOptions(predicate);
-  }
-
   public getDataFromResultPool(): ResultCollection<PObject<string>> {
     const collection = this.resultPool.getData();
     if (collection.instabilityMarker !== undefined)
@@ -409,28 +412,79 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
     );
   }
 
+  /**
+   * Single source-of-truth cache of upstream-block ctx pairs as raw
+   * `PlTreeNodeAccessor`s. Both the sandbox-facing `getUpstreamBlockCtx()`
+   * (string-handle view) and the host column registry are derived from this.
+   */
+  private _rawUpstreamBlockCtx: UpstreamBlockCtx<PlTreeNodeAccessor>[] | undefined = undefined;
+  /**
+   * Raw upstream-block ctx accessors for the current render. Public so the
+   * `ColumnsCollection` service injector (`service_injectors.ts`) can build
+   * `ColumnsCollectionDriverHost.getUpstreamBlockCtxes` bindings without
+   * round-tripping through the handle-shape view.
+   */
+  public getRawUpstreamBlockCtx(): UpstreamBlockCtx<PlTreeNodeAccessor>[] {
+    if (this._rawUpstreamBlockCtx === undefined) {
+      const prjEntry = notEmpty(this.blockCtx.projectEntry, "projectEntry");
+      this._rawUpstreamBlockCtx = collectUpstreamBlockCtx(
+        this.requireComputableCtx,
+        prjEntry,
+        this.blockCtx.blockId,
+      );
+    }
+    return this._rawUpstreamBlockCtx;
+  }
+
+  private _upstreamBlockCtx: UpstreamBlockCtx<string>[] | undefined = undefined;
+  public getUpstreamBlockCtx(): UpstreamBlockCtx<string>[] {
+    if (this._upstreamBlockCtx === undefined) {
+      this._upstreamBlockCtx = this.getRawUpstreamBlockCtx().map((b) => ({
+        blockId: b.blockId,
+        prodCtx: this.wrapAccessor(b.prodCtx),
+        stagingCtx: this.wrapAccessor(b.stagingCtx),
+        prodIncomplete: b.prodIncomplete,
+        stagingIncomplete: b.stagingIncomplete,
+      }));
+    }
+    return this._upstreamBlockCtx;
+  }
+
+  //
+  // Host-side column registry — id → LeafEntry index over outputs/prerun/raw pool.
+  // Mirrors the sandbox `ColumnRegistry` so `createPFrame` / `createPTable` can
+  // accept `PObjectId` in addition to materialised `PColumn`s.
+  //
+  private _columnRegistry: ColumnRegistry<PlTreeNodeAccessor> | undefined = undefined;
+  public getColumnRegistry(): ColumnRegistry<PlTreeNodeAccessor> {
+    if (this._columnRegistry !== undefined) return this._columnRegistry;
+    return (this._columnRegistry = buildColumnRegistry({
+      outputs: this.accessors.get(MainAccessorName) ?? undefined,
+      prerun: this.accessors.get(StagingAccessorName) ?? undefined,
+      upstreamBlockCtxes: this.getRawUpstreamBlockCtx(),
+    }));
+  }
+
   //
   // PFrames / PTables
   //
 
   public createPFrame(
-    def: PFrameDef<PColumn<string | PColumnValues | DataInfo<string>>>,
+    def: PFrameDef<PObjectId | PColumn<string | PColumnValues | DataInfo<string>>>,
   ): PFrameHandle {
     using guard = new PoolEntryGuard(
-      this.env.driverKit.pFrameDriver.createPFrame(
-        def.map((c) => mapPObjectData(c, (d) => this.transformInputPData(d))),
-      ),
+      this.env.driverKit.pFrameDriver.createPFrame(def.map((c) => this.resolvePColumn(c))),
     );
     this.requireComputableCtx.addOnDestroy(guard.entry.unref);
     return guard.keep().key;
   }
 
   public createPTable(
-    def: PTableDef<PColumn<string | PColumnValues | DataInfo<string>>>,
+    def: PTableDef<PObjectId | PColumn<string | PColumnValues | DataInfo<string>>>,
   ): PTableHandle {
     using guard = new PoolEntryGuard(
       this.env.driverKit.pFrameDriver.createPTable(
-        mapPTableDef(def, (c) => mapPObjectData(c, (d) => this.transformInputPData(d))),
+        mapPTableDef(def, (c) => this.resolvePColumn(c)),
       ),
     );
     this.requireComputableCtx.addOnDestroy(guard.entry.unref);
@@ -438,17 +492,35 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
   }
 
   public createPTableV2(
-    def: PTableDefV2<PColumn<string | PColumnValues | DataInfo<string>>>,
+    def: PTableDefV2<PObjectId | PColumn<string | PColumnValues | DataInfo<string>>>,
   ): PTableHandle {
     using guard = new PoolEntryGuard(
       this.env.driverKit.pFrameDriver.createPTableV2(
-        mapPTableDefV2(def, (c) => mapPObjectData(c, (d) => this.transformInputPData(d))),
+        mapPTableDefV2(def, {
+          column: (c) => this.resolvePColumn(c),
+          node: collapseSpecOverrideNode,
+        }),
       ),
     );
     this.requireComputableCtx.addOnDestroy(guard.entry.unref);
     return guard.keep().key;
   }
 
+  /**
+   * Normalise a def element: pass-through for materialised columns (after
+   * mapping their `data` payload via {@link transformInputPData}), or resolve
+   * a bare id through the host column registry. `resolvePColumnById` extracts
+   * the underlying {@link PObjectId} from any id encoding
+   * ({@link ColumnOverriddenId}, etc.).
+   */
+  private resolvePColumn(
+    item: PObjectId | PColumn<string | PColumnValues | DataInfo<string>>,
+  ): PColumn<PlTreeNodeAccessor | PColumnValues | DataInfo<PlTreeNodeAccessor>> {
+    return typeof item === "string"
+      ? resolvePColumnById(this.getColumnRegistry(), item)
+      : mapPObjectData(item, (d) => this.transformInputPData(d));
+  }
+
   /**
    * Transforms input data for PFrame/PTable creation
    * - Converts string handles to accessors
@@ -495,7 +567,13 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
   // Helpers
   //
 
-  private getAccessor(handle: string): PlTreeNodeAccessor {
+  /**
+   * Resolve a sandbox-side `AccessorHandle` string back to its concrete
+   * host accessor. Used internally by every `traverse`/`getX(handle)`
+   * method and externally by the `ColumnsCollection` service injector
+   * for `SerializedColumnsSource` of kind `"accessor"`.
+   */
+  public getAccessor(handle: string): PlTreeNodeAccessor {
     const accessor = this.accessors.get(handle);
     if (accessor === undefined) throw new Error("No such accessor");
     return accessor;
@@ -649,6 +727,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
         return parent.exportSingleValue(this.getIsFinal(vm.getString(handle)), undefined);
       });
 
+      exportCtxFunction("hasData", (handle) => {
+        return parent.exportSingleValue(this.hasData(vm.getString(handle)), undefined);
+      });
+
       exportCtxFunction("getError", (handle) => {
         return parent.exportSingleValue(this.getError(vm.getString(handle)), undefined);
       });
@@ -816,13 +898,6 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
         return parent.exportObjectUniversal(this.getSpecsFromResultPool(), undefined);
       });
 
-      exportCtxFunction("calculateOptions", (predicate) => {
-        return parent.exportObjectUniversal(
-          this.calculateOptions(parent.importObjectViaJson(predicate) as PSpecPredicate),
-          undefined,
-        );
-      });
-
       exportCtxFunction("getSpecFromResultPoolByRef", (blockId, exportName) => {
         return parent.exportObjectUniversal(
           this.getSpecFromResultPoolByRef(vm.getString(blockId), vm.getString(exportName)),
@@ -837,6 +912,10 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
         );
       });
 
+      exportCtxFunction("getUpstreamBlockCtx", () => {
+        return parent.exportObjectUniversal(this.getUpstreamBlockCtx(), undefined);
+      });
+
       exportCtxFunction("createPFrame", (def) => {
         return parent.exportSingleValue(
           this.createPFrame(
@@ -938,3 +1017,7 @@ export class ComputableContextHelper implements JsRenderInternal.GlobalCfgRender
     });
   }
 }
+
+function bytesToBase64(data: Uint8Array | undefined): string | undefined {
+  return data !== undefined ? Buffer.from(data).toString("base64") : undefined;
+}

package/src/js_render/service_injectors.ts

@@ -1,6 +1,15 @@
 import type { QuickJSHandle, VmFunctionImplementation } from "quickjs-emscripten";
-import type { InferServiceModel, ServiceBrand } from "@milaboratories/pl-model-common";
+import type {
+  CollectionHandle,
+  ColumnsCollectionDriverHost,
+  DiscoverColumnsOptions,
+  InferServiceModel,
+  PoolEntry,
+  SerializedColumnsSource,
+  ServiceBrand,
+} from "@milaboratories/pl-model-common";
 import { Services, ServiceNotRegisteredError } from "@milaboratories/pl-model-common";
+import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 import type {
   AxesId,
   AxesSpec,
@@ -8,6 +17,7 @@ import type {
   PColumn,
   PColumnSpec,
   PColumnValues,
+  PObjectId,
   PTableColumnId,
   PTableColumnSpec,
   PTableRecordFilter,
@@ -141,16 +151,12 @@ export function getServiceInjectors(): ServiceInjectorMap {
       };
     },
 
-    // Dialog has no model-side surface — workflow scripts cannot open save dialogs.
-    // Declared as an empty injector to satisfy the exhaustive ServiceInjectorMap.
-    Dialog: () => ({}) as Record<never, VmMethod>,
-
     PFrame: ({ host, vm }: ServiceInjectorContext) => ({
       createPFrame: (def: QuickJSHandle) =>
         vm.exportSingleValue(
           host.createPFrame(
             vm.importObjectViaJson(def) as PFrameDef<
-              PColumn<string | PColumnValues | DataInfo<string>>
+              PObjectId | PColumn<string | PColumnValues | DataInfo<string>>
             >,
           ),
         ),
@@ -159,7 +165,7 @@ export function getServiceInjectors(): ServiceInjectorMap {
         vm.exportSingleValue(
           host.createPTable(
             vm.importObjectViaJson(def) as PTableDef<
-              PColumn<string | PColumnValues | DataInfo<string>>
+              PObjectId | PColumn<string | PColumnValues | DataInfo<string>>
             >,
           ),
         ),
@@ -168,10 +174,102 @@ export function getServiceInjectors(): ServiceInjectorMap {
         vm.exportSingleValue(
           host.createPTableV2(
             vm.importObjectViaJson(def) as PTableDefV2<
-              PColumn<string | PColumnValues | DataInfo<string>>
+              PObjectId | PColumn<string | PColumnValues | DataInfo<string>>
             >,
           ),
         ),
     }),
+
+    // Dialog has no model-side surface — workflow scripts cannot open save dialogs.
+    // Declared as an empty injector to satisfy the exhaustive ServiceInjectorMap.
+    Dialog: () => ({}) as Record<never, VmMethod>,
+
+    ColumnsCollection: ({ host, vm }: ServiceInjectorContext) => {
+      const driver = host.serviceRegistry.get(Services.ColumnsCollection);
+      if (!driver)
+        throw new ServiceNotRegisteredError(
+          `Service "${Services.ColumnsCollection}" has no factory in ModelServiceRegistry. Provide a non-null factory.`,
+        );
+      const specDriver = host.serviceRegistry.get(Services.PFrameSpec);
+      if (!specDriver)
+        throw new ServiceNotRegisteredError(
+          `Service "${Services.PFrameSpec}" has no factory in ModelServiceRegistry. Provide a non-null factory.`,
+        );
+
+      const bindings: ColumnsCollectionDriverHost<PlTreeNodeAccessor> = {
+        resolveAccessor: (h) => host.getAccessor(h),
+        getUpstreamBlockCtxes: () => host.getRawUpstreamBlockCtx(),
+        getSpecDriver: () => specDriver,
+        resolveSpec: (id: PObjectId) => {
+          const leaf = host.getColumnRegistry().resolve(id);
+          if (leaf === undefined) return undefined;
+          const specNode = leaf.accessor.traverse({
+            field: `${leaf.name}.spec`,
+            assertFieldType: "Input",
+            ignoreError: true,
+          });
+          return specNode?.getDataAsJson<PColumnSpec>();
+        },
+      };
+
+      const pinHandle = (entry: PoolEntry<CollectionHandle>): CollectionHandle => {
+        using guard = new PoolEntryGuard(entry);
+        host.addOnDestroy(guard.entry.unref);
+        return guard.keep().key;
+      };
+
+      return {
+        create: (sources: QuickJSHandle) =>
+          vm.exportSingleValue(
+            pinHandle(
+              driver.create(vm.importObjectViaJson(sources) as SerializedColumnsSource[], bindings),
+            ),
+          ),
+
+        isEmpty: (handle: QuickJSHandle) =>
+          vm.exportSingleValue(driver.isEmpty(vm.vm.getString(handle) as CollectionHandle)),
+
+        isFinal: (handle: QuickJSHandle) =>
+          vm.exportSingleValue(driver.isFinal(vm.vm.getString(handle) as CollectionHandle)),
+
+        getColumns: (handle: QuickJSHandle) =>
+          vm.exportObjectViaJson(
+            driver.getColumns(vm.vm.getString(handle) as CollectionHandle, bindings),
+          ),
+
+        addSource: (handle: QuickJSHandle, sources: QuickJSHandle) =>
+          vm.exportSingleValue(
+            pinHandle(
+              driver.addSource(
+                vm.vm.getString(handle) as CollectionHandle,
+                vm.importObjectViaJson(sources) as SerializedColumnsSource[],
+                bindings,
+              ),
+            ),
+          ),
+
+        discover: (handle: QuickJSHandle, opts: QuickJSHandle) =>
+          vm.exportSingleValue(
+            pinHandle(
+              driver.discover(
+                vm.vm.getString(handle) as CollectionHandle,
+                vm.importObjectViaJson(opts) as DiscoverColumnsOptions,
+                bindings,
+              ),
+            ),
+          ),
+
+        filter: (handle: QuickJSHandle, opts: QuickJSHandle) =>
+          vm.exportSingleValue(
+            pinHandle(
+              driver.filter(
+                vm.vm.getString(handle) as CollectionHandle,
+                vm.importObjectViaJson(opts) as DiscoverColumnsOptions,
+                bindings,
+              ),
+            ),
+          ),
+      };
+    },
   };
 }