@milaboratories/pl-middle-layer 1.65.8 → 1.66.0

package/src/js_render/spec_override_collapse.ts

@@ -0,0 +1,211 @@
+import type { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import type { DataInfo, PColumn, PColumnValues } from "@platforma-sdk/model";
+import { applySpecOverrides, matchAxis } from "@milaboratories/pl-model-common";
+import type {
+  AxesSpec,
+  AxisPatches,
+  SpecOverrides,
+  SpecQuery,
+} from "@milaboratories/pl-model-common";
+
+type Leaf = PColumn<PlTreeNodeAccessor | PColumnValues | DataInfo<PlTreeNodeAccessor>>;
+type Node = SpecQuery<Leaf>;
+
+/**
+ * Collapse the client-side `specOverride` query node at the host boundary —
+ * pframe-engine never sees it. The override is pushed down through the
+ * subtree until it lands on `column`-leaf nodes, where it is folded into
+ * the leaf's spec via {@link applySpecOverrides}. Intermediate join nodes
+ * are left untouched — pframe-engine will recompute their result spec
+ * from the patched leaves.
+ *
+ * Designed to run as the `node` visitor in `mapPTableDefV2` (post-order),
+ * so by the time we see a `specOverride` its inner subtree is already
+ * mapped to {@link PColumn} leaves.
+ *
+ * Currently only `column` and `linkerJoin` are walked. Other node shapes
+ * under `specOverride` throw — pframe-engine support for them is still
+ * in progress. Extend the switch in {@link pushSpecOverrideDown} as more
+ * shapes become needed.
+ */
+export function collapseSpecOverrideNode(node: Node): Node {
+  if (node.type !== "specOverride") return node;
+  return pushSpecOverrideDown(node.input, node.override);
+}
+
+function pushSpecOverrideDown(node: Node, overrides: SpecOverrides): Node {
+  switch (node.type) {
+    case "column":
+      return {
+        type: "column",
+        column: { ...node.column, spec: applySpecOverrides(node.column.spec, overrides) },
+      };
+
+    case "linkerJoin": {
+      // `axesSpec` patches are positional against the linkerJoin's OUTPUT axes,
+      // which equal the secondary side's axes (ColumnDiscoveredRecipe emits the
+      // hit on the many-side of the linker). Route axesSpec only into secondary;
+      // applying the same positional indices to the linker would silently
+      // mis-target the linker's own axes.
+      //
+      // Non-axesSpec patches (domain / contextDomain / annotations) are
+      // column-level and not positional — keep the existing both-sides walk.
+      const { axesSpec: _axesSpec, ...nonAxesOverrides } = overrides;
+      const hasNonAxes =
+        nonAxesOverrides.domain !== undefined ||
+        nonAxesOverrides.contextDomain !== undefined ||
+        nonAxesOverrides.annotations !== undefined;
+      return {
+        ...node,
+        linker: hasNonAxes ? pushSpecOverrideDown(node.linker, nonAxesOverrides) : node.linker,
+        secondary: node.secondary.map((e) => ({
+          ...e,
+          entry: pushSpecOverrideDown(e.entry, overrides),
+        })),
+      };
+    }
+
+    // case "outerJoin":
+    //   return {
+    //     ...node,
+    //     primary: {
+    //       ...node.primary,
+    //       entry: pushSpecOverrideDown(node.primary.entry, overrides),
+    //     },
+    //     secondary: node.secondary.map((e) => ({
+    //       ...e,
+    //       entry: pushSpecOverrideDown(e.entry, overrides),
+    //     })),
+    //   };
+
+    // case "innerJoin":
+    // case "fullJoin":
+    // case "symmetricJoin":
+    //   return {
+    //     ...node,
+    //     entries: node.entries.map((e) => ({
+    //       ...e,
+    //       entry: pushSpecOverrideDown(e.entry, overrides),
+    //     })),
+    //   };
+
+    case "sliceAxes": {
+      // axesSpec patches are positional against the slice OUTPUT axes (the
+      // recipe-layer spec the consumer sees), but the inner input still
+      // carries the full pre-slice axes list. Translate outer indices to
+      // inner indices before pushing the override down.
+      //
+      // Non-axesSpec overrides (domain / contextDomain / annotations) are
+      // column-level and pass through transparently.
+      if (overrides.axesSpec === undefined || Object.keys(overrides.axesSpec).length === 0) {
+        return { ...node, input: pushSpecOverrideDown(node.input, overrides) };
+      }
+
+      const innerAxes = effectiveAxesSpec(node.input);
+      if (innerAxes === undefined) {
+        throw new Error(
+          `specOverride over sliceAxes: cannot translate axesSpec — ` +
+            `inner shape unknown for input type "${node.input.type}"`,
+        );
+      }
+
+      const filteredInnerIdxs = new Set<number>();
+      for (const filter of node.axisFilters) {
+        const idx = innerAxes.findIndex((a) => matchAxis(filter.axisSelector, a));
+        if (idx < 0) {
+          throw new Error(
+            `specOverride over sliceAxes: filter selector ` +
+              `${JSON.stringify(filter.axisSelector)} not found in inner axes`,
+          );
+        }
+        filteredInnerIdxs.add(idx);
+      }
+
+      // outer index j (post-slice) ↔ inner index = j-th surviving position.
+      const outerToInner: number[] = [];
+      for (let i = 0; i < innerAxes.length; i++) {
+        if (!filteredInnerIdxs.has(i)) outerToInner.push(i);
+      }
+
+      const translated: AxisPatches = {};
+      for (const [outerKey, patch] of Object.entries(overrides.axesSpec)) {
+        const outerIdx = Number(outerKey);
+        const innerIdx = outerToInner[outerIdx];
+        if (innerIdx === undefined) {
+          throw new Error(
+            `specOverride over sliceAxes: outer axesSpec index ${outerIdx} ` +
+              `out of range (slice output has ${outerToInner.length} axes)`,
+          );
+        }
+        translated[innerIdx] = patch;
+      }
+
+      const translatedOverrides: SpecOverrides = { ...overrides, axesSpec: translated };
+      return { ...node, input: pushSpecOverrideDown(node.input, translatedOverrides) };
+    }
+
+    // // Transparent transforms — passthrough.
+    // case "sort":
+    // case "filter":
+    // case "transformColumns":
+    //   return { ...node, input: pushSpecOverrideDown(node.input, overrides) };
+
+    // // Synthetic leaves — patch the spec carrier they expose, or
+    // // throw if the node has no natural place to absorb overrides.
+    // case "inlineColumn":
+    //   return {
+    //     ...node,
+    //     column: { ...node.column, spec: applySpecOverrides(node.column.spec, overrides) },
+    //   };
+
+    // case "sparseToDenseColumn":
+    //   // Same idea: patch the new dense column's spec carrier.
+    //   return {
+    //     ...node,
+    //     newSpec: applySpecOverrides(node.newSpec, overrides),
+    //   };
+
+    default:
+      throw new Error(
+        `specOverride: inner node "${node.type}" is not supported yet — ` +
+          "only column / linkerJoin can carry an override " +
+          "(pframe-engine support for the rest is in progress)",
+      );
+  }
+}
+
+/**
+ * Effective post-traversal {@link AxesSpec} of a {@link Node} — used to
+ * translate positional axesSpec patches across structural nodes whose
+ * indexing differs from their input (currently only `sliceAxes`).
+ *
+ * Returns `undefined` when the node shape is not introspectable here.
+ * Callers must then refuse axesSpec translation and surface a clear error,
+ * rather than silently mis-targeting indices.
+ */
+function effectiveAxesSpec(node: Node): AxesSpec | undefined {
+  switch (node.type) {
+    case "column":
+      return node.column.spec.axesSpec;
+
+    case "linkerJoin":
+      // Output axes equal the secondary side's axes (linker side drops out).
+      // All secondary entries share the same axis identity post-join — read
+      // the first one.
+      return node.secondary.length === 0 ? undefined : effectiveAxesSpec(node.secondary[0].entry);
+
+    case "sliceAxes": {
+      const inner = effectiveAxesSpec(node.input);
+      if (inner === undefined) return undefined;
+      const filtered = new Set<number>();
+      for (const f of node.axisFilters) {
+        const i = inner.findIndex((a) => matchAxis(f.axisSelector, a));
+        if (i >= 0) filtered.add(i);
+      }
+      return inner.filter((_, i) => !filtered.has(i));
+    }
+
+    default:
+      return undefined;
+  }
+}

package/src/middle_layer/block_ctx.ts

@@ -22,9 +22,13 @@ export type BlockContextFull = BlockContextArgsOnly & {
   readonly prod: (cCtx: ComputableCtx) => PlTreeEntry | undefined;
   readonly staging: (cCtx: ComputableCtx) => PlTreeEntry | undefined;
   readonly getResultsPool: (cCtx: ComputableCtx) => ResultPool;
+  readonly projectEntry: PlTreeEntry;
 };
 
-export type BlockContextAny = Optional<BlockContextFull, "prod" | "staging" | "getResultsPool">;
+export type BlockContextAny = Optional<
+  BlockContextFull,
+  "prod" | "staging" | "getResultsPool" | "projectEntry"
+>;
 
 export function constructBlockContextArgsOnly(
   projectEntry: PlTreeEntry,
@@ -159,5 +163,6 @@ export function constructBlockContext(
       return result;
     },
     getResultsPool: (cCtx: ComputableCtx) => ResultPool.create(cCtx, projectEntry, blockId),
+    projectEntry,
   };
 }

package/src/mutator/template/template_cache.test.ts

@@ -39,8 +39,6 @@ function createTestCacheInTx(pl: Parameters<Parameters<typeof TestHelpers.withTe
   });
 }
 
-// ─── flattenTemplateTree ─────────────────────────────────────────────────────
-
 describe("flattenTemplateTree", () => {
   test("produces nodes in topological order for V2 template", () => {
     const data = parseTemplate(ExplicitTemplateEnterNumbers);
@@ -77,8 +75,6 @@ describe("flattenTemplateTree", () => {
   });
 });
 
-// ─── getOrCreateTemplateCache / dropTemplateCache ────────────────────────────
-
 describe("getOrCreateTemplateCache", () => {
   test("creates cache on first call and reuses on second", async () => {
     await TestHelpers.withTempRoot(async (pl) => {
@@ -100,8 +96,6 @@ describe("dropTemplateCache", () => {
   });
 });
 
-// ─── loadTemplateCached ──────────────────────────────────────────────────────
-
 describe("loadTemplateCached", () => {
   test("cache miss then cache hit returns same SignedResourceId", async () => {
     await TestHelpers.withTempRoot(async (pl) => {
@@ -184,8 +178,6 @@ describe("loadTemplateCached", () => {
   }, 15000);
 });
 
-// ─── cacheBlockPackTemplate ──────────────────────────────────────────────────
-
 describe("cacheBlockPackTemplate", () => {
   test("replaces prepared template with cached reference", async () => {
     await TestHelpers.withTempRoot(async (pl) => {
@@ -235,8 +227,6 @@ describe("cacheBlockPackTemplate", () => {
   }, 15000);
 });
 
-// ─── Equivalence: cached vs legacy produce identical resources ───────────────
-
 describe("template cache produces equivalent resources", () => {
   test("cached and legacy templates deduplicate to same original (V2)", async () => {
     await TestHelpers.withTempRoot(async (pl) => {
@@ -272,8 +262,6 @@ describe("template cache produces equivalent resources", () => {
   }, 30000);
 });
 
-// ─── Shared library dedup ────────────────────────────────────────────────────
-
 describe("shared library dedup", () => {
   test("two different templates sharing a library reuse the same cache entry", async () => {
     await TestHelpers.withTempRoot(async (pl) => {
@@ -303,8 +291,6 @@ describe("shared library dedup", () => {
   }, 15000);
 });
 
-// ─── GC ──────────────────────────────────────────────────────────────────────
-
 describe("GC", () => {
   test("evicts entries when cache exceeds max entries", async () => {
     await TestHelpers.withTempRoot(async (pl) => {

package/src/mutator/template/template_cache.ts

@@ -48,8 +48,6 @@ export const ACCESS_COUNT_KEY = "_accessCount";
 /** @internal exported for testing */
 export const ACCESS_KEY_PREFIX = "access_";
 
-// ─── Stats ───────────────────────────────────────────────────────────────────
-
 export type TemplateCacheStat = {
   totalMs: number;
   flattenMs: number;
@@ -82,8 +80,6 @@ function initialStat(): TemplateCacheStat {
   };
 }
 
-// ─── Tree node abstraction ───────────────────────────────────────────────────
-
 interface CacheableNode {
   /** SHA-256 content hash (includes all descendant content) */
   hash: string;
@@ -94,8 +90,6 @@ interface CacheableNode {
   childHashes: string[];
 }
 
-// ─── Hash computation helpers ────────────────────────────────────────────────
-
 function getSourceCode(name: string, sources: Record<string, string>, sourceHash: string): string {
   return notEmpty(
     sources[sourceHash],
@@ -448,8 +442,6 @@ export function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): Ca
   }
 }
 
-// ─── Cache operations ────────────────────────────────────────────────────────
-
 /** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */
 const cacheRidMap = new WeakMap<PlClient, SignedResourceId>();
 
@@ -502,8 +494,6 @@ export async function dropTemplateCache(pl: PlClient): Promise<void> {
   invalidateTemplateCacheId(pl);
 }
 
-// ─── GC ──────────────────────────────────────────────────────────────────────
-
 /**
  * Run count-based garbage collection on the template cache.
  * Evicts least-recently-used entries when the cache exceeds maxEntries.
@@ -549,8 +539,6 @@ export async function runGc(
   });
 }
 
-// ─── Batched materialization ─────────────────────────────────────────────────
-
 /** Create a batch of cache nodes in the current transaction. */
 function createBatchNodes(
   tx: PlTransaction,
@@ -788,8 +776,6 @@ export async function loadTemplateCached(
   }
 }
 
-// ─── Caller helper ───────────────────────────────────────────────────────────
-
 /**
  * Pre-materialize a block pack's template via cache.
  * Returns a copy of the spec with the template replaced by a cached reference.

package/src/pool/data.ts

@@ -1,11 +1,12 @@
 import {
+  createGlobalPObjectId,
+  createLocalPObjectId,
   PFrameDriverError,
   type BinaryChunk,
   type ParquetChunk,
   type ParquetChunkMapping,
   type ParquetChunkMetadata,
   type PColumnValue,
-  type PlRef,
   type PObjectId,
   type PObjectSpec,
 } from "@platforma-sdk/model";
@@ -331,9 +332,9 @@ export function deriveLegacyPObjectId(spec: PObjectSpec, data: PlTreeNodeAccesso
 }
 
 export function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId {
-  return canonicalize({ __isRef: true, blockId, name: exportName } satisfies PlRef)! as PObjectId;
+  return createGlobalPObjectId(blockId, exportName);
 }
 
 export function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId {
-  return canonicalize({ resolvePath, name: outputName })! as PObjectId;
+  return createLocalPObjectId(resolvePath, outputName);
 }

package/src/pool/result_pool.ts

@@ -4,13 +4,12 @@ import type {
   Option,
   PObject,
   PObjectSpec,
-  PSpecPredicate,
   PlRef,
   ResultCollection,
   ResultPoolEntry,
   ValueOrError,
 } from "@platforma-sdk/model";
-import { executePSpecPredicate, mapValueInVOE } from "@platforma-sdk/model";
+import { mapValueInVOE } from "@platforma-sdk/model";
 import { notEmpty } from "@milaboratories/ts-helpers";
 import { outputRef } from "../model/args";
 import type { Block, ProjectStructure } from "../model/project_model";
@@ -239,28 +238,6 @@ export class ResultPool {
     return { entries, isComplete, instabilityMarker };
   }
 
-  public calculateOptions(predicate: PSpecPredicate): ExtendedOption[] {
-    const found: ExtendedOption[] = [];
-    for (const block of this.blocks.values()) {
-      const exportsChecked = new Set<string>();
-      const addToFound = (ctx: RawPObjectCollection) => {
-        for (const [exportName, result] of ctx.results) {
-          if (exportsChecked.has(exportName) || result.spec === undefined) continue;
-          exportsChecked.add(exportName);
-          if (executePSpecPredicate(predicate, result.spec))
-            found.push({
-              label: block.info.label + " / " + exportName,
-              ref: outputRef(block.info.id, exportName),
-              spec: result.spec,
-            });
-        }
-      };
-      if (block.staging !== undefined) addToFound(block.staging);
-      if (block.prod !== undefined) addToFound(block.prod);
-    }
-    return found;
-  }
-
   public static create(ctx: ComputableCtx, prjEntry: PlTreeEntry, rootBlockId: string): ResultPool {
     const prj = ctx.accessor(prjEntry).node();
 

package/src/pool/upstream_block_ctx.ts

@@ -0,0 +1,85 @@
+import type { ComputableCtx } from "@milaboratories/computable";
+import type { PlTreeEntry, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
+import { notEmpty } from "@milaboratories/ts-helpers";
+import type { UpstreamBlockCtx } from "@milaboratories/pl-model-common";
+import type { ProjectStructure } from "../model/project_model";
+import { ProjectStructureKey, projectFieldName } from "../model/project_model";
+import { allBlocks, stagingGraph } from "../model/project_model_util";
+
+export type { UpstreamBlockCtx } from "@milaboratories/pl-model-common";
+
+/**
+ * Collect upstream-block ctx accessors for the given root block.
+ *
+ * For each upstream block (per the staging graph) we try to resolve its
+ * `prodUiCtx` and `stagingUiCtx` resources. Only the `.ok` outcomes are
+ * surfaced; failures and missing fields collapse to `undefined`.
+ *
+ * SDK-side providers compose enumerate/status/data operations on top of these
+ * accessors using the helpers in `column_providers/accessor_traversal`.
+ *
+ * Note: this function is the only place that reads the project tree to build
+ * the upstream-pool view — the legacy `ResultPool` class is not used.
+ */
+export function collectUpstreamBlockCtx(
+  ctx: ComputableCtx,
+  prjEntry: PlTreeEntry,
+  rootBlockId: string,
+): UpstreamBlockCtx<PlTreeNodeAccessor>[] {
+  const prj = ctx.accessor(prjEntry).node();
+  const structure = notEmpty(prj.getKeyValueAsJson<ProjectStructure>(ProjectStructureKey));
+  const graph = stagingGraph(structure);
+  const targetBlocks = graph.traverseIds("upstream", rootBlockId);
+
+  const out: UpstreamBlockCtx<PlTreeNodeAccessor>[] = [];
+  for (const blockInfo of allBlocks(structure)) {
+    if (!targetBlocks.has(blockInfo.id)) continue;
+
+    const prod = resolveCtxAccessor(prj, blockInfo.id, "prod");
+    const staging = resolveCtxAccessor(prj, blockInfo.id, "staging");
+
+    const prodIncomplete = prod.calculated && prod.accessor === undefined;
+    const stagingIncomplete = staging.calculated && staging.accessor === undefined;
+
+    out.push({
+      blockId: blockInfo.id,
+      prodCtx: prod.accessor,
+      stagingCtx: staging.accessor,
+      prodIncomplete: prodIncomplete || undefined,
+      stagingIncomplete: stagingIncomplete || undefined,
+    });
+  }
+  return out;
+}
+
+interface CtxProbe {
+  /** True if the ctx-holder field exists in the project tree (block has started rendering). */
+  readonly calculated: boolean;
+  /** Resolved ui-ctx accessor (only when `.ok`). */
+  readonly accessor: PlTreeNodeAccessor | undefined;
+}
+
+function resolveCtxAccessor(
+  prj: PlTreeNodeAccessor,
+  blockId: string,
+  kind: "prod" | "staging",
+): CtxProbe {
+  const ctxField = kind === "prod" ? "prodCtx" : "stagingCtx";
+  const uiCtxField = kind === "prod" ? "prodUiCtx" : "stagingUiCtx";
+
+  const calculated =
+    prj.traverse({
+      field: projectFieldName(blockId, ctxField),
+      ignoreError: true,
+      pureFieldErrorToUndefined: true,
+      stableIfNotFound: true,
+    }) !== undefined;
+
+  const uiCtx = prj.traverseOrError({
+    field: projectFieldName(blockId, uiCtxField),
+    stableIfNotFound: true,
+  });
+
+  if (uiCtx === undefined || !uiCtx.ok) return { calculated, accessor: undefined };
+  return { calculated, accessor: uiCtx.value };
+}

package/src/service_factories.ts

@@ -8,6 +8,7 @@
 
 import { ModelServiceRegistry, Services } from "@milaboratories/pl-model-common";
 import { SpecDriver } from "@milaboratories/pf-spec-driver";
+import { ColumnsCollectionDriverImpl } from "@milaboratories/columns-collection-driver";
 import { type MiLogger } from "@milaboratories/ts-helpers";
 
 export type ModelServiceOptions = {
@@ -19,5 +20,6 @@ export function createModelServiceRegistry(options: ModelServiceOptions) {
     PFrameSpec: () => new SpecDriver({ logger: options.logger }),
     PFrame: null,
     Dialog: null,
+    ColumnsCollection: () => new ColumnsCollectionDriverImpl(),
   });
 }