@milaboratories/pl-middle-layer 1.74.19 → 1.76.0

package/src/model/args_derivation.test.ts

@@ -0,0 +1,326 @@
+import type { ArgsIssue } from "@milaboratories/pl-model-common";
+import type { BlockConfig } from "@platforma-sdk/model";
+import { BlockStorageFacadeCallbacks } from "@platforma-sdk/model";
+import { canonicalJsonBytes } from "@milaboratories/ts-helpers";
+import { getQuickJS } from "quickjs-emscripten";
+import { beforeAll, describe, expect, test } from "vitest";
+import {
+  argsFingerprint,
+  derivedArgsValue,
+  parseArgsDeriveResult,
+  readArgsErrorsField,
+  argsErrorsFieldChange,
+  argsErrorsOfBlock,
+  type DerivedArgs,
+  type ArgsErrorsRecord,
+} from "./args_derivation";
+import { ProjectHelper } from "./project_helper";
+
+let helper: ProjectHelper;
+
+beforeAll(async () => {
+  helper = new ProjectHelper(await getQuickJS());
+});
+
+const missingThreshold: ArgsIssue = { severity: "error", message: "Set a threshold" };
+const fewSamples: ArgsIssue = { severity: "warning", message: "Only two samples" };
+
+describe("parseArgsDeriveResult", () => {
+  test("reads the results of a model without args issues as before", () => {
+    expect(parseArgsDeriveResult({ value: { n: 1 } })).toEqual({
+      type: "runnable",
+      args: { n: 1 },
+    });
+    expect(parseArgsDeriveResult({ error: "args() threw: no input" })).toEqual({
+      type: "absent",
+      error: new Error("args() threw: no input"),
+    });
+  });
+
+  test("args that are undefined are no args", () => {
+    expect(parseArgsDeriveResult({ value: undefined })).toMatchObject({ type: "absent" });
+    expect(parseArgsDeriveResult({})).toMatchObject({ type: "absent" });
+    expect(
+      parseArgsDeriveResult({ error: "Set a threshold", issues: [missingThreshold] }),
+    ).toMatchObject({ type: "absent" });
+  });
+
+  test("warnings leave the args runnable", () => {
+    expect(parseArgsDeriveResult({ value: { n: 1 }, issues: [fewSamples] })).toEqual({
+      type: "runnable",
+      args: { n: 1 },
+    });
+  });
+
+  test("errors of argsIssues block the args, warnings are left out", () => {
+    expect(
+      parseArgsDeriveResult({
+        error: "Set a threshold",
+        args: { n: 1 },
+        issues: [fewSamples, missingThreshold],
+      }),
+    ).toEqual({ type: "blocked", args: { n: 1 }, errors: [missingThreshold] });
+  });
+
+  test("a severity other than warning is an error", () => {
+    expect(
+      parseArgsDeriveResult({
+        error: "Set a threshold",
+        args: {},
+        issues: [{ severity: "fatal", message: "Set a threshold" }],
+      }),
+    ).toEqual({ type: "blocked", args: {}, errors: [missingThreshold] });
+  });
+
+  test("issues that cannot be read, or hold no error, give way to the reported error", () => {
+    const reported: ArgsIssue = { severity: "error", message: "Set a threshold; Pick a column" };
+    expect(
+      parseArgsDeriveResult({ error: reported.message, args: {}, issues: "not a list" }),
+    ).toEqual({ type: "blocked", args: {}, errors: [reported] });
+    expect(parseArgsDeriveResult({ error: reported.message, args: {} })).toEqual({
+      type: "blocked",
+      args: {},
+      errors: [reported],
+    });
+    expect(
+      parseArgsDeriveResult({ error: reported.message, args: {}, issues: [fewSamples] }),
+    ).toEqual({ type: "blocked", args: {}, errors: [reported] });
+  });
+
+  test("refuses a result that is not an object, or whose error is not a string", () => {
+    expect(() => parseArgsDeriveResult(undefined)).toThrow();
+    expect(() => parseArgsDeriveResult("args")).toThrow();
+    expect(() => parseArgsDeriveResult({ error: 42 })).toThrow();
+  });
+});
+
+describe("derivedArgsValue", () => {
+  test("gives the args of runnable and blocked derivations, nothing for absent ones", () => {
+    expect(derivedArgsValue(runnable({ n: 1 }))).toEqual({ n: 1 });
+    expect(derivedArgsValue(blocked({ n: 2 }, [missingThreshold]))).toEqual({ n: 2 });
+    expect(derivedArgsValue(absent())).toBeUndefined();
+  });
+});
+
+describe("argsFingerprint", () => {
+  test("is the same for args that encode to the same JSON, whatever the key order", () => {
+    expect(argsFingerprint({ a: 1, b: { c: [1, 2], d: "x" } })).toBe(
+      argsFingerprint(JSON.parse('{ "b": { "d": "x", "c": [1, 2] }, "a": 1 }')),
+    );
+    expect(argsFingerprint({ a: 1, skipped: undefined })).toBe(argsFingerprint({ a: 1 }));
+  });
+
+  test("differs for args with other content", () => {
+    expect(argsFingerprint({ a: 1 })).not.toBe(argsFingerprint({ a: 2 }));
+    expect(argsFingerprint({ a: [1, 2] })).not.toBe(argsFingerprint({ a: [2, 1] }));
+  });
+});
+
+describe("argsErrorsFieldChange", () => {
+  test("writes the errors of blocked args, with the fingerprint of the args, into a field not there yet", () => {
+    expect(argsErrorsFieldChange(undefined, blocked({ n: 1 }, [missingThreshold]))).toEqual({
+      type: "write",
+      data: recordBytes({ n: 1 }, [missingThreshold]),
+    });
+  });
+
+  test("writes a record with no errors for args that may run", () => {
+    expect(argsErrorsFieldChange(undefined, runnable({ n: 1 }))).toEqual({
+      type: "write",
+      data: recordBytes({ n: 1 }, []),
+    });
+    const current = { value: recordBytes({ n: 1 }, [missingThreshold]) };
+    expect(argsErrorsFieldChange(current, runnable({ n: 1 }))).toEqual({
+      type: "write",
+      data: recordBytes({ n: 1 }, []),
+    });
+  });
+
+  test("keeps a field that already holds the same errors for the same args", () => {
+    const current = { value: recordBytes({ n: 3 }, [missingThreshold]) };
+    expect(argsErrorsFieldChange(current, blocked({ n: 3 }, [missingThreshold]))).toEqual({
+      type: "keep",
+    });
+    const noErrors = { value: recordBytes({ n: 3 }, []) };
+    expect(argsErrorsFieldChange(noErrors, runnable({ n: 3 }))).toEqual({ type: "keep" });
+  });
+
+  test("rewrites a field holding other errors", () => {
+    const pickColumn: ArgsIssue = { severity: "error", message: "Pick a column" };
+    const current = { value: recordBytes({}, [missingThreshold]) };
+    expect(argsErrorsFieldChange(current, blocked({}, [pickColumn]))).toEqual({
+      type: "write",
+      data: recordBytes({}, [pickColumn]),
+    });
+  });
+
+  test("rewrites a field holding the same errors found in other args", () => {
+    const current = { value: recordBytes({ n: 1 }, [missingThreshold]) };
+    expect(argsErrorsFieldChange(current, blocked({ n: 2 }, [missingThreshold]))).toEqual({
+      type: "write",
+      data: recordBytes({ n: 2 }, [missingThreshold]),
+    });
+  });
+
+  test("deletes the field once there are no args, and never creates it for none", () => {
+    const current = { value: recordBytes({}, [missingThreshold]) };
+    expect(argsErrorsFieldChange(current, absent())).toEqual({ type: "delete" });
+    expect(argsErrorsFieldChange(undefined, absent())).toEqual({ type: "keep" });
+  });
+});
+
+describe("readArgsErrorsField", () => {
+  test("reads the errors the field was written with, for the args they were found in", () => {
+    const change = argsErrorsFieldChange(undefined, blocked({ n: 1 }, [missingThreshold]));
+    if (change.type !== "write") throw new Error(`expected a write, got ${change.type}`);
+    expect(readArgsErrorsField(change.data, { n: 1 })).toEqual([missingThreshold]);
+    // Current args are read back from their field, which need not be canonical JSON.
+    expect(readArgsErrorsField(change.data, JSON.parse('{ "n": 1 }'))).toEqual([missingThreshold]);
+  });
+
+  test("reads a record with no errors as none", () => {
+    expect(readArgsErrorsField(recordBytes({ n: 1 }, []), { n: 1 })).toEqual([]);
+    // Warnings are not errors.
+    expect(readArgsErrorsField(recordBytes({ n: 1 }, [fewSamples]), { n: 1 })).toEqual([]);
+  });
+
+  test("a record written for other args, as a middle layer unaware of the field leaves it, records none", () => {
+    expect(readArgsErrorsField(recordBytes({ n: 1 }, [missingThreshold]), { n: 2 })).toEqual([]);
+  });
+
+  test("no record records none, as for args written before the record existed", () => {
+    expect(readArgsErrorsField(undefined, { n: 1 })).toEqual([]);
+  });
+
+  test("holds no errors while the block has no current args", () => {
+    expect(readArgsErrorsField(recordBytes({ n: 1 }, [missingThreshold]), undefined)).toEqual([]);
+    expect(readArgsErrorsField(undefined, undefined)).toEqual([]);
+  });
+
+  test("a field that cannot be read records none", () => {
+    const unreadable = [
+      Buffer.from("{"),
+      Buffer.from("true"),
+      canonicalJsonBytes([missingThreshold]),
+      canonicalJsonBytes({ errors: [missingThreshold] }),
+      canonicalJsonBytes({ argsFingerprint: argsFingerprint({ n: 1 }) }),
+    ];
+    for (const data of unreadable) {
+      expect(readArgsErrorsField(data, { n: 1 })).toEqual([]);
+    }
+  });
+});
+
+describe("argsErrorsOfBlock", () => {
+  const inputsValid = { value: recordBytes({ n: 1 }, [missingThreshold]) };
+
+  test("gives the errors of a block with model API version 2 for its current args", () => {
+    expect(argsErrorsOfBlock({ modelAPIVersion: 2, inputsValid, currentArgs: { n: 1 } })).toEqual([
+      missingThreshold,
+    ]);
+  });
+
+  test("a block whose record was written for other args, or that has none, has none", () => {
+    expect(argsErrorsOfBlock({ modelAPIVersion: 2, inputsValid, currentArgs: { n: 2 } })).toEqual(
+      [],
+    );
+    expect(
+      argsErrorsOfBlock({ modelAPIVersion: 2, inputsValid: undefined, currentArgs: { n: 1 } }),
+    ).toEqual([]);
+  });
+
+  test("ignores a field left over for a block whose model is not on model API version 2", () => {
+    expect(argsErrorsOfBlock({ modelAPIVersion: 1, inputsValid, currentArgs: { n: 1 } })).toEqual(
+      [],
+    );
+    expect(
+      argsErrorsOfBlock({ modelAPIVersion: undefined, inputsValid, currentArgs: { n: 1 } }),
+    ).toEqual([]);
+  });
+});
+
+describe("ProjectHelper.deriveArgsFromStorage", () => {
+  test("a model without args issues whose args() throws has no args and no args errors", () => {
+    // What the SDK's __pl_args_derive returned before args issues, and still returns for a
+    // block that declares no argsIssues.
+    const derived = helper.deriveArgsFromStorage(
+      configWith('() => ({ error: "args() threw: Select a dataset" })'),
+      "{}",
+    );
+
+    expect(derived).toEqual({ type: "absent", error: new Error("args() threw: Select a dataset") });
+    expect(argsErrorsFieldChange(undefined, derived)).toEqual({ type: "keep" });
+  });
+
+  test("a model without args issues whose args() succeeds has runnable args", () => {
+    const derived = helper.deriveArgsFromStorage(configWith("() => ({ value: { n: 1 } })"), "{}");
+
+    expect(derived).toEqual({ type: "runnable", args: { n: 1 } });
+    expect(argsErrorsFieldChange(undefined, derived)).toEqual({
+      type: "write",
+      data: recordBytes({ n: 1 }, []),
+    });
+  });
+
+  test("argsIssues that found errors block the args they were given", () => {
+    const derived = helper.deriveArgsFromStorage(
+      configWith(
+        "(storageJson) => ({ error: 'Set a threshold', args: JSON.parse(storageJson), " +
+          "issues: [{ severity: 'warning', message: 'Only two samples' }, " +
+          "{ severity: 'error', message: 'Set a threshold' }] })",
+      ),
+      JSON.stringify({ n: 5 }),
+    );
+
+    expect(derived).toEqual({ type: "blocked", args: { n: 5 }, errors: [missingThreshold] });
+  });
+
+  test("a model that fails, or returns something unreadable, gives no args", () => {
+    expect(
+      helper.deriveArgsFromStorage(configWith("() => { throw new Error('boom'); }"), "{}"),
+    ).toMatchObject({ type: "absent" });
+    expect(helper.deriveArgsFromStorage(configWith("() => 42"), "{}")).toMatchObject({
+      type: "absent",
+    });
+  });
+});
+
+// Internals
+
+function runnable(args: unknown): DerivedArgs {
+  return { type: "runnable", args };
+}
+
+function blocked(args: unknown, errors: ArgsIssue[]): DerivedArgs {
+  return { type: "blocked", args, errors };
+}
+
+function recordBytes(args: unknown, errors: ArgsIssue[]): Uint8Array {
+  const record: ArgsErrorsRecord = { argsFingerprint: argsFingerprint(args), errors };
+  return canonicalJsonBytes(record);
+}
+
+function absent(): DerivedArgs {
+  return { type: "absent", error: new Error("args() threw: no input") };
+}
+
+/**
+ * A block config whose model code registers `body` as its __pl_args_derive callback, the way the
+ * SDK registers callbacks inside a real bundle.
+ */
+function configWith(body: string): BlockConfig {
+  const handle = BlockStorageFacadeCallbacks.ArgsDerive;
+  return {
+    configVersion: 4,
+    modelAPIVersion: 2,
+    sdkVersion: "1.0.0",
+    renderingMode: "Heavy",
+    sections: { __renderLambda: true, handle: "sections" },
+    outputs: {},
+    code: {
+      type: "plain",
+      content: `globalThis.cfgRenderCtx.callbackRegistry[${JSON.stringify(handle)}] = ${body};`,
+    },
+    blockLifecycleCallbacks: { [handle]: { __renderLambda: true, handle } },
+  };
+}

package/src/model/args_derivation.ts

@@ -0,0 +1,194 @@
+import { parseArgsIssues, type ArgsIssue } from "@milaboratories/pl-model-common";
+import { cachedDeserialize, canonicalJsonBytes } from "@milaboratories/ts-helpers";
+import { isString } from "es-toolkit";
+import { createHash } from "node:crypto";
+import { BLOCK_STORAGE_FACADE_VERSION } from "@platforma-sdk/model";
+
+/**
+ * What the args derivation of a block with model API version 2 made of the block's storage.
+ *
+ * - `runnable`: the block has args and may run with them.
+ * - `blocked`: the block has args, but its `argsIssues()` found errors in them, so it must not run.
+ *   `errors` is never empty.
+ * - `absent`: the block has no args, because `args()` threw or returned nothing.
+ */
+export type DerivedArgs =
+  | { readonly type: "runnable"; readonly args: unknown }
+  | { readonly type: "blocked"; readonly args: unknown; readonly errors: readonly ArgsIssue[] }
+  | { readonly type: "absent"; readonly error: Error };
+
+/**
+ * What a block's `inputsValid` field holds: the errors found in the block's current args, and the
+ * fingerprint of the args they were found in. Every args derivation that produces args writes
+ * it, with no errors when the block may run.
+ *
+ * A middle layer that predates args issues never writes or deletes the field, so after it edits
+ * the block, the field can be missing or outlive the args it was computed for. The fingerprint
+ * tells such a stale field apart: it no longer matches the current args, and says nothing of them.
+ */
+export type ArgsErrorsRecord = {
+  /** {@link argsFingerprint} of the current args the errors were found in. */
+  readonly argsFingerprint: string;
+  /** The errors; empty when the block may run with the args. */
+  readonly errors: readonly ArgsIssue[];
+};
+
+/** How a block's `inputsValid` field must change to hold what an args derivation found. */
+export type ArgsErrorsFieldChange =
+  | { readonly type: "keep" }
+  | { readonly type: "write"; readonly data: Uint8Array }
+  | { readonly type: "delete" };
+
+/**
+ * Reads the result of the model's `__pl_args_derive` callback.
+ *
+ * A model built before args issues returns `{ value }` for args and `{ error }` for none; both
+ * read as they did then. A newer model adds the args issues, and returns `{ error, args, issues }`
+ * when its `argsIssues()` found errors. Args that are `undefined` are no args, whatever
+ * `argsIssues()` said.
+ *
+ * @throws when `value` is not an object, or carries an `error` that is not a string
+ */
+export function parseArgsDeriveResult(value: unknown): DerivedArgs {
+  if (typeof value !== "object" || value === null) {
+    throw new Error(`Args derivation returned ${value === null ? "null" : typeof value}`);
+  }
+
+  const error = "error" in value ? value.error : undefined;
+  if (error !== undefined) {
+    if (!isString(error)) throw new Error(`Args derivation error is ${typeof error}, not a string`);
+    const args = "args" in value ? value.args : undefined;
+    if (args === undefined) return { type: "absent", error: new Error(error) };
+    const issues = "issues" in value ? value.issues : undefined;
+    return { type: "blocked", args, errors: argsErrorsOf(issues, error) };
+  }
+
+  const args = "value" in value ? value.value : undefined;
+  if (args === undefined) return { type: "absent", error: new Error("args() returned no args") };
+  return { type: "runnable", args };
+}
+
+/** The args an args derivation produced; `undefined` when it produced none. */
+export function derivedArgsValue(derived: DerivedArgs): unknown {
+  return derived.type === "absent" ? undefined : derived.args;
+}
+
+/**
+ * Identifies a value of a block's current args by its content: the SHA-256 of its canonical JSON,
+ * in hex. Args that encode to the same JSON, whatever the key order, share a fingerprint.
+ */
+export function argsFingerprint(args: unknown): string {
+  return createHash("sha256").update(canonicalJsonBytes(args)).digest("hex");
+}
+
+/**
+ * What a block's `inputsValid` field must become after an args derivation, given the field as it
+ * is now.
+ *
+ * The field holds a {@link ArgsErrorsRecord} as canonical JSON, and exists while the block has
+ * args: with the errors its `argsIssues()` found in them, and with none when it found none. A
+ * field already holding the same record is kept as it is, so that readers of the project are not
+ * woken by a write that changes nothing.
+ */
+export function argsErrorsFieldChange(
+  current: { readonly value?: Uint8Array } | undefined,
+  derived: DerivedArgs,
+): ArgsErrorsFieldChange {
+  if (derived.type === "absent")
+    return current === undefined ? { type: "keep" } : { type: "delete" };
+  const record: ArgsErrorsRecord = {
+    argsFingerprint: argsFingerprint(derived.args),
+    errors: derived.type === "blocked" ? derived.errors : [],
+  };
+  const data = canonicalJsonBytes(record);
+  if (current?.value !== undefined && Buffer.compare(current.value, data) === 0) {
+    return { type: "keep" };
+  }
+  return { type: "write", data };
+}
+
+/**
+ * The errors a block's `inputsValid` field, as {@link argsErrorsFieldChange} writes it, records
+ * for the block's current args.
+ *
+ * A field that is missing, cannot be read, or was written for other args records none: the args
+ * were written by a middle layer that does not know the field. Such a middle layer refuses every
+ * block whose `argsIssues()` could find errors, so it wrote args only for blocks whose `args()`
+ * throws instead, and those args may run, as they always could. No current args hold no errors:
+ * without args the block may not run anyway.
+ *
+ * @param data - Content of the field; `undefined` when there is none, or while it is not loaded
+ * @param currentArgs - The block's current args as decoded from its `currentArgs` field;
+ *   `undefined` when the block has none
+ */
+export function readArgsErrorsField(
+  data: Uint8Array | undefined,
+  currentArgs: unknown,
+): readonly ArgsIssue[] {
+  if (currentArgs === undefined || data === undefined) return [];
+  let record: ArgsErrorsRecord;
+  try {
+    record = parseArgsErrorsRecord(cachedDeserialize(data));
+  } catch {
+    return [];
+  }
+  return record.argsFingerprint === argsFingerprint(currentArgs) ? record.errors : [];
+}
+
+/**
+ * The errors found in a block's current args, read from its `inputsValid` field as
+ * {@link readArgsErrorsField} reads it.
+ *
+ * Only a block with model API version 2 has args errors. An earlier model tells whether the block
+ * may run by itself, and a field left over from a later model version does not apply to it: such
+ * a block has none.
+ */
+export function argsErrorsOfBlock(block: {
+  /** Model API version of the block's current block pack. */
+  readonly modelAPIVersion: number | undefined;
+  /** The block's `inputsValid` field; `undefined` when there is none. */
+  readonly inputsValid: { readonly value?: Uint8Array } | undefined;
+  /** The block's current args; `undefined` when it has none. */
+  readonly currentArgs: unknown;
+}): readonly ArgsIssue[] {
+  const { modelAPIVersion, inputsValid, currentArgs } = block;
+  if (modelAPIVersion !== BLOCK_STORAGE_FACADE_VERSION) return [];
+  return readArgsErrorsField(inputsValid?.value, currentArgs);
+}
+
+// Internals
+
+/**
+ * The errors among the issues of a derivation whose `argsIssues()` found errors. Issues that
+ * cannot be read, or that hold no error, give way to `error`, the joined messages the model
+ * reported.
+ */
+function argsErrorsOf(issues: unknown, error: string): readonly ArgsIssue[] {
+  try {
+    const errors = parseArgsIssues(issues).filter(isError);
+    if (errors.length > 0) return errors;
+  } catch {
+    // An older or broken model: `error` alone tells why the block must not run.
+  }
+  return [{ severity: "error", message: error }];
+}
+
+/**
+ * Reads a {@link ArgsErrorsRecord} from decoded field content. Issues other than errors are
+ * dropped.
+ *
+ * @throws when `value` is not an object carrying a string fingerprint and a list of issues
+ */
+function parseArgsErrorsRecord(value: unknown): ArgsErrorsRecord {
+  if (typeof value !== "object" || value === null) {
+    throw new Error("Args errors record must be an object");
+  }
+  const fingerprint = "argsFingerprint" in value ? value.argsFingerprint : undefined;
+  if (!isString(fingerprint)) throw new Error("Args errors record must carry a fingerprint");
+  const errors = parseArgsIssues("errors" in value ? value.errors : undefined).filter(isError);
+  return { argsFingerprint: fingerprint, errors };
+}
+
+function isError(issue: ArgsIssue): boolean {
+  return issue.severity === "error";
+}

package/src/model/project_helper.ts

@@ -1,5 +1,6 @@
 import type { ResultOrError, BlockConfig, BlockStorage, PlRef } from "@platforma-sdk/model";
-import type { StringifiedJson } from "@milaboratories/pl-model-common";
+import type { ColumnReference, StringifiedJson } from "@milaboratories/pl-model-common";
+import { collectColumnReferences } from "@milaboratories/pl-model-common";
 import {
   extractCodeWithInfo,
   ensureError,
@@ -12,6 +13,7 @@ import { executeSingleLambda } from "../js_render";
 import type { SignedResourceId } from "@milaboratories/pl-client";
 import { ConsoleLoggerAdapter, type MiLogger } from "@milaboratories/ts-helpers";
 import { getDebugFlags } from "../debug";
+import { parseArgsDeriveResult, type DerivedArgs } from "./args_derivation";
 
 type EnrichmentTargetsRequest = {
   blockConfig: () => BlockConfig;
@@ -34,10 +36,11 @@ export type MigrationResult =
   | { error?: undefined; newStorageJson: StringifiedJson<BlockStorage>; info: string };
 
 /**
- * Result of args derivation from storage.
- * Returned by __pl_args_derive and __pl_prerunArgs_derive VM callbacks.
+ * Result of a derivation from storage that either produced a value or failed.
+ * Returned by the __pl_prerunArgs_derive and __pl_initializationParams_derive VM callbacks; a
+ * newer model adds the issues found in the prerun args, which are not read here.
  */
-type ArgsDeriveResult = { error: string } | { error?: undefined; value: unknown };
+type DeriveResult = { error: string } | { error?: undefined; value: unknown };
 
 /**
  * Result of building initial storage from params.
@@ -65,6 +68,12 @@ export class ProjectHelper {
     },
   });
 
+  // An args resource never changes, so its id alone identifies the columns found in it.
+  private readonly columnReferencesCache = new LRUCache<
+    SignedResourceId,
+    readonly ColumnReference[]
+  >({ max: 1024 });
+
   constructor(
     private readonly quickJs: QuickJSWASMModule,
     public readonly logger: MiLogger = new ConsoleLoggerAdapter(),
@@ -76,39 +85,37 @@ export class ProjectHelper {
 
   /**
    * Derives args directly from storage JSON using VM callback.
-   * The VM extracts data from storage and calls the block's args() function.
+   * The VM extracts data from storage, calls the block's args() function and then its argsIssues().
    *
    * This allows the middle layer to work only with storage JSON,
    * without needing to know the underlying data structure.
    *
    * @param blockConfig The block configuration (provides the model code)
    * @param storageJson Storage as JSON string
-   * @returns The derived args object, or error if derivation fails
+   * @returns The derived args and whether the block may run with them, or why there are none
    */
-  public deriveArgsFromStorage(
-    blockConfig: BlockConfig,
-    storageJson: string,
-  ): ResultOrError<unknown> {
+  public deriveArgsFromStorage(blockConfig: BlockConfig, storageJson: string): DerivedArgs {
     if (blockConfig.modelAPIVersion !== BLOCK_STORAGE_FACADE_VERSION) {
       return {
+        type: "absent",
         error: new Error("deriveArgsFromStorage is only supported for model API version 2"),
       };
     }
 
     try {
-      const result = executeSingleLambda(
-        this.quickJs,
-        blockConfig.blockLifecycleCallbacks[BlockStorageFacadeCallbacks.ArgsDerive],
-        extractCodeWithInfo(blockConfig),
-        storageJson,
-      ) as ArgsDeriveResult;
-
-      if (result.error !== undefined) {
-        return { error: new Error(result.error) };
-      }
-      return { value: result.value };
+      return parseArgsDeriveResult(
+        executeSingleLambda(
+          this.quickJs,
+          blockConfig.blockLifecycleCallbacks[BlockStorageFacadeCallbacks.ArgsDerive],
+          extractCodeWithInfo(blockConfig),
+          storageJson,
+        ),
+      );
     } catch (e) {
-      return { error: new Error("Args derivation from storage failed", { cause: ensureError(e) }) };
+      return {
+        type: "absent",
+        error: new Error("Args derivation from storage failed", { cause: ensureError(e) }),
+      };
     }
   }
 
@@ -116,9 +123,13 @@ export class ProjectHelper {
    * Derives prerunArgs directly from storage JSON using VM callback.
    * Falls back to args() if prerunArgs is not defined in the block model.
    *
+   * Errors the block's `prerunArgsIssues()` finds in its prerun args, or for a block without its
+   * own prerunArgs, the errors its `argsIssues()` finds in its args, come back as an `error` and
+   * skip staging the same way.
+   *
    * @param blockConfig The block configuration (provides the model code)
    * @param storageJson Storage as JSON string
-   * @returns The derived prerunArgs, or undefined if derivation fails
+   * @returns The derived prerunArgs, or undefined if derivation fails or found errors
    */
   public derivePrerunArgsFromStorage(blockConfig: BlockConfig, storageJson: string): unknown {
     if (blockConfig.modelAPIVersion !== BLOCK_STORAGE_FACADE_VERSION) {
@@ -131,7 +142,7 @@ export class ProjectHelper {
         blockConfig.blockLifecycleCallbacks[BlockStorageFacadeCallbacks.PrerunArgsDerive],
         extractCodeWithInfo(blockConfig),
         storageJson,
-      ) as ArgsDeriveResult;
+      ) as DeriveResult;
 
       if (result.error !== undefined) {
         // Return undefined if derivation fails (skip block in staging)
@@ -207,7 +218,7 @@ export class ProjectHelper {
         callback,
         extractCodeWithInfo(blockConfig),
         storageJson,
-      ) as ArgsDeriveResult;
+      ) as DeriveResult;
 
       if (result.error !== undefined) {
         return { error: new Error(result.error) };
@@ -250,6 +261,25 @@ export class ProjectHelper {
     return this.enrichmentTargetsCache.memo(cacheKey, { context: req }).value;
   }
 
+  /**
+   * Columns referenced by a block's args, with the values of them the args picked, read from the
+   * args resource `argsRid`. `undefined` while the resource data is not loaded yet: `args`
+   * returns `undefined` then, and nothing is remembered for such a read, so the references are
+   * found once the data arrives.
+   */
+  public getColumnReferences(
+    argsRid: SignedResourceId,
+    args: () => unknown,
+  ): undefined | readonly ColumnReference[] {
+    const cached = this.columnReferencesCache.get(argsRid);
+    if (cached !== undefined) return cached;
+    const data = args();
+    if (data === undefined) return undefined;
+    const references = collectColumnReferences(data);
+    this.columnReferencesCache.set(argsRid, references);
+    return references;
+  }
+
   // =============================================================================
   // VM-based Storage Operations
   // =============================================================================