@milaboratories/pl-middle-layer 1.74.18 → 1.75.0

package/src/model/project_model.ts

@@ -110,6 +110,7 @@ export interface ProjectField {
     | "prodArgs"
     | "currentArgs"
     | "currentPrerunArgs" // Derived args for staging/prerun rendering (from prerunArgs() or args())
+    | "inputsValid" // Model API v2: errors found in currentArgs, none when the block may run, with the fingerprint of those args; present while currentArgs are
     | "prodChainCtx" // Pre-built production context chain node (accumulated prodCtx from all blocks above)
     | "prodCtx"
     | "prodUiCtx"
@@ -132,6 +133,7 @@ export const FieldsToDuplicate: Set<ProjectField["fieldName"]> = new Set([
   "prodArgs",
   "currentArgs",
   "currentPrerunArgs",
+  "inputsValid",
   "prodCtx",
   "prodUiCtx",
   "prodOutput",

package/src/mutator/project-v3.test.ts

@@ -1,7 +1,9 @@
-import { field, poll, TestHelpers, toGlobalResourceId } from "@milaboratories/pl-client";
+import { field, Pl, poll, TestHelpers, toGlobalResourceId } from "@milaboratories/pl-client";
+import { canonicalJsonBytes } from "@milaboratories/ts-helpers";
 import { getQuickJS } from "quickjs-emscripten";
 import { expect, test } from "vitest";
 import { outputRef } from "../model/args";
+import { argsFingerprint } from "../model/args_derivation";
 import { ProjectHelper } from "../model/project_helper";
 import { blockArgsAuthorKey, projectFieldName } from "../model/project_model";
 import { TestBPPreparer } from "../test/block_packs";
@@ -622,3 +624,105 @@ test("v3 blocks: migrateBlockPack does not crash when args() returns undefined",
     });
   });
 });
+
+test("v3 blocks: every args derivation records the errors found in the args, none when there are none", async () => {
+  const quickJs = await getQuickJS();
+
+  await TestHelpers.withTempRoot(async (pl) => {
+    const prj = await pl.withWriteTx("CreatingProject", async (tx) => {
+      const prjRef = await createProject(tx);
+      tx.createField(field(tx.clientRoot, "prj"), "Dynamic", prjRef);
+      await tx.commit();
+      return await toGlobalResourceId(prjRef);
+    });
+
+    await pl.withWriteTx("AddBlock", async (tx) => {
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
+      mut.addBlock(
+        { id: "enter1", label: "Enter Numbers V3", renderingMode: "Heavy" },
+        { storageMode: "fromModel", blockPack: await TestBPPreparer.prepare(BPSpecEnterV3) },
+      );
+      mut.setStates([
+        {
+          modelAPIVersion: 2,
+          blockId: "enter1",
+          payload: { operation: "update-block-data", value: { numbers: [3, 1, 2] } },
+        },
+      ]);
+      mut.save();
+      await tx.commit();
+    });
+
+    await poll(pl, async (tx) => {
+      const prjR = await tx.get(prj);
+      const record = await prjR.get(projectFieldName("enter1", "inputsValid"));
+      expect(jsonOf(record.data.data)).toStrictEqual({
+        argsFingerprint: argsFingerprint({ numbers: [1, 2, 3] }),
+        errors: [],
+      });
+    });
+  });
+});
+
+test("v3 blocks: args without an args errors record of their own are kept and run, as before the record", async () => {
+  const quickJs = await getQuickJS();
+
+  await TestHelpers.withTempRoot(async (pl) => {
+    const prj = await pl.withWriteTx("CreatingProject", async (tx) => {
+      const prjRef = await createProject(tx);
+      tx.createField(field(tx.clientRoot, "prj"), "Dynamic", prjRef);
+      await tx.commit();
+      return await toGlobalResourceId(prjRef);
+    });
+
+    await pl.withWriteTx("AddBlock", async (tx) => {
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
+      mut.addBlock(
+        { id: "enter1", label: "Enter Numbers V3", renderingMode: "Heavy" },
+        { storageMode: "fromModel", blockPack: await TestBPPreparer.prepare(BPSpecEnterV3) },
+      );
+      mut.setStates([
+        {
+          modelAPIVersion: 2,
+          blockId: "enter1",
+          payload: { operation: "update-block-data", value: { numbers: [3, 1, 2] } },
+        },
+      ]);
+      mut.save();
+      await tx.commit();
+    });
+
+    // What a middle layer that does not record args errors leaves behind: current args it wrote
+    // itself, and no record of their args errors.
+    await pl.withWriteTx("WriteArgsWithoutRecord", async (tx) => {
+      tx.removeField(field(prj, projectFieldName("enter1", "inputsValid")));
+      tx.setField(
+        field(prj, projectFieldName("enter1", "currentArgs")),
+        tx.createValue(Pl.JsonObject, canonicalJsonBytes({ numbers: [9] })),
+      );
+      await tx.commit();
+    });
+
+    await pl.withWriteTx("Run", async (tx) => {
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
+      mut.renderProduction(["enter1"]);
+      mut.save();
+      await tx.commit();
+    });
+
+    await poll(pl, async (tx) => {
+      const prjR = await tx.get(prj);
+      const currentArgs = await prjR.get(projectFieldName("enter1", "currentArgs"));
+      expect(jsonOf(currentArgs.data.data)).toStrictEqual({ numbers: [9] });
+      const prodArgs = await prjR.get(projectFieldName("enter1", "prodArgs"));
+      expect(jsonOf(prodArgs.data.data)).toStrictEqual({ numbers: [9] });
+    });
+  });
+});
+
+// Internals
+
+function jsonOf(data: Uint8Array | undefined): unknown {
+  if (data === undefined) throw new Error("The field holds no data");
+  return JSON.parse(Buffer.from(data).toString());
+}

package/src/mutator/project.ts

@@ -77,6 +77,14 @@ import {
 } from "@milaboratories/ts-helpers";
 import { gzipSync } from "node:zlib";
 import type { ProjectHelper } from "../model/project_helper";
+import {
+  derivedArgsValue,
+  argsErrorsFieldChange,
+  argsErrorsOfBlock,
+  type DerivedArgs,
+  type ArgsErrorsFieldChange,
+} from "../model/args_derivation";
+import type { ArgsIssue } from "@milaboratories/pl-model-common";
 import {
   extractConfig,
   UiError,
@@ -273,6 +281,24 @@ class BlockInfo {
     return this.currentPrerunArgsC();
   }
 
+  private readonly argsErrorsC = cached(
+    () => `${this.fields.inputsValid?.modCount}:${this.fields.currentArgs?.modCount}`,
+    () =>
+      argsErrorsOfBlock({
+        modelAPIVersion: this.config.modelAPIVersion,
+        inputsValid: this.fields.inputsValid,
+        currentArgs: this.currentArgs,
+      }),
+  );
+
+  /**
+   * The errors the block's `argsIssues()` found in its current args. A block whose model API
+   * version has no args issues, or that has no current args, has none.
+   */
+  get argsErrors(): readonly ArgsIssue[] {
+    return this.argsErrorsC();
+  }
+
   get stagingRendered(): boolean {
     return this.fields.stagingCtx !== undefined;
   }
@@ -756,6 +782,32 @@ export class ProjectMutator {
     return deleted;
   }
 
+  /**
+   * Brings the block's `inputsValid` field in line with what its args derivation found: the run
+   * errors of its args, none when it may run with them, and no field when it has no args.
+   */
+  private setArgsErrors(blockId: string, derivedArgs: DerivedArgs): void {
+    this.applyArgsErrorsChange(
+      blockId,
+      argsErrorsFieldChange(this.getBlockInfo(blockId).fields.inputsValid, derivedArgs),
+    );
+  }
+
+  private applyArgsErrorsChange(blockId: string, change: ArgsErrorsFieldChange): void {
+    switch (change.type) {
+      case "keep":
+        return;
+      case "delete":
+        this.deleteBlockFields(blockId, "inputsValid");
+        return;
+      case "write": {
+        const ref = this.tx.createValue(Pl.JsonObject, change.data);
+        this.setBlockField(blockId, "inputsValid", ref, "Ready", change.data);
+        return;
+      }
+    }
+  }
+
   private updateLastModified() {
     this.lastModified = Date.now();
     this.lastModifiedChanged = true;
@@ -908,21 +960,16 @@ export class ProjectMutator {
     }
 
     // Derive args from storage - set or clear currentArgs based on derivation result
-    const deriveArgsResult = this.projectHelper.deriveArgsFromStorage(
-      blockConfig,
-      initialStorageJson,
-    );
+    const derivedArgs = this.projectHelper.deriveArgsFromStorage(blockConfig, initialStorageJson);
     // args() legitimately returns undefined when inputs are incomplete/invalid;
     // treat that like a derivation failure and clear the field (never JSON-encode undefined).
-    if (!deriveArgsResult.error && deriveArgsResult.value !== undefined) {
-      this.setBlockFieldObj(
-        blockId,
-        "currentArgs",
-        this.createJsonFieldValue(deriveArgsResult.value),
-      );
+    const args = derivedArgsValue(derivedArgs);
+    if (args !== undefined) {
+      this.setBlockFieldObj(blockId, "currentArgs", this.createJsonFieldValue(args));
     } else {
       this.deleteBlockFields(blockId, "currentArgs");
     }
+    this.setArgsErrors(blockId, derivedArgs);
   }
 
   /** Optimally sets inputs for multiple blocks in one go */
@@ -974,11 +1021,12 @@ export class ProjectMutator {
         );
 
         // Derive args from storage (VM extracts data internally)
-        const derivedArgsResult = this.projectHelper.deriveArgsFromStorage(
+        const derivedArgs = this.projectHelper.deriveArgsFromStorage(
           blockConfig,
           updatedStorageJson,
         );
-        args = derivedArgsResult.error ? undefined : derivedArgsResult.value;
+        args = derivedArgsValue(derivedArgs);
+        this.setArgsErrors(req.blockId, derivedArgs);
       } else {
         this.setBlockFieldObj(req.blockId, "blockStorage", this.createGzJsonFieldValue(req.state));
         if (req.state !== null && typeof req.state === "object" && "args" in req.state) {
@@ -1194,6 +1242,13 @@ export class ProjectMutator {
       throw new Error(`Can't render production for block ${blockId}: currentArgs not set`);
     }
 
+    // Nor if its argsIssues() found errors in them
+    const argsErrors = info.argsErrors;
+    if (argsErrors.length > 0) {
+      const messages = argsErrors.map(({ message }) => message).join("; ");
+      throw new Error(`Can't render production for block ${blockId}: ${messages}`);
+    }
+
     const ctx = this.createProdCtx(this.getPendingProductionGraph().nodes.get(blockId)!.upstream);
 
     if (this.getBlock(blockId).renderingMode === "Light")
@@ -1266,11 +1321,9 @@ export class ProjectMutator {
       prerunArgs = this.projectHelper.derivePrerunArgsFromStorage(blockConfig, storageToWrite);
 
       // Derive args from storage (VM extracts data internally)
-      const deriveArgsResult = this.projectHelper.deriveArgsFromStorage(
-        blockConfig,
-        storageToWrite,
-      );
-      args = deriveArgsResult.error ? undefined : deriveArgsResult.value;
+      const derivedArgs = this.projectHelper.deriveArgsFromStorage(blockConfig, storageToWrite);
+      args = derivedArgsValue(derivedArgs);
+      this.setArgsErrors(blockId, derivedArgs);
     } else if (spec.storageMode === "legacy") {
       // Model API v1: use legacyState from spec
       const parsedState = JSON.parse(spec.legacyState);
@@ -1560,18 +1613,16 @@ export class ProjectMutator {
         this.deleteBlockFields(blockId, "currentPrerunArgs");
       }
 
-      const deriveArgsResult = this.projectHelper.deriveArgsFromStorage(newConfig, storageJson);
+      const derivedArgs = this.projectHelper.deriveArgsFromStorage(newConfig, storageJson);
       // args() legitimately returns undefined when inputs are incomplete/invalid;
       // treat that like a derivation failure and clear the field (never JSON-encode undefined).
-      if (!deriveArgsResult.error && deriveArgsResult.value !== undefined) {
-        this.setBlockFieldObj(
-          blockId,
-          "currentArgs",
-          this.createJsonFieldValue(deriveArgsResult.value),
-        );
+      const args = derivedArgsValue(derivedArgs);
+      if (args !== undefined) {
+        this.setBlockFieldObj(blockId, "currentArgs", this.createJsonFieldValue(args));
       } else {
         this.deleteBlockFields(blockId, "currentArgs");
       }
+      this.setArgsErrors(blockId, derivedArgs);
     };
 
     if (newClearState !== undefined) {
@@ -1587,6 +1638,9 @@ export class ProjectMutator {
       } else {
         // V1: Use setStates with legacy state format
         persistBlockPack();
+        // A V1 model tells whether the block may run by itself; args errors of an earlier V2 model
+        // no longer apply.
+        this.deleteBlockFields(blockId, "inputsValid");
         this.setStates([{ modelAPIVersion: 1, blockId, state: newClearState.state }]);
       }
     } else {
@@ -1617,6 +1671,9 @@ export class ProjectMutator {
       } else {
         // Legacy blocks (modelAPIVersion 1): persist block pack, set prerunArgs = currentArgs
         persistBlockPack();
+        // A V1 model tells whether the block may run by itself; args errors of an earlier V2 model
+        // no longer apply.
+        this.deleteBlockFields(blockId, "inputsValid");
         if (info.fields.currentArgs !== undefined) {
           this.setBlockFieldObj(blockId, "currentPrerunArgs", info.fields.currentArgs);
         }

package/src/test/fake_project.ts

@@ -0,0 +1,222 @@
+import {
+  createSignedResourceId,
+  DefaultFinalResourceDataPredicate,
+  NullSignedResourceId,
+  type FieldData,
+  type FieldType,
+  type SignedResourceId,
+} from "@milaboratories/pl-client";
+import type { PColumnSpec, ArgsIssue } from "@milaboratories/pl-model-common";
+import { PlTreeState, type ExtendedResourceData } from "@milaboratories/pl-tree";
+import { argsFingerprint, type ArgsErrorsRecord } from "../model/args_derivation";
+import {
+  projectFieldName,
+  ProjectStructureKey,
+  type ProjectField,
+  type ProjectStructure,
+} from "../model/project_model";
+
+/** What a project field holds: a resource, an error, or nothing yet. */
+export type FieldContent = SignedResourceId | { error: SignedResourceId } | "unresolved";
+
+/** Columns of a result list by name: the spec of each and the cells of its data. */
+export type Columns = Record<string, { spec: PColumnSpec; cells: string[] }>;
+
+/**
+ * State of a result list: `locked` — complete; `open` — more columns may come; `failed` — it has
+ * an error and will never lock.
+ */
+export type FrameState = { state: "locked" | "open" | "failed" };
+
+/** The block fields a {@link FakeProject} holds. */
+export type FakeProjectField = Extract<
+  ProjectField["fieldName"],
+  | "currentArgs"
+  | "inputsValid"
+  | "prodArgs"
+  | "prodCtx"
+  | "prodUiCtx"
+  | "stagingCtx"
+  | "stagingUiCtx"
+>;
+
+/** A project resource tree built in memory: its blocks, in project order, and their fields. */
+export class FakeProject {
+  private readonly nodes: ExtendedResourceData[] = [];
+  private sent = 0;
+  private readonly blocks = new Map<string, Partial<Record<FakeProjectField, FieldContent>>>();
+  private readonly frameFields = new Map<SignedResourceId, FieldData[]>();
+
+  /** Adds a block below the others, or replaces the fields of one already added. */
+  block(blockId: string, fields: Partial<Record<FakeProjectField, FieldContent>>): void {
+    this.blocks.set(blockId, fields);
+  }
+
+  /** Args of a block. */
+  args(args: unknown): SignedResourceId {
+    return this.value("json/object", args);
+  }
+
+  /** Args errors of a block found in the args `forArgs`, as its `inputsValid` field holds them. */
+  argsErrors(errors: readonly ArgsIssue[], forArgs: unknown): SignedResourceId {
+    const record: ArgsErrorsRecord = { argsFingerprint: argsFingerprint(forArgs), errors };
+    return this.value("json/object", record);
+  }
+
+  /** A ready value resource holding `json`. */
+  value(type: string, json: unknown): SignedResourceId {
+    const id = this.newId();
+    this.nodes.push({
+      ...READY,
+      id,
+      kind: "Value",
+      type: { name: type, version: "1" },
+      data: Buffer.from(JSON.stringify(json)),
+      fields: [],
+    });
+    return id;
+  }
+
+  /** An error a resource or field can hold. */
+  error(message: string): SignedResourceId {
+    return this.value("json/resourceError", { message });
+  }
+
+  /** A result list exporting the given columns, each with JSON data holding `cells`. */
+  frame(columns: Columns, state: FrameState = { state: "locked" }): SignedResourceId {
+    const id = this.newId();
+    this.frameFields.set(id, []);
+    this.extend(id, columns, state);
+    return id;
+  }
+
+  /** Adds columns to a result list made by {@link frame}, and moves it to `state`. */
+  extend(id: SignedResourceId, columns: Columns, { state }: FrameState): void {
+    const fields = this.frameFields.get(id);
+    if (fields === undefined) throw new Error("Not a result list of this project");
+    for (const [name, { spec, cells }] of Object.entries(columns)) {
+      const data = Object.fromEntries(cells.map((cell, i) => [`["s${i}"]`, cell]));
+      fields.push(field("Input", `${name}.spec`, this.value("PColumnSpec", spec)));
+      fields.push(
+        field("Input", `${name}.data`, this.value("PColumnData/Json", { keyLength: 1, data })),
+      );
+    }
+    const error = state === "failed" ? this.error("render failed") : NullSignedResourceId;
+    this.nodes.push({
+      ...READY,
+      id,
+      kind: "Structural",
+      type: { name: "PFrame", version: "1" },
+      inputsLocked: state === "locked",
+      resourceReady: state === "locked",
+      final: state === "locked",
+      error,
+      fields: [...fields],
+    });
+  }
+
+  /** The tree of the project as it is now. */
+  tree(): PlTreeState {
+    const tree = new PlTreeState(ROOT_ID, DefaultFinalResourceDataPredicate);
+    tree.updateFromResourceData(this.changes());
+    return tree;
+  }
+
+  /**
+   * The project resource and the resources added or changed since the last call: a final
+   * resource already in the tree is not sent again.
+   */
+  changes(): ExtendedResourceData[] {
+    const structure: ProjectStructure = {
+      groups: [
+        {
+          id: "main",
+          label: "Main",
+          blocks: [...this.blocks.keys()].map((id) => ({ id, label: id, renderingMode: "Heavy" })),
+        },
+      ],
+    };
+    const fields = [...this.blocks].flatMap(([blockId, contents]) =>
+      PROJECT_FIELDS.flatMap((name) => {
+        const content = contents[name];
+        return content === undefined
+          ? []
+          : [projectField(projectFieldName(blockId, name), content)];
+      }),
+    );
+    const root: ExtendedResourceData = {
+      ...READY,
+      final: false,
+      id: ROOT_ID,
+      kind: "Structural",
+      type: { name: "UserProject", version: "2" },
+      fields,
+      kv: [{ key: ProjectStructureKey, value: Buffer.from(JSON.stringify(structure)) }],
+    };
+    const added = this.nodes.slice(this.sent);
+    this.sent = this.nodes.length;
+    return [root, ...added];
+  }
+
+  private newId(): SignedResourceId {
+    return createSignedResourceId(nextResourceId++);
+  }
+}
+
+// Internals
+
+const ROOT_ID = createSignedResourceId(1_000_001n);
+
+/**
+ * Resource ids stay unique across tests, as they are on a backend: the middle layer caches the
+ * columns an args resource references by its id.
+ */
+let nextResourceId = 100n;
+
+const READY: Omit<ExtendedResourceData, "id" | "kind" | "type" | "fields"> = {
+  originalResourceId: NullSignedResourceId,
+  error: NullSignedResourceId,
+  inputsLocked: true,
+  outputsLocked: true,
+  resourceReady: true,
+  final: true,
+  kv: [],
+};
+
+const PROJECT_FIELDS: readonly FakeProjectField[] = [
+  "currentArgs",
+  "inputsValid",
+  "prodArgs",
+  "prodCtx",
+  "prodUiCtx",
+  "stagingCtx",
+  "stagingUiCtx",
+];
+
+function field(type: FieldType, name: string, value: SignedResourceId): FieldData {
+  return { name, type, value, error: NullSignedResourceId, status: "Resolved", valueIsFinal: true };
+}
+
+function projectField(name: string, content: FieldContent): FieldData {
+  if (content === "unresolved") {
+    return {
+      name,
+      type: "Dynamic",
+      value: NullSignedResourceId,
+      error: NullSignedResourceId,
+      status: "Empty",
+      valueIsFinal: false,
+    };
+  }
+  if (typeof content === "object" && "error" in content) {
+    return {
+      name,
+      type: "Dynamic",
+      value: NullSignedResourceId,
+      error: content.error,
+      status: "Resolved",
+      valueIsFinal: true,
+    };
+  }
+  return field("Dynamic", name, content);
+}