@milaboratories/pl-middle-layer 1.74.18 → 1.75.0

package/src/middle_layer/project.ts

@@ -24,10 +24,15 @@ import { finalityStopRules } from "./finality_stop_rules";
 import type { ComputableStableDefined, ComputableValueOrErrors } from "@milaboratories/computable";
 import { Computable } from "@milaboratories/computable";
 import { projectOverview } from "./project_overview";
+import { blockDependencies, blockDependents, labelBlockColumnLinks } from "./column_links";
 import type { BlockPackSpecAny } from "../model";
 import { randomUUID } from "node:crypto";
 import { withProject, withProjectAuthored } from "../mutator/project";
-import type { ExtendedResourceData, PruningFunction } from "@milaboratories/pl-tree";
+import type {
+  ExtendedResourceData,
+  PlTreeNodeAccessor,
+  PruningFunction,
+} from "@milaboratories/pl-tree";
 import {
   SynchronizedTreeState,
   treeDumpStats,
@@ -38,6 +43,7 @@ import { setTimeout } from "node:timers/promises";
 import { frontendData } from "./frontend_path";
 import type { NavigationState } from "@milaboratories/pl-model-common";
 import { getBlockParameters, blockOutputs } from "./block";
+import { getBlockPackInfo } from "./util";
 import type { FrontendData } from "../model/frontend";
 import type { ProjectId, ProjectStructure } from "../model/project_model";
 import { projectFieldName } from "../model/project_model";
@@ -56,9 +62,10 @@ import type {
   BlockSettings,
   BlockStateInternalV3,
 } from "@milaboratories/pl-model-middle-layer";
+import type { ColumnLink } from "@milaboratories/pl-model-common";
 import { activeConfigs } from "./active_cfg";
 import { NavigationStates } from "./navigation_states";
-import { extractConfig, BLOCK_STORAGE_FACADE_VERSION } from "@platforma-sdk/model";
+import { checkBlockFlag, extractConfig, BLOCK_STORAGE_FACADE_VERSION } from "@platforma-sdk/model";
 import fs from "node:fs/promises";
 import canonicalize from "canonicalize";
 import type { ProjectOverviewLight } from "./project_overview_light";
@@ -68,6 +75,9 @@ import { cacheBlockPackTemplate } from "../mutator/template/template_cache";
 
 type BlockStateComputables = {
   readonly fullState: Computable<BlockStateInternalV3>;
+  /** Column links of the block, nested into its full state. */
+  readonly dependencies: Computable<ColumnLink[] | undefined>;
+  readonly dependents: Computable<ColumnLink[] | undefined>;
 };
 
 function stringifyForDump(object: unknown): string {
@@ -94,6 +104,13 @@ function stringifyForDump(object: unknown): string {
   });
 }
 
+/** Releases what the computables of a block hold. */
+function resetBlockState(computables: BlockStateComputables): void {
+  computables.fullState.resetState();
+  computables.dependencies.resetState();
+  computables.dependents.resetState();
+}
+
 /** Data access object, to manipulate and read single opened (!) project data. */
 export class Project {
   /** Data for the left panel, contain basic information about block status. */
@@ -292,8 +309,8 @@ export class Project {
         // Doing cleanup for deleted blocks
         for (const blockId of this.blockComputables.keys()) {
           if (!existingBlocks.has(blockId)) {
-            const computable = this.blockComputables.get(blockId);
-            if (computable !== undefined && computable !== null) computable.fullState.resetState();
+            const computables = this.blockComputables.get(blockId);
+            if (computables !== undefined && computables !== null) resetBlockState(computables);
             this.blockComputables.set(blockId, null);
           }
         }
@@ -749,13 +766,19 @@ export class Project {
     if (cached === undefined) {
       // state consists of inputs (args + ui state) and outputs
       const outputs = blockOutputs(this.projectTree.entry(), blockId, this.env);
+      const dependencies = blockDependencies(this.projectTree.entry(), blockId, this.env);
+      const dependents = blockDependents(this.projectTree.entry(), blockId, this.env);
       const fullState = Computable.make(
         (ctx) => {
+          // Links are computed only for a block whose UI reads them.
+          const linked = readsColumnLinks(ctx.accessor(this.projectTree.entry()).node(), blockId);
           return {
             parameters: getBlockParameters(this.projectTree.entry(), blockId, ctx),
             outputs,
             navigationState: this.navigationStates.getState(blockId),
             overview: this.overview,
+            dependencies: linked ? dependencies : undefined,
+            dependents: linked ? dependents : undefined,
           };
         },
         {
@@ -770,6 +793,9 @@ export class Project {
               ...v.parameters,
               outputs: newOutputs,
               navigationState: v.navigationState,
+              // The block state already nests the overview, so labelling by its titles adds
+              // nothing to watch.
+              columnLinks: labelBlockColumnLinks({ ...v, blockId }, v.overview?.blocks ?? []),
             } as BlockStateInternalV3;
           },
         },
@@ -777,6 +803,8 @@ export class Project {
 
       const computables: BlockStateComputables = {
         fullState: fullState.withPreCalculatedValueTree(),
+        dependencies,
+        dependents,
       };
 
       this.blockComputables.set(blockId, computables);
@@ -839,7 +867,7 @@ export class Project {
     this.overview.resetState();
     this.blockFrontends.forEach((c) => c.resetState());
     this.blockComputables.forEach((c) => {
-      if (c !== null) c.fullState.resetState();
+      if (c !== null) resetBlockState(c);
     });
     this.activeConfigs.resetState();
   }
@@ -1022,6 +1050,12 @@ export function projectTreeTraverseStopRules(): Filter {
   return finalityStopRules(TreeFinality);
 }
 
+/** Whether the block's model declares that its UI reads the column links of the block state. */
+function readsColumnLinks(prj: PlTreeNodeAccessor, blockId: string): boolean {
+  const info = getBlockPackInfo(prj, blockId);
+  return info !== undefined && checkBlockFlag(info.cfg.featureFlags, "supportsColumnLinks");
+}
+
 /** Returns true if sdk version of the block is old and we need to convert
  * ErrorLike errors to strings like it was.
  * We need it for keeping old blocks and new UI compatibility. */

package/src/middle_layer/project_overview.test.ts

@@ -0,0 +1,200 @@
+import { Computable } from "@milaboratories/computable";
+import type { ArgsIssue } from "@milaboratories/pl-model-common";
+import type { PlTreeState } from "@milaboratories/pl-tree";
+import { describe, expect, test } from "vitest";
+import { projectFieldName } from "../model/project_model";
+import { FakeProject } from "../test/fake_project";
+import { argsAllowRun, type RunStateSource, withRunState } from "./project_overview";
+
+const setThreshold: ArgsIssue = { severity: "error", message: "Set a threshold" };
+
+describe("argsAllowRun", () => {
+  test("a block whose argsIssues found no errors in its current args may run", async () => {
+    const prj = new FakeProject();
+    prj.block("x", { currentArgs: prj.args({ n: 1 }), inputsValid: prj.argsErrors([], { n: 1 }) });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(true);
+  });
+
+  test("a block with current args and no args errors record may run, as before the record", async () => {
+    // What a project saved by an older middle layer holds: current args and no `inputsValid`.
+    const prj = new FakeProject();
+    prj.block("x", { currentArgs: prj.args({ n: 1 }) });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(true);
+  });
+
+  test("a block without current args may not run", async () => {
+    const prj = new FakeProject();
+    prj.block("x", { prodArgs: prj.args({ n: 1 }) });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(false);
+  });
+
+  test("a block whose argsIssues found errors in its current args may not run", async () => {
+    const prj = new FakeProject();
+    prj.block("x", {
+      currentArgs: prj.args({ n: 1 }),
+      inputsValid: prj.argsErrors([setThreshold], { n: 1 }),
+    });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(false);
+  });
+
+  test("errors found in the same args, whatever their key order, still apply", async () => {
+    const prj = new FakeProject();
+    prj.block("x", {
+      currentArgs: prj.args({ a: 1, b: { c: 2, d: 3 } }),
+      inputsValid: prj.argsErrors([setThreshold], { b: { d: 3, c: 2 }, a: 1 }),
+    });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(false);
+  });
+
+  test("errors recorded for other args do not keep the block from running", async () => {
+    // What an older middle layer leaves behind: it writes new current args and never touches
+    // the `inputsValid` field.
+    const prj = new FakeProject();
+    prj.block("x", {
+      currentArgs: prj.args({ n: 2 }),
+      inputsValid: prj.argsErrors([setThreshold], { n: 1 }),
+    });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(true);
+  });
+
+  test("a field that cannot be read does not keep the block from running", async () => {
+    const prj = new FakeProject();
+    prj.block("x", {
+      currentArgs: prj.args({ n: 1 }),
+      inputsValid: prj.value("json/object", [setThreshold]),
+    });
+
+    expect(await allowRun(prj.tree(), "x")).toBe(true);
+  });
+
+  test("may run again once the args and their record have no errors", async () => {
+    const prj = new FakeProject();
+    prj.block("x", {
+      currentArgs: prj.args({ n: 1 }),
+      inputsValid: prj.argsErrors([setThreshold], { n: 1 }),
+    });
+    const tree = prj.tree();
+    const computable = allowRunComputable(tree, "x");
+    expect(await computable.getValue()).toBe(false);
+
+    prj.block("x", { currentArgs: prj.args({ n: 2 }), inputsValid: prj.argsErrors([], { n: 2 }) });
+    tree.updateFromResourceData(prj.changes());
+
+    expect(await computable.getValue()).toBe(true);
+  });
+
+  test("current args changed without their record leave the old errors behind", async () => {
+    const prj = new FakeProject();
+    const argsErrors = prj.argsErrors([setThreshold], { n: 1 });
+    prj.block("x", { currentArgs: prj.args({ n: 1 }), inputsValid: argsErrors });
+    const tree = prj.tree();
+    const computable = allowRunComputable(tree, "x");
+    expect(await computable.getValue()).toBe(false);
+
+    prj.block("x", { currentArgs: prj.args({ n: 2 }), inputsValid: argsErrors });
+    tree.updateFromResourceData(prj.changes());
+
+    expect(await computable.getValue()).toBe(true);
+  });
+});
+
+describe("withRunState", () => {
+  test("a block whose results are up to date lets a changed dependent run despite its own args errors", () => {
+    const [upstream, dependent] = withRunState([
+      block({ id: "a", inputsValid: false }),
+      block({ id: "b", stale: true, upstreams: ["a"] }),
+    ]);
+    expect(upstream.canRun).toBe(false);
+    expect(dependent.canRun).toBe(true);
+  });
+
+  test("a changed block with args errors keeps its dependents from running", () => {
+    const [upstream, dependent, further] = withRunState([
+      block({ id: "a", stale: true, inputsValid: false }),
+      block({ id: "b", stale: true, upstreams: ["a"] }),
+      block({ id: "c", upstreams: ["a", "b"] }),
+    ]);
+    expect(upstream.canRun).toBe(false);
+    expect(dependent.canRun).toBe(false);
+    expect(further).toMatchObject({ stale: true, canRun: false });
+  });
+
+  test("a block with failed outputs and args errors keeps its dependents from running", () => {
+    const [, dependent] = withRunState([
+      block({ id: "a", outputErrors: true, inputsValid: false }),
+      block({ id: "b", stale: true, upstreams: ["a"] }),
+    ]);
+    expect(dependent.canRun).toBe(false);
+  });
+
+  test("a block becomes stale and runnable when an upstream block is stale", () => {
+    const [upstream, dependent] = withRunState([
+      block({ id: "a", stale: true }),
+      block({ id: "b", upstreams: ["a"] }),
+    ]);
+    expect(upstream).toMatchObject({ stale: true, canRun: true });
+    expect(dependent).toMatchObject({ stale: true, canRun: true });
+  });
+
+  test("a block with up-to-date results has nothing to run", () => {
+    const [b] = withRunState([block({ id: "a" })]);
+    expect(b).toMatchObject({ stale: false, canRun: false });
+  });
+
+  test("an up-to-date block whose validity is not about its args keeps its dependents from running", () => {
+    // A block whose code cannot run here, or whose model predates args issues and checks more
+    // than its args, as before args issues.
+    const [upstream, dependent] = withRunState([
+      block({ id: "a", inputsValid: undefined, inputsValidFromArgs: false }),
+      block({ id: "b", stale: true, upstreams: ["a"] }),
+    ]);
+    expect(upstream.canRun).toBe(false);
+    expect(dependent.canRun).toBe(false);
+  });
+
+  test("a changed block that references a missing block cannot run", () => {
+    const [b] = withRunState([block({ id: "a", stale: true, missingReference: true })]);
+    expect(b.canRun).toBe(false);
+  });
+});
+
+// Internals
+
+/** Whether the block may run, read the way the project overview reads it. */
+function allowRunComputable(tree: PlTreeState, blockId: string): Computable<boolean> {
+  const entry = tree.entry();
+  return Computable.make((ctx) => {
+    const prj = ctx.accessor(entry).node();
+    const currentArgs = prj
+      .traverse({
+        field: projectFieldName(blockId, "currentArgs"),
+        assertFieldType: "Dynamic",
+        stableIfNotFound: true,
+      })
+      ?.getDataAsJson();
+    return argsAllowRun(prj, blockId, currentArgs);
+  });
+}
+
+function allowRun(tree: PlTreeState, blockId: string): Promise<boolean> {
+  return allowRunComputable(tree, blockId).getValue();
+}
+
+/** A block with model API version 2 that ran with its current args, without errors, with no upstream blocks. */
+function block(overrides: Partial<RunStateSource> & Pick<RunStateSource, "id">): RunStateSource {
+  return {
+    stale: false,
+    outputErrors: false,
+    inputsValid: true,
+    inputsValidFromArgs: true,
+    missingReference: false,
+    upstreams: [],
+    ...overrides,
+  };
+}

package/src/middle_layer/project_overview.ts

@@ -1,4 +1,4 @@
-import type { PlTreeEntry } from "@milaboratories/pl-tree";
+import type { PlTreeEntry, PlTreeNodeAccessor } from "@milaboratories/pl-tree";
 import type { ComputableStableDefined } from "@milaboratories/computable";
 import { Computable } from "@milaboratories/computable";
 import type { ProjectRenderingState, ProjectStructure } from "../model/project_model";
@@ -31,6 +31,7 @@ import { getBlockPackInfo } from "./util";
 import { resourceIdToString, type SignedResourceId } from "@milaboratories/pl-client";
 import { omitBy, isEqual } from "es-toolkit";
 import { getDebugFlags } from "../debug";
+import { readArgsErrorsField } from "../model/args_derivation";
 
 type BlockInfo = {
   argsRid?: SignedResourceId;
@@ -168,6 +169,7 @@ export function projectOverview(
           subtitle,
           tags,
           inputsValid,
+          inputsValidFromArgs,
           sdkVersion,
           featureFlags,
           isIncompatibleWithRuntime,
@@ -260,10 +262,11 @@ export function projectOverview(
                   }) as ComputableStableDefined<string[]>,
               ),
               // inputsValid: for modelAPIVersion 2, it's true if currentArgs exists (args derivation succeeded)
+              // and the block's argsIssues() is known to have found no errors in them.
               // For older blocks, use the inputsValid callback from config
               inputsValid:
                 cfg.modelAPIVersion === BLOCK_STORAGE_FACADE_VERSION
-                  ? info.currentArguments !== undefined
+                  ? argsAllowRun(prj, id, info.currentArguments)
                   : cfg.inputsValid
                     ? (computableFromCfgOrRF(
                         env,
@@ -280,6 +283,7 @@ export function projectOverview(
                         },
                       }) as ComputableStableDefined<boolean>)
                     : undefined,
+              inputsValidFromArgs: cfg.modelAPIVersion === BLOCK_STORAGE_FACADE_VERSION,
               sdkVersion: codeWithInfo?.sdkVersion,
               featureFlags: codeWithInfo?.featureFlags ?? {},
               isIncompatibleWithRuntime: false,
@@ -317,6 +321,7 @@ export function projectOverview(
           settings,
           sections,
           inputsValid,
+          inputsValidFromArgs: inputsValidFromArgs === true,
           updateInfo: {},
           currentBlockPack: bp?.info?.source,
           updates,
@@ -336,32 +341,14 @@ export function projectOverview(
       };
     },
     {
-      postprocessValue: (value) => {
-        const cantRun = new Set<string>();
-        const staleBlocks = new Set<string>();
-        return {
-          ...value,
-          blocks: value.blocks.map((b) => {
-            if (!b.inputsValid) cantRun.add(b.id);
-            if (b.stale) staleBlocks.add(b.id);
-            const stale = b.stale || b.upstreams.findIndex((u) => staleBlocks.has(u)) !== -1;
-            const canRun =
-              (stale || b.outputErrors) &&
-              Boolean(b.inputsValid) &&
-              !b.missingReference &&
-              b.upstreams.findIndex((u) => cantRun.has(u)) === -1;
-            const bb = {
-              ...b,
-              canRun,
-              stale,
-              updateSuggestions: b.updates?.suggestions ?? [],
-              updatedBlockPack: b.updates?.mainSuggestion,
-            };
-            delete bb["updates"];
-            return bb;
-          }),
-        };
-      },
+      postprocessValue: (value) => ({
+        ...value,
+        blocks: withRunState(value.blocks).map(({ updates, inputsValidFromArgs: _, ...b }) => ({
+          ...b,
+          updateSuggestions: updates?.suggestions ?? [],
+          updatedBlockPack: updates?.mainSuggestion,
+        })),
+      }),
       onRecalculation: getDebugFlags().logProjectOverviewStat
         ? (stats) => {
             console.log(`[projectOverview] ${JSON.stringify(stats)}`);
@@ -370,3 +357,70 @@ export function projectOverview(
     },
   ).withStableType();
 }
+
+/**
+ * Whether the args of a block with model API version 2 let it run: the block has current args,
+ * and its `inputsValid` field records no errors its `argsIssues()` found in them (see
+ * `readArgsErrorsField`).
+ *
+ * @param currentArgs - The block's current args as decoded from its `currentArgs` field;
+ *   `undefined` when it has none
+ */
+export function argsAllowRun(
+  prj: PlTreeNodeAccessor,
+  blockId: string,
+  currentArgs: unknown,
+): boolean {
+  if (currentArgs === undefined) return false;
+  const argsErrors = prj.getField({
+    field: projectFieldName(blockId, "inputsValid"),
+    assertFieldType: "Dynamic",
+    stableIfNotFound: true,
+  });
+  return readArgsErrorsField(argsErrors?.value?.getData(), currentArgs).length === 0;
+}
+
+/** What the run state of a block is derived from. */
+export type RunStateSource = {
+  id: string;
+  /** Whether the block's own args differ from those its results were rendered with. */
+  stale: boolean;
+  outputErrors: boolean;
+  inputsValid: boolean | undefined;
+  /**
+   * Whether `inputsValid` tells only whether the block's current args may run: a block with model
+   * API version 2 this runtime is compatible with. Otherwise it also stands for a block whose code
+   * cannot run here, or for checks of a model that predates args issues.
+   */
+  inputsValidFromArgs: boolean;
+  missingReference: boolean;
+  /** Every block upstream of this one, direct or not. */
+  upstreams: string[];
+};
+
+/**
+ * Adds the run state to each block of a project, the blocks listed upstream first. A block is
+ * stale when its own args changed or an upstream block is stale. It can run when it needs to run
+ * (it is stale or its outputs failed), its args allow it, it references no missing block, and no
+ * upstream block that needs to run is kept from running. An upstream block whose `inputsValid`
+ * comes from its args and whose results are up to date feeds its dependents even when its args
+ * could not run again; any other upstream block that is not valid keeps them from running.
+ */
+export function withRunState<B extends RunStateSource>(
+  blocks: B[],
+): (B & { stale: boolean; canRun: boolean })[] {
+  const cantRun = new Set<string>();
+  const staleBlocks = new Set<string>();
+  return blocks.map((b) => {
+    if (b.stale) staleBlocks.add(b.id);
+    const stale = b.stale || b.upstreams.some((u) => staleBlocks.has(u));
+    const needsRun = stale || b.outputErrors;
+    if (!b.inputsValid && (needsRun || !b.inputsValidFromArgs)) cantRun.add(b.id);
+    const canRun =
+      needsRun &&
+      Boolean(b.inputsValid) &&
+      !b.missingReference &&
+      !b.upstreams.some((u) => cantRun.has(u));
+    return { ...b, stale, canRun };
+  });
+}

package/src/model/args.test.ts

@@ -1,7 +1,12 @@
 import { describe, expect, it, test } from "vitest";
 import canonicalize from "canonicalize";
 import type { ColumnUniversalId } from "@milaboratories/pl-model-common";
-import { canonicalizeJson, createColumnDiscoveredId } from "@milaboratories/pl-model-common";
+import {
+  canonicalizeJson,
+  createColumnDiscoveredId,
+  createColumnValueId,
+  createGlobalPObjectId,
+} from "@milaboratories/pl-model-common";
 import { inferAllReferencedBlocks, outputRef } from "./args";
 
 describe("inferAllReferencedBlocks", () => {
@@ -158,3 +163,36 @@ test("a block id in a map KEY is an upstream too", () => {
 
   expect([...result.upstreams].sort()).toEqual(["A", "B", "D"]);
 });
+
+describe("a column value id names its producer as an upstream", () => {
+  const value = createColumnValueId({
+    column: createGlobalPObjectId("samples", "metadata.condition"),
+    value: "treated",
+  });
+
+  test("as a value", () => {
+    const result = inferAllReferencedBlocks({ target: value });
+
+    expect(result.upstreams).toEqual(new Set(["samples"]));
+    expect(result.missingReferences).toBe(false);
+  });
+
+  test("as a map key", () => {
+    const result = inferAllReferencedBlocks({ ranks: { [value]: 1 } });
+
+    expect(result.upstreams).toEqual(new Set(["samples"]));
+  });
+
+  test("under one extra stringify pass", () => {
+    const result = inferAllReferencedBlocks({ blob: JSON.stringify(value) });
+
+    expect(result.upstreams).toEqual(new Set(["samples"]));
+  });
+
+  test("outside the allowed set it is a missing reference", () => {
+    const result = inferAllReferencedBlocks({ target: value }, new Set(["other"]));
+
+    expect(result.upstreams.size).toBe(0);
+    expect(result.missingReferences).toBe(true);
+  });
+});