@milaboratories/pl-middle-layer 1.75.0 → 1.76.1

package/src/middle_layer/project.ts

@@ -43,6 +43,8 @@ import { setTimeout } from "node:timers/promises";
 import { frontendData } from "./frontend_path";
 import type { NavigationState } from "@milaboratories/pl-model-common";
 import { getBlockParameters, blockOutputs } from "./block";
+import type { BlockActionContextMode, BlockActionContextNode } from "./block_action_context";
+import { readBlockActionContextTree } from "./block_action_context";
 import { getBlockPackInfo } from "./util";
 import type { FrontendData } from "../model/frontend";
 import type { ProjectId, ProjectStructure } from "../model/project_model";
@@ -818,6 +820,17 @@ export class Project {
    * Returns a computable, that can be used to retrieve and watch full block state,
    * including outputs, arguments, ui state.
    * */
+  /**
+   * Reads the block action context tree of a block render (see readBlockActionContextTree): current state, not
+   * watched. Undefined when the block has no action context.
+   */
+  public async getBlockActionContext(
+    blockId: string,
+    mode: BlockActionContextMode = "prod",
+  ): Promise<BlockActionContextNode | undefined> {
+    return await readBlockActionContextTree(this.env.pl, this.rid, blockId, mode);
+  }
+
   public getBlockState(blockId: string): Computable<BlockStateInternalV3> {
     return this.getBlockComputables(blockId).fullState;
   }

package/src/model/project_model.ts

@@ -123,7 +123,10 @@ export interface ProjectField {
     | "stagingOutput"
     | "stagingCtxPrevious"
     | "stagingUiCtxPrevious"
-    | "stagingOutputPrevious";
+    | "stagingOutputPrevious"
+    // Root action context of the current render. The project holds it so it outlives the render.
+    | "prodActionContext"
+    | "stagingActionContext";
 }
 
 export const FieldsToDuplicate: Set<ProjectField["fieldName"]> = new Set([
@@ -144,7 +147,7 @@ export function projectFieldName(blockId: string, fieldName: ProjectField["field
 }
 
 const projectFieldPattern =
-  /^(?<blockId>.*)-(?<fieldName>blockPack|blockSettings|blockStorage|inputsValid|prodArgs|currentArgs|currentPrerunArgs|prodChainCtx|prodCtx|prodUiCtx|prodOutput|prodCtxPrevious|prodUiCtxPrevious|prodOutputPrevious|stagingCtx|stagingUiCtx|stagingOutput|stagingCtxPrevious|stagingUiCtxPrevious|stagingOutputPrevious)$/;
+  /^(?<blockId>.*)-(?<fieldName>blockPack|blockSettings|blockStorage|inputsValid|prodArgs|currentArgs|currentPrerunArgs|prodChainCtx|prodCtx|prodUiCtx|prodOutput|prodCtxPrevious|prodUiCtxPrevious|prodOutputPrevious|stagingCtx|stagingUiCtx|stagingOutput|stagingCtxPrevious|stagingUiCtxPrevious|stagingOutputPrevious|prodActionContext|stagingActionContext)$/;
 
 export function parseProjectField(name: string): ProjectField | undefined {
   const match = name.match(projectFieldPattern);

package/src/mutator/project.ts

@@ -22,6 +22,7 @@ import {
 } from "@milaboratories/pl-client";
 import {
   createRenderHeavyBlock,
+  type BlockActionContextBaggage,
   createBContextFromUpstreams,
   createBContextEnd,
 } from "./template/render_block";
@@ -732,6 +733,15 @@ export class ProjectMutator {
     return { ref, value: jsonBytes, status: "Ready" };
   }
 
+  /** Baggage of the root action context of a block render. */
+  private blockActionContextBaggage(blockId: string, info: BlockInfo): BlockActionContextBaggage {
+    return {
+      name: this.getBlock(blockId).label,
+      blockPack: JSON.stringify(info.source),
+      blockId,
+    };
+  }
+
   private getBlock(blockId: string): Block {
     for (const block of allBlocks(this.struct)) if (block.id === blockId) return block;
     throw new Error("block not found");
@@ -834,6 +844,7 @@ export class ProjectMutator {
       this.setBlockFieldObj(blockId, "stagingCtxPrevious", fields.stagingCtx!);
       this.setBlockFieldObj(blockId, "stagingUiCtxPrevious", fields.stagingUiCtx!);
     }
+    this.deleteBlockFields(blockId, "stagingActionContext");
     if (this.deleteBlockFields(blockId, "stagingOutput", "stagingCtx", "stagingUiCtx"))
       this.resetStagingRefreshTimestamp();
   }
@@ -849,7 +860,14 @@ export class ProjectMutator {
       this.setBlockFieldObj(blockId, "prodCtxPrevious", fields.prodCtx!);
       this.setBlockFieldObj(blockId, "prodUiCtxPrevious", fields.prodUiCtx!);
     }
-    this.deleteBlockFields(blockId, "prodOutput", "prodCtx", "prodUiCtx", "prodArgs");
+    this.deleteBlockFields(
+      blockId,
+      "prodOutput",
+      "prodCtx",
+      "prodUiCtx",
+      "prodArgs",
+      "prodActionContext",
+    );
   }
 
   /** Running blocks are reset, settled ones (Ready or finished Error) moved to limbo. Returns if
@@ -878,6 +896,7 @@ export class ProjectMutator {
       return true;
     } else {
       // reset - clean up any partial/inconsistent production stat
+      this.deleteBlockFields(blockId, "prodActionContext");
       return this.deleteBlockFields(
         blockId,
         "prodOutput",
@@ -1214,12 +1233,17 @@ export class ProjectMutator {
     const tpl = info.getTemplate(this.tx);
 
     // Use currentPrerunArgs for staging rendering
-    const results = createRenderHeavyBlock(this.tx, tpl, {
-      args: prerunArgsRef,
-      blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
-      isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(false)),
-      context: ctx,
-    });
+    const results = createRenderHeavyBlock(
+      this.tx,
+      tpl,
+      {
+        args: prerunArgsRef,
+        blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
+        isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(false)),
+        context: ctx,
+      },
+      this.blockActionContextBaggage(blockId, info),
+    );
 
     // Here we set the staging ctx to the input context of the staging workflow, not the output because exports
     // of one staging context should stay within the same block, and not travel downstream.
@@ -1230,6 +1254,8 @@ export class ProjectMutator {
     // thus creating a certain discrepancy between staging workflow context behavior and desktop's result pool.
     this.setBlockField(blockId, "stagingUiCtx", this.exportCtx(results.context), "NotReady");
     this.setBlockField(blockId, "stagingOutput", results.result, "NotReady");
+    if (results.actionContext !== undefined)
+      this.setBlockField(blockId, "stagingActionContext", results.actionContext, "NotReady");
   }
 
   private renderProductionFor(blockId: string) {
@@ -1256,12 +1282,17 @@ export class ProjectMutator {
 
     const tpl = info.getTemplate(this.tx);
 
-    const results = createRenderHeavyBlock(this.tx, tpl, {
-      args: info.fields.currentArgs.ref!,
-      blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
-      isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(true)),
-      context: ctx,
-    });
+    const results = createRenderHeavyBlock(
+      this.tx,
+      tpl,
+      {
+        args: info.fields.currentArgs.ref!,
+        blockId: this.tx.createValue(Pl.JsonString, JSON.stringify(blockId)),
+        isProduction: this.tx.createValue(Pl.JsonBool, JSON.stringify(true)),
+        context: ctx,
+      },
+      this.blockActionContextBaggage(blockId, info),
+    );
     this.setBlockField(
       blockId,
       "prodCtx",
@@ -1270,6 +1301,8 @@ export class ProjectMutator {
     );
     this.setBlockField(blockId, "prodUiCtx", this.exportCtx(results.context), "NotReady");
     this.setBlockField(blockId, "prodOutput", results.result, "NotReady");
+    if (results.actionContext !== undefined)
+      this.setBlockField(blockId, "prodActionContext", results.actionContext, "NotReady");
 
     // saving inputs for which we rendered the production
     this.setBlockFieldObj(blockId, "prodArgs", info.fields.currentArgs);
@@ -1794,6 +1827,7 @@ export class ProjectMutator {
         // skipping finished blocks
         continue;
 
+      this.deleteBlockFields(blockId, "prodActionContext");
       if (this.deleteBlockFields(blockId, "prodOutput", "prodCtx", "prodUiCtx", "prodArgs")) {
         // was actually stopped
         stopped.push(blockId);

package/src/mutator/project_action_context.test.ts

@@ -0,0 +1,178 @@
+import type { PlClient, SignedResourceId } from "@milaboratories/pl-client";
+import { ActionContextLinkCapability } from "./template/render_block";
+import {
+  ContinuePolling,
+  field,
+  isNullSignedResourceId,
+  parseSignedResourceId,
+  poll,
+  TestHelpers,
+  toGlobalResourceId,
+} from "@milaboratories/pl-client";
+import { getQuickJS } from "quickjs-emscripten";
+import { expect, test } from "vitest";
+import { ProjectHelper } from "../model/project_helper";
+import { projectFieldName } from "../model/project_model";
+import { BPSpecEnterV041NotPrepared, TestBPPreparer } from "../test/block_packs";
+import { readBlockActionContextTree } from "../middle_layer/block_action_context";
+import { createProject, ProjectMutator } from "./project";
+
+type ActionContextBaggage = Record<string, string>;
+
+/** Reads the action context that prodActionContext points to, or undefined if the field is absent.
+ * Polls until the field reference is resolved: it resolves asynchronously after the commit. */
+async function readActionContext(
+  pl: PlClient,
+  prj: Awaited<ReturnType<typeof toGlobalResourceId>>,
+  blockId: string,
+): Promise<
+  { contextId: string; contextRef: SignedResourceId; baggage: ActionContextBaggage } | undefined
+> {
+  return await poll(pl, async ({ tx }) => {
+    const project = await tx.getResourceData(prj, true);
+    const actionContextField = project.fields.find(
+      (f) => f.name === projectFieldName(blockId, "prodActionContext"),
+    );
+    if (actionContextField === undefined) return undefined;
+
+    const contextRef = actionContextField.value;
+    if (isNullSignedResourceId(contextRef)) throw new ContinuePolling();
+    const context = await tx.getResourceData(contextRef, false);
+    expect(context.type.name).toEqual("ActionContext");
+    return {
+      contextId: String(parseSignedResourceId(contextRef).globalId),
+      contextRef,
+      baggage: JSON.parse(Buffer.from(context.data!).toString()) as ActionContextBaggage,
+    };
+  });
+}
+
+/** Returns the render an action context belongs to: the resource of the Running transition that the
+ * workflow controller records when it evaluates the render. The key names the decimal resource ID.
+ * Polls until the transition lands: reports are applied asynchronously after the commit. */
+async function contextRender(pl: PlClient, contextRef: SignedResourceId): Promise<string> {
+  return await poll(pl, async ({ tx }) => {
+    const kv = await tx.listKeyValuesString(contextRef);
+    const running = kv.find((e) => e.key.startsWith("transition/render/"));
+    if (running === undefined) throw new ContinuePolling();
+    expect(JSON.parse(running.value)).toMatchObject({ state: "Running" });
+    return running.key.split("/")[2];
+  });
+}
+
+test("block action context outlives its renderer and is replaced on re-render", async () => {
+  //
+  // Scenario:
+  //  1. Create a project and render production of one block.
+  //  2. Wait until the production output is final.
+  //  3. Without a link capability the render has no action context. Otherwise read the action
+  //     context through prodActionContext and check its baggage.
+  //  4. The context has the Running transition of the render. Wait until the render is garbage
+  //     collected; the context must stay readable.
+  //  5. Re-render production: prodActionContext must point at a new context with a new run-id.
+  //  6. Render and stop production in one transaction: prodActionContext must be removed.
+  //
+
+  const quickJs = await getQuickJS();
+
+  await TestHelpers.withTempRoot(async (pl) => {
+    // 1. Create a project and render production of one block.
+    const prj = await pl.withWriteTx("CreatingProject", async (tx) => {
+      const prjRef = await createProject(tx);
+      tx.createField(field(tx.clientRoot, "prj"), "Dynamic", prjRef);
+      await tx.commit();
+      return await toGlobalResourceId(prjRef);
+    });
+
+    const renderBlock = async (numbers: number[], addBlock: boolean) =>
+      await pl.withWriteTx("RenderBlock", async (tx) => {
+        const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
+        if (addBlock)
+          mut.addBlock(
+            { id: "block1", label: "Block1", renderingMode: "Heavy" },
+            {
+              storageMode: "legacy",
+              legacyState: JSON.stringify({ args: { numbers } }),
+              blockPack: await TestBPPreparer.prepare(BPSpecEnterV041NotPrepared),
+            },
+          );
+        else
+          mut.setStates([
+            { modelAPIVersion: 1, blockId: "block1", state: { args: { numbers }, uiState: {} } },
+          ]);
+        mut.renderProduction(["block1"], true);
+        mut.save();
+        await tx.commit();
+      });
+
+    await renderBlock([1, 2, 3], true);
+
+    // 2. Wait until the production output is final.
+    await poll(pl, async (tx) => {
+      const prjR = await tx.get(prj);
+      await prjR.get(projectFieldName("block1", "prodOutput")).then((r) => r.final());
+    });
+
+    // 3. Read the action context through prodActionContext and check its baggage.
+    if (!pl.hasCapability(ActionContextLinkCapability)) {
+      expect(await readActionContext(pl, prj, "block1")).toBeUndefined();
+      return;
+    }
+    const first = (await readActionContext(pl, prj, "block1"))!;
+    expect(first).toBeDefined();
+    expect(Object.keys(first.baggage).sort()).toEqual(["block-id", "block-pack", "name", "run-id"]);
+    expect(first.baggage["block-id"]).toEqual("block1");
+    expect(first.baggage["name"]).toEqual("Block1");
+    expect(JSON.parse(first.baggage["block-pack"])).toBeTypeOf("object");
+    expect(first.baggage["run-id"]).toMatch(/^[0-9a-f-]{36}$/);
+
+    // 4. The Running transition names the render; the context outlives it. The render's own end is
+    //    awaited through its Completed report, which every reader can see: reading the render
+    //    itself needs a signed reference that no field of the project holds once it is done.
+    const firstRender = await contextRender(pl, first.contextRef);
+    await poll(pl, async ({ tx }) => {
+      const kv = await tx.listKeyValuesString(first.contextRef);
+      const completed = kv.some(
+        (e) =>
+          e.key.startsWith(`transition/render/${firstRender}/`) &&
+          JSON.parse(e.value).state === "Completed",
+      );
+      if (!completed) throw new ContinuePolling();
+    });
+    const afterRenderEnd = (await readActionContext(pl, prj, "block1"))!;
+    expect(afterRenderEnd.contextId).toEqual(first.contextId);
+    expect(afterRenderEnd.baggage).toEqual(first.baggage);
+
+    // 4b. The read-back sees the same context with its baggage, and names the render.
+    const tree = (await readBlockActionContextTree(pl, prj, "block1", "prod"))!;
+    expect(tree).toBeDefined();
+    expect(tree.contextId).toEqual(first.contextId);
+    expect(tree.baggage).toEqual(first.baggage);
+    expect(tree.resource).toEqual(firstRender);
+    expect(tree.transitions.map((t) => [t.namespace, t.resource, t.state])).toContainEqual([
+      "render",
+      firstRender,
+      "Running",
+    ]);
+
+    // 5. Re-render production: prodActionContext must point at a new context with a new run-id.
+    await renderBlock([4, 5, 6], false);
+    const second = (await readActionContext(pl, prj, "block1"))!;
+    expect(second.contextId).not.toEqual(first.contextId);
+    expect(second.baggage["run-id"]).not.toEqual(first.baggage["run-id"]);
+    expect(await contextRender(pl, second.contextRef)).not.toEqual(firstRender);
+
+    // 6. Render and stop production in one transaction: prodActionContext must be removed.
+    await pl.withWriteTx("RenderAndStop", async (tx) => {
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
+      mut.setStates([
+        { modelAPIVersion: 1, blockId: "block1", state: { args: { numbers: [7] }, uiState: {} } },
+      ]);
+      mut.renderProduction(["block1"], true);
+      mut.stopProduction("block1");
+      mut.save();
+      await tx.commit();
+    });
+    expect(await readActionContext(pl, prj, "block1")).toBeUndefined();
+  });
+});

package/src/mutator/template/render_block.ts

@@ -1,7 +1,7 @@
 import type { AnyRef, PlTransaction, ResourceRef, ResourceType } from "@milaboratories/pl-client";
 import { field, Pl } from "@milaboratories/pl-client";
 import { randomUUID } from "node:crypto";
-import { createRenderTemplate } from "./render_template";
+import { createRenderTemplateWithResource } from "./render_template";
 
 export const BContextEnd: ResourceType = { name: "BContextEnd", version: "1" };
 export const BContext: ResourceType = { name: "BContext", version: "1" };
@@ -30,12 +30,63 @@ export type HeavyBlockOutputs = {
 
 export const HeavyBlockOutputNames: (keyof HeavyBlockOutputs)[] = ["context", "result"];
 
+/** Baggage of the root action context of a block render. See the block action context spec. */
+export type BlockActionContextBaggage = {
+  /** Display name of the block. */
+  name: string;
+  /** JSON of the block pack spec. */
+  blockPack: string;
+  blockId: string;
+};
+
+/** Resource type of a block action context. See ActionContextV1Type in the backend. */
+export const ActionContextType: ResourceType = { name: "ActionContext", version: "1" };
+
+/** The capability under which the backend accepts the "actionContext" link on a render. */
+export const ActionContextLinkCapability = "optionalFieldsInSchemas:v1";
+
+/**
+ * Creates the root action context of a block render: an ActionContext resource carrying the
+ * baggage, with a new random `run-id`, since each render is one run of the block and the contexts
+ * the templates create copy it. Undefined without baggage, or on a backend that cannot take the link.
+ */
+function createBlockActionContext(
+  tx: PlTransaction,
+  baggage: BlockActionContextBaggage | undefined,
+): ResourceRef | undefined {
+  if (baggage === undefined || !tx.hasCapability(ActionContextLinkCapability)) return undefined;
+  return tx.createEphemeral(
+    ActionContextType,
+    JSON.stringify({
+      name: baggage.name,
+      "block-pack": baggage.blockPack,
+      "block-id": baggage.blockId,
+      "run-id": randomUUID(),
+    }),
+  );
+}
+
+/**
+ * Creates the heavy block render, and with `baggage` its root action context in the same
+ * transaction. Returns `actionContext`: the render is garbage collected after its outputs resolve,
+ * so the caller must hold the context to keep it.
+ */
 export function createRenderHeavyBlock(
   tx: PlTransaction,
   tpl: AnyRef,
   inputs: HeavyBlockInputs,
-): HeavyBlockOutputs {
-  return createRenderTemplate(tx, tpl, true, inputs, HeavyBlockOutputNames);
+  baggage?: BlockActionContextBaggage,
+): HeavyBlockOutputs & { actionContext?: ResourceRef } {
+  const actionContext = createBlockActionContext(tx, baggage);
+  const { outputs } = createRenderTemplateWithResource(
+    tx,
+    tpl,
+    true,
+    inputs,
+    HeavyBlockOutputNames,
+    actionContext,
+  );
+  return actionContext === undefined ? outputs : { ...outputs, actionContext };
 }
 
 export type LightBlockInputs = {
@@ -51,12 +102,26 @@ export type LightBlockOutput = {
 
 export const LightBlockOutputNames: (keyof LightBlockOutput)[] = ["result"];
 
+/**
+ * Creates the light block render, with the same action context as a heavy one, so template code
+ * of either kind knows its block and run.
+ */
 export function createRenderLightBlock(
   tx: PlTransaction,
   tpl: ResourceRef,
   inputs: LightBlockInputs,
-): LightBlockOutput {
-  return createRenderTemplate(tx, tpl, true, inputs, LightBlockOutputNames);
+  baggage?: BlockActionContextBaggage,
+): LightBlockOutput & { actionContext?: ResourceRef } {
+  const actionContext = createBlockActionContext(tx, baggage);
+  const { outputs } = createRenderTemplateWithResource(
+    tx,
+    tpl,
+    true,
+    inputs,
+    LightBlockOutputNames,
+    actionContext,
+  );
+  return actionContext === undefined ? outputs : { ...outputs, actionContext };
 }
 
 export function createBContextEnd(tx: PlTransaction): ResourceRef {

package/src/mutator/template/render_template.ts

@@ -1,4 +1,4 @@
-import type { AnyRef, FieldRef, PlTransaction } from "@milaboratories/pl-client";
+import type { AnyRef, FieldRef, PlTransaction, ResourceRef } from "@milaboratories/pl-client";
 import { field, Pl } from "@milaboratories/pl-client";
 import type { ResourceType } from "@platforma-sdk/model";
 
@@ -27,6 +27,24 @@ export function createRenderTemplate<O extends string>(
   inputs: Pl.PlRecord,
   outputNames: O[],
 ): Record<O, FieldRef> {
+  return createRenderTemplateWithResource(tx, tpl, ephemeral, inputs, outputNames).outputs;
+}
+
+/**
+ * Same as {@link createRenderTemplate}, and also returns the render resource itself.
+ *
+ * With `actionContext`, the render's "actionContext" service field points to it. The link is created
+ * before the inputs are locked: the backend accepts it only in the creating transaction, ahead of
+ * any explicit input lock.
+ */
+export function createRenderTemplateWithResource<O extends string>(
+  tx: PlTransaction,
+  tpl: AnyRef,
+  ephemeral: boolean,
+  inputs: Pl.PlRecord,
+  outputNames: O[],
+  actionContext?: AnyRef,
+): { render: ResourceRef; outputs: Record<O, FieldRef> } {
   if (outputNames.length === 0) throw new Error("Zero output names provided");
   const rId = ephemeral ? tx.createEphemeral(EphRenderTemplate) : tx.createStruct(RenderTemplate);
 
@@ -35,7 +53,9 @@ export function createRenderTemplate<O extends string>(
 
   tx.createField(tplField, "Input", tpl);
   tx.createField(inputsField, "Input", Pl.createPlMap(tx, inputs, ephemeral));
+  if (actionContext !== undefined)
+    tx.createField(field(rId, "actionContext"), "Service", actionContext);
   tx.lockInputs(rId);
 
-  return Pl.futureRecord(tx, rId, outputNames, "Output", "outputs/");
+  return { render: rId, outputs: Pl.futureRecord(tx, rId, outputNames, "Output", "outputs/") };
 }