@nanobpm/nano-workforce 0.200.5 → 0.200.6

package/app/deliveryRunner.ts

@@ -122,6 +122,10 @@ export interface PreparedDeliveryGraph {
   processDefinitionId: string;
   bpmn: string;
   nodeInputs: Record<string, NodeInput>;
+  /** The per-run token this prepare resolved (`options.runKey` or a fresh random). Surfaced so
+   *  `runDeliveryGraph` seeds the SAME value as a run-root process variable — the reconciliation
+   *  handle a stale-claim relaunch matches a still-running original instance on (issue #852). */
+  runKey: string;
 }
 
 export type PrepareDeliveryResult =
@@ -193,7 +197,7 @@ export async function prepareDeliveryGraph(
     if (element === undefined) continue; // unreachable — resolved covers every node — but keep total.
     nodeInputs[element] = buildNodeInput(node, { runKey, element, ...timeouts, requiredEmits: requiredEmitsByNodeId.get(node.id) ?? EMPTY_REQUIRED_EMITS });
   }
-  return { ok: true, prepared: { processDefinitionId, bpmn, nodeInputs } };
+  return { ok: true, prepared: { processDefinitionId, bpmn, nodeInputs, runKey } };
 }
 
 /** Deploy + start a compiled graph as a running engine-native instance. Idempotent at the DEFINITION
@@ -207,7 +211,7 @@ export async function runDeliveryGraph(
 ): Promise<RunDeliveryResult> {
   const prep = await prepareDeliveryGraph(graph, options);
   if (!prep.ok) return prep;
-  const { processDefinitionId, bpmn, nodeInputs } = prep.prepared;
+  const { processDefinitionId, bpmn, nodeInputs, runKey } = prep.prepared;
 
   await engine.deployResources([{ name: `${processDefinitionId}.bpmn`, content: bpmn, contentType: "application/xml" }]);
   // Per-node repository isolation (#739): the `io.nanobpm.agentTask.repository` envelope is now seeded
@@ -218,10 +222,18 @@ export async function runDeliveryGraph(
   // repository variable is seeded here; the resolution + loud-failure invariant (unresolved agent cell
   // on a non-`repoless` run) and the `repoless`/repo-base conflict guard all live in
   // `injectAgentRepoEnvelopes`, which the prepare step above already ran (throwing before deploy).
+  //
+  // `runKey` IS seeded as a run-root variable (unlike the repo envelope): it is the durable
+  // reconciliation handle a stale-claim relaunch uses to find a still-running ORIGINAL instance of this
+  // run before starting a second one (issue #852 review — thread deliveryGraphRun.ts:180). Every
+  // instance of this content-addressed definition carries its run key, so
+  // `searchProcessInstances({ processDefinitionId, state:"ACTIVE" })` + a `runKey` variable match
+  // identifies the live duplicate a timeout-only reclaim cannot distinguish from a dead claim.
   const { processInstanceKey } = await engine.createInstance({
     processDefinitionId,
     variables: {
       nodeInputs,
+      runKey,
       // Stage 0 transcript correlation (#543): the transcript-endpoint base every agent node's
       // completing worker appends its jobKey-scoped stream to, to emit `transcriptUrl` (see the agent
       // node ioMapping in deliveryGraphCompiler). Seeded once at the run root — the same value for
@@ -233,7 +245,7 @@ export async function runDeliveryGraph(
   // downstream consumers expect a string — coerce (codebase-wide `String(...)` pattern, e.g. app/plan.ts).
   return {
     ok: true,
-    handle: { processDefinitionId, bpmn, nodeInputs, processInstanceKey: String(processInstanceKey) },
+    handle: { processDefinitionId, bpmn, nodeInputs, runKey, processInstanceKey: String(processInstanceKey) },
   };
 }
 

package/app/service.ts

@@ -31,8 +31,9 @@ import { envContract, readEnvOr } from "./contracts.ts";
 import { makeDefaultReadHead } from "./currentHead.ts";
 import { isUniqueConstraintFence } from "./dbFence.ts";
 import { deriveDelivery, EPIC_LIVE_STATUSES, TERMINAL_STATUSES } from "./delivery.ts";
+import { DELIVERY_GRAPH_PROCESS_ID } from "./deliveryGraphCompiler.ts";
 import { sweepExpiredProposals } from "./deliveryGraphProposals.ts";
-import { deliveryGraphRuns, deriveDeliveryPhase, parseHumanLabels } from "./deliveryGraphRun.ts";
+import { deliveryGraphRuns, deriveDeliveryPhase, parseHumanLabels, reconcileOriginalInstanceBeforeRelaunch, reconcileStaleLaunchClaim } from "./deliveryGraphRun.ts";
 import { deliveryHumanContextQuestion, deliveryHumanContextUrl, isDeliveryHumanElement } from "./deliveryHuman.ts";
 import { fleetSupportsDurableResume } from "./durableResume.ts";
 import { deriveEpicPhaseLive, deriveTerminalEpicPhase } from "./epicPhase.ts";
@@ -3049,10 +3050,58 @@ export async function pollTasklessPlanTermination(
  * Scoped to `running` rows (an `awaiting-approval` run has no instance yet), so it stays O(in-flight). */
 export async function pollDeliveryGraphPhase(
   data: DataLayer,
-  engine: Pick<EngineClient, "searchProcessInstances" | "searchUserTasks">,
+  engine: Pick<EngineClient, "searchProcessInstances" | "searchUserTasks" | "searchVariables" | "cancelInstance">,
 ) {
   for (const run of await deliveryGraphRuns(data).find({ status: "running" })) {
-    if (!run.process_key) continue;
+    if (!run.process_key) {
+      // #852: a launch claim that never got an instance key within the TTL died mid-launch (the dispatch
+      // process went down before deploy, so its own `markClaimFailed` never ran). Reconcile it to
+      // `failed` so the cockpit stops showing a phantom in-flight run; a re-dispatch re-claims it. The
+      // flip is a CAS on the snapshot we read (via `reconcileStaleLaunchClaim`), so a claim a concurrent
+      // dispatch has already re-claimed between our `find()` and here is left untouched, never clobbered.
+      // Reconcile-before-retire (issue #852 review — thread deliveryGraphRun.ts:180): the crash may have
+      // happened AFTER `createInstance` succeeded but BEFORE the key was stamped, leaving a LIVE instance
+      // executing this graph's side-effecting nodes. Retiring the row to `failed` and letting a
+      // re-dispatch relaunch would then double-run those nodes. Cancel any still-ACTIVE instance of THIS
+      // run (matched by its seeded `runKey` variable) BEFORE retiring the claim, so the recovery stays
+      // at-most-once. Best-effort: a reconcile read failure must not wedge the pass — log and skip the
+      // retire this pass (the row stays stale-claim `running` and is retried next pass). The definition
+      // id is derived from the row's `digest` (`delivery-graph-<digest>`), NOT the `process_definition_id`
+      // column — that column is stamped only AFTER a successful launch, so it is NULL on exactly the
+      // crashed-mid-launch claim this path exists to recover.
+      //
+      // Fence the engine cancellation with the stale-claim CAS lease (issue #853 review — thread
+      // service.ts:3077): the cancel is passed to `reconcileStaleLaunchClaim` as `finalize`, run INSIDE
+      // the CAS transaction ONLY after the lease flip lands. Two races close:
+      //   • a FRESH `running`/NULL-key claim (still within its TTL) is non-stale, so
+      //     `reconcileStaleLaunchClaim` returns before the flip and the cancel never runs — a legitimate
+      //     just-launched instance (createInstance succeeded, key stamp pending) is never cancelled; and
+      //   • a concurrent dispatch that RE-CLAIMS the row first (refreshing `updated_at`) makes the CAS
+      //     match zero rows, so `finalize` never runs and the live replacement is never cancelled — the
+      //     retire is finalized only for the lease owner, and the lease is non-reclaimable for the
+      //     cancel's duration (the flipped `failed` row fails the dispatch's `status <> 'running'` guard).
+      // Best-effort: a reconcile read failure must not wedge the pass — `finalize` throws, the
+      // transaction rolls back (the row stays stale-claim `running`), we log and skip the retire this
+      // pass, and the next pass retries. A fail-closed `ReconcileConflictError` (an ACTIVE
+      // same-definition instance with no readable run-root `runKey`, e.g. a pre-upgrade legacy original)
+      // takes this same path: the retire is deferred rather than relaunching a possible duplicate.
+      const cancelOriginal = async () => {
+        if (!run.digest) return;
+        const cancelled = await reconcileOriginalInstanceBeforeRelaunch(engine, {
+          runKey: run.run_key,
+          processDefinitionId: `${DELIVERY_GRAPH_PROCESS_ID}-${run.digest}`,
+        });
+        if (cancelled.length > 0) {
+          console.error(`[poller] delivery graph ${run.run_key}: cancelled a still-running original instance before retiring the stale claim: ${cancelled.join(", ")}`);
+        }
+      };
+      try {
+        await reconcileStaleLaunchClaim(data, run, cancelOriginal);
+      } catch (err) {
+        console.error(`[poller] delivery graph ${run.run_key}: reconcile-before-retire failed, leaving the claim for next pass: ${err}`);
+      }
+      continue;
+    }
     const processKey = run.process_key;
     try {
       const [snapshots, parks] = await Promise.all([

package/operations/dispatchDeliveryGraph.test.ts

@@ -15,7 +15,7 @@ import assert from "node:assert/strict";
 import { bootTestApp, type TestApp } from "@nanobpm/urban-testkit";
 import { redactFreeText } from "../app/deliveryGraphCompiler.ts";
 import { deliveryGraphProposals } from "../app/deliveryGraphProposals.ts";
-import { deliveryGraphRuns } from "../app/deliveryGraphRun.ts";
+import { deliveryGraphRuns, LAUNCH_CLAIM_TTL_MS } from "../app/deliveryGraphRun.ts";
 import { isEquivalentReStage, stableProposalRunKey } from "./dispatchDeliveryGraph.ts";
 
 const APP_ROOT = resolve(import.meta.dirname, "..");
@@ -190,6 +190,28 @@ describe("dispatchDeliveryGraph — operator dispatch by staged-proposal digest"
     assert.equal((await deliveryGraphProposals(app.db).get(staged.body.digest))?.status, "dispatched");
   });
 
+  test("#852: a launch claim stranded by a crash mid-launch (running, no process key, past the TTL) does not wedge the proposal — a re-dispatch LAUNCHES it", async () => {
+    const app = await boot();
+    assert.ok(app.api);
+    const api = app.api;
+    const staged = await api.call<{ digest: string }>("compileDeliveryGraph", { body: HUMAN_ONLY });
+    const digest = staged.body.digest;
+    // The merlin state: the claim row + identity were written, then the process died before deploy.
+    const stale = new Date(Date.now() - LAUNCH_CLAIM_TTL_MS - 60_000).toISOString();
+    await deliveryGraphRuns(app.db).insert({
+      run_key: digest, process_key: null, process_definition_id: null, digest, status: "running", side_effecting: 0,
+      node_count: 1, human_node_count: 1, side_effect_count: 0, title: "manual gate", phase: "Running", phase_node_id: null,
+      human_labels: null, created_at: stale, updated_at: stale, acknowledged_at: null,
+    });
+    const res = await api.call<{ ok: boolean; alreadyRunning: boolean; processInstanceKey?: string }>("dispatchDeliveryGraph", {
+      body: { digest, repoless: true },
+    });
+    assert.equal(res.status, 202, JSON.stringify(res.body));
+    assert.equal(res.body.alreadyRunning, false, "must not short-circuit onto the phantom run");
+    assert.ok(res.body.processInstanceKey, "a real instance was started");
+    assert.equal((await deliveryGraphProposals(app.db).get(digest))?.status, "dispatched");
+  });
+
   test("re-dispatching an ALREADY-dispatched digest → 400 (the proposal is consumed; the run shows in the in-flight grid)", async () => {
     const app = await boot();
     assert.ok(app.api);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nanobpm/nano-workforce",
-  "version": "0.200.5",
+  "version": "0.200.6",
   "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
   "type": "module",
   "main": "main.ts",
@@ -70,7 +70,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.11",
-    "@nanobpm/urban-testkit": "^1.4.0",
+    "@nanobpm/urban-testkit": "^1.6.0",
     "@nanobpm/workflow": "^0.15.0",
     "@semantic-release/changelog": "^7.0.0",
     "@semantic-release/git": "^11.0.0",

package/pages/delivery-graphs/staged.mount.js

@@ -53,6 +53,12 @@ const DEFAULT_REFRESH_MS = 5000;
 // forever, so the busy() lock never clears and the UI is stranded; on timeout the AbortController
 // rejects the fetch, surfacing as an error banner and re-enabling the controls via the finally blocks.
 const REQUEST_TIMEOUT_MS = 30000;
+// The dispatch POST gets its own, much longer budget (#852): the server lays out + deploys the graph
+// before it answers, which for a large graph legitimately takes minutes (54 nodes ≈ 105 s on merlin).
+// Aborting at 30 s showed the operator a failure while the launch was still in flight. Stays bounded so
+// a genuinely hung door still releases the controls. Kept under the server's launch-claim TTL
+// (`LAUNCH_CLAIM_TTL_MS`, 15 min) so the UI never gives up AFTER a crashed launch became re-claimable.
+const DISPATCH_TIMEOUT_MS = 10 * 60_000;
 
 // The confirm shown before a dispatch — dispatching authorises every side-effecting node, so the
 // operator acknowledges that the launch (and its side effects) is content-addressed to this graph.
@@ -273,9 +279,9 @@ export function mountStagedProposals(host, config = {}) {
 
   /** Fetch JSON from a door and return { status, body } (never throws on an HTTP error). Rejects
    * (AbortError) if the request outlives REQUEST_TIMEOUT_MS so a hung door can't wedge the busy lock. */
-  async function request(url, init) {
+  async function request(url, init, timeoutMs = REQUEST_TIMEOUT_MS) {
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
     try {
       const res = await fetch(url, { ...init, headers: headers(url), signal: controller.signal });
       let body = {};
@@ -291,7 +297,7 @@ export function mountStagedProposals(host, config = {}) {
   }
 
   const get = (url) => request(url, { method: "GET" });
-  const post = (url, payload) => request(url, { method: "POST", body: JSON.stringify(payload) });
+  const post = (url, payload, timeoutMs) => request(url, { method: "POST", body: JSON.stringify(payload) }, timeoutMs);
 
   let disposed = false;
   // True while the last completed load failed — so a subsequent successful load knows to clear its own
@@ -372,9 +378,9 @@ export function mountStagedProposals(host, config = {}) {
   // the list on the next poll — refresh immediately so the operator sees it leave.
   async function doDispatch(payload) {
     busy(true);
-    setStatus("Dispatching…");
+    setStatus("Dispatching… laying out and deploying the graph — a large graph can take a few minutes.");
     try {
-      const { status, body } = await post(dispatchUrl, payload);
+      const { status, body } = await post(dispatchUrl, payload, DISPATCH_TIMEOUT_MS);
       if ((status === 202 || status === 200) && body.ok) {
         setStatus("\u2713 Dispatched — the run is now in flight.", "ok");
         await refresh();
@@ -382,7 +388,12 @@ export function mountStagedProposals(host, config = {}) {
         setStatus(body && body.error ? body.error : "Dispatch failed.", "err");
       }
     } catch (err) {
-      setStatus(err && err.message ? err.message : "Dispatch request failed.", "err");
+      setStatus(
+        err && err.name === "AbortError"
+          ? "Dispatch is taking longer than expected — it may still be launching. Check Active Delivery Graphs before dispatching again."
+          : err && err.message ? err.message : "Dispatch request failed.",
+        "err",
+      );
     } finally {
       busy(false);
     }

package/test/delivery-graphs-staged-slow-dispatch.test.ts

@@ -0,0 +1,85 @@
+// #852 — a large graph's dispatch legitimately takes minutes (layout + deploy: ~105 s for 54 nodes on
+// merlin). The staged view aborted EVERY request at a 30 s timeout and rendered the AbortError as the
+// dispatch failure — the operator saw a "crash" while the server was still launching. The dispatch POST
+// must outlive a slow-but-healthy launch (bounded by its own, much longer budget) and tell the operator
+// it is still working.
+import { mock, test } from "node:test";
+import { assert, assertEquals } from "#test-assert";
+import { parseHTML } from "linkedom";
+import { mountStagedProposals } from "../pages/delivery-graphs/staged.mount.js";
+
+const STAGED_URL = "https://app.test/app/api/delivery-graph/staged";
+const DISPATCH_URL = "https://app.test/app/api/actions/delivery-graph/dispatch";
+const PROPOSAL = {
+  digest: "abc123def456",
+  title: "Big graph",
+  sideEffecting: true,
+  nodeCount: 54,
+  humanNodeCount: 2,
+  sideEffectCount: 34,
+  createdAt: "2026-01-01T00:00:00.000Z",
+  expiresAt: "2026-01-02T00:00:00.000Z",
+};
+
+const flush = async () => {
+  for (let i = 0; i < 6; i++) await new Promise((resolve) => setImmediate(resolve));
+};
+
+test("#852: a dispatch slower than the generic 30 s request timeout is NOT aborted — it completes and reports success", async () => {
+  const { window: domWindow, document } = parseHTML("<!doctype html><html><body><div id='host'></div></body></html>");
+  const host = document.getElementById("host");
+  assert(host, "host");
+  let resolveDispatch: (() => void) | undefined;
+  let aborted = false;
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = async (input, init) => {
+    const url = String(input);
+    if (url === STAGED_URL) return new Response(JSON.stringify({ proposals: [PROPOSAL] }), { status: 200 });
+    if (url === DISPATCH_URL) {
+      return await new Promise<Response>((resolve, reject) => {
+        init?.signal?.addEventListener("abort", () => {
+          aborted = true;
+          reject(new DOMException("signal is aborted without reason", "AbortError"));
+        });
+        resolveDispatch = () => resolve(new Response(JSON.stringify({ ok: true, status: "running" }), { status: 202 }));
+      });
+    }
+    return new Response("{}", { status: 404 });
+  };
+  const win = { confirm: () => false, prompt: () => null, location: { href: "https://app.test/delivery-graphs/", origin: "https://app.test" }, parent: undefined as unknown };
+  win.parent = win;
+  const origWindow = Reflect.get(globalThis, "window");
+  Reflect.set(globalThis, "window", win);
+  mock.timers.enable({ apis: ["setTimeout", "setInterval"] });
+  const dispose = mountStagedProposals(host, { stagedUrl: STAGED_URL, dispatchUrl: DISPATCH_URL, refreshMs: 1_000_000_000 });
+  const click = (sel: string) => {
+    const el = host.querySelector(sel);
+    assert(el, `missing ${sel}`);
+    el.dispatchEvent(new domWindow.Event("click", { bubbles: true, cancelable: true }));
+  };
+  try {
+    await flush();
+    click("[data-dispatch]");
+    await flush();
+    click("[data-dispatch-confirm]");
+    await flush();
+    assert(resolveDispatch, "the dispatch POST was sent");
+    // Two minutes of a healthy-but-slow launch (layout + deploy) — well past the old 30 s abort.
+    for (let s = 0; s < 120; s++) {
+      mock.timers.tick(1000);
+      await flush();
+    }
+    assertEquals(aborted, false, "the dispatch request must not be aborted while the server is still launching");
+    const working = host.querySelector(".status")?.textContent ?? "";
+    assert(/dispatching/i.test(working), `the operator is told it is still working, got: ${working}`);
+    resolveDispatch();
+    await flush();
+    const status = host.querySelector(".status")?.textContent ?? "";
+    assert(/dispatched/i.test(status), `expected success, got: ${status}`);
+  } finally {
+    dispose();
+    mock.timers.reset();
+    globalThis.fetch = origFetch;
+    Reflect.set(globalThis, "window", origWindow);
+  }
+});