@nanobpm/nano-workforce 0.200.5 → 0.200.7

package/app/deliveryGraphRun.test.ts

@@ -8,7 +8,7 @@ import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { test } from "node:test";
-import { assert, assertEquals, assertStringIncludes } from "#test-assert";
+import { assert, assertEquals, assertRejects, assertStringIncludes } from "#test-assert";
 import type { DataLayer } from "@nanobpm/urban";
 import { bootTestApp } from "@nanobpm/urban-testkit";
 import { compileDeliveryGraph } from "./deliveryGraphCompiler.ts";
@@ -22,7 +22,12 @@ import {
   deliveryGraphRunIdentities,
   deliveryGraphRuns,
   humanTaskElementId,
+  isStaleLaunchClaim,
+  LAUNCH_CLAIM_TTL_MS,
   parseHumanLabels,
+  ReconcileConflictError,
+  reconcileOriginalInstanceBeforeRelaunch,
+  reconcileStaleLaunchClaim,
 } from "./deliveryGraphRun.ts";
 import { pollDeliveryGraphPhase } from "./service.ts";
 
@@ -127,6 +132,324 @@ test("claimRunForLaunch: winning a re-run claim ATOMICALLY invalidates the prior
   });
 });
 
+// ── #852: a launch claim stranded by a crash mid-launch must not wedge the run forever ──────────────
+// The dispatch claims the row `running` (process_key null) BEFORE deploy; if the process dies mid-launch
+// (merlin: a 54-node graph spent ~105 s in layout, the app went down) `markClaimFailed` never runs, and
+// the phantom row short-circuited every re-dispatch onto a run that does not exist.
+const ago = (ms: number) => new Date(Date.now() - ms).toISOString();
+
+test("#852 isStaleLaunchClaim: only a `running` row with NO process key older than the launch TTL is stale", () => {
+  const base = claimRow("running");
+  assertEquals(isStaleLaunchClaim({ ...base, updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) }), true);
+  assertEquals(isStaleLaunchClaim({ ...base, updated_at: ago(1000) }), false, "an in-flight launch is not stale");
+  assertEquals(isStaleLaunchClaim({ ...base, process_key: "29", updated_at: ago(LAUNCH_CLAIM_TTL_MS * 10) }), false, "a launched run is never stale");
+  assertEquals(isStaleLaunchClaim({ ...base, status: "failed", updated_at: ago(LAUNCH_CLAIM_TTL_MS * 10) }), false);
+  assert(LAUNCH_CLAIM_TTL_MS >= 10 * 60_000, "the TTL must comfortably exceed a slow large-graph launch");
+});
+
+test("#852 claimRunForLaunch: a STALE launch claim is re-claimable; a fresh in-flight claim still blocks the double-launch", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    await runs.insert({ ...claimRow("running"), updated_at: ago(1000) });
+    assertEquals(await claimRunForLaunch(data, true, claimRow("running")), false, "fresh claim: launch still in flight");
+    await runs.update("rk", { updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) });
+    assertEquals(await claimRunForLaunch(data, true, claimRow("running")), true, "stale claim: the crashed launch is re-claimed");
+    assertEquals(await claimRunForLaunch(data, true, claimRow("running")), false, "…by exactly one racer");
+  });
+});
+
+test("#852 pollDeliveryGraphPhase: a stale launch claim is reconciled to failed; a fresh one is left alone", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    await runs.insert({ ...claimRow("running"), run_key: "stale", updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) });
+    await runs.insert({ ...claimRow("running"), run_key: "fresh", updated_at: ago(1000) });
+    const engine = {
+      searchProcessInstances: async () => [],
+      searchUserTasks: async () => [],
+    };
+    await pollDeliveryGraphPhase(data, engine as never);
+    assertEquals((await runs.get("stale"))?.status, "failed");
+    assertEquals((await runs.get("stale"))?.phase, DELIVERY_PHASE.FAILED);
+    assertEquals((await runs.get("fresh"))?.status, "running");
+  });
+});
+
+test("#852 pollDeliveryGraphPhase: retiring a stale claim cancels a still-running ORIGINAL instance first (at-most-once recovery)", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    // A stale claim whose dispatch died AFTER createInstance but BEFORE stamping the key: the row has a
+    // NULL process_key AND a NULL process_definition_id (the stamp never landed) — the poller derives the
+    // definition id from the row's `digest` (`delivery-graph-d`). A LIVE instance of this run is ACTIVE.
+    await runs.insert({
+      ...claimRow("running"),
+      run_key: "rk",
+      digest: "d",
+      updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000),
+    });
+    const cancelled: string[] = [];
+    const engine = {
+      searchProcessInstances: async (filter?: { processDefinitionId?: string; state?: string }) =>
+        filter?.state === "ACTIVE" ? [{ processInstanceKey: "PI-live", state: "ACTIVE" }] : [],
+      searchVariables: async (filter?: { processInstanceKey?: string; name?: string }) =>
+        filter?.name === "runKey" && filter.processInstanceKey === "PI-live"
+          ? [{ variableKey: "v", name: "runKey", value: JSON.stringify("rk"), scopeKey: "PI-live", processInstanceKey: "PI-live", isTruncated: false }]
+          : [],
+      cancelInstance: async (req: { processInstanceKey: string }) => {
+        cancelled.push(req.processInstanceKey);
+      },
+      searchUserTasks: async () => [],
+    };
+    await pollDeliveryGraphPhase(data, engine as never);
+    assertEquals(cancelled, ["PI-live"], "the live original is cancelled before the claim is retired");
+    assertEquals((await runs.get("rk"))?.status, "failed");
+  });
+});
+
+test("#852 pollDeliveryGraphPhase: a reconcile read failure leaves the stale claim for the next pass (no wedge, no blind retire)", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    await runs.insert({
+      ...claimRow("running"),
+      run_key: "rk",
+      digest: "d",
+      updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000),
+    });
+    const engine = {
+      searchProcessInstances: async () => {
+        throw new Error("engine read unavailable");
+      },
+      searchUserTasks: async () => [],
+    };
+    await pollDeliveryGraphPhase(data, engine as never);
+    // The claim is NOT retired this pass (the reconcile could not prove no live original), so a
+    // re-dispatch is not waved through into a possible double-launch; the next pass retries.
+    assertEquals((await runs.get("rk"))?.status, "running");
+  });
+});
+
+test("#853 pollDeliveryGraphPhase: a FRESH in-flight claim (within TTL) is never engine-cancelled — the cancel is fenced behind the staleness CAS", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    // A launch that is still in flight: running, no process key yet, but WELL within the launch TTL.
+    // `createInstance` may have just succeeded (a legitimate live instance) with the key stamp still
+    // pending. The poller must NOT cancel that instance — the row is not stale, so the reconcile lease
+    // is never acquired and `finalize` (the cancel) never runs.
+    await runs.insert({ ...claimRow("running"), run_key: "fresh", digest: "d", updated_at: ago(1000) });
+    const cancelled: string[] = [];
+    const engine = {
+      searchProcessInstances: async (filter?: { state?: string }) =>
+        filter?.state === "ACTIVE" ? [{ processInstanceKey: "PI-legit", state: "ACTIVE" }] : [],
+      searchVariables: async (filter?: { processInstanceKey?: string; name?: string }) =>
+        filter?.name === "runKey" && filter.processInstanceKey === "PI-legit"
+          ? [{ variableKey: "v", name: "runKey", value: JSON.stringify("fresh"), scopeKey: "PI-legit", processInstanceKey: "PI-legit", isTruncated: false }]
+          : [],
+      cancelInstance: async (req: { processInstanceKey: string }) => {
+        cancelled.push(req.processInstanceKey);
+      },
+      searchUserTasks: async () => [],
+    };
+    await pollDeliveryGraphPhase(data, engine as never);
+    assertEquals(cancelled, [], "a fresh in-flight claim's legitimate instance is never cancelled");
+    assertEquals((await runs.get("fresh"))?.status, "running", "the fresh claim is left running, not retired");
+  });
+});
+
+test("#853 reconcileStaleLaunchClaim: a concurrent dispatch that re-claims the stale row first is NOT cancelled — the finalize cancel runs only for the lease owner", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    // The poller read this row as a stale launch claim …
+    const observed = { ...claimRow("running"), updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) };
+    await runs.insert(observed);
+    // … but before the poller's CAS, a concurrent dispatch atomically RE-CLAIMS the stale row
+    // (refreshing `updated_at`) and starts its replacement instance.
+    assertEquals(await claimRunForLaunch(data, true, claimRow("running")), true, "concurrent dispatch re-claims the stale row");
+    let cancelled = false;
+    // The poller now reconciles off its STALE snapshot, passing the engine cancel as `finalize`. The CAS
+    // matches zero rows (updated_at moved), so the lease is not acquired and `finalize` must NOT run —
+    // the live replacement is never cancelled.
+    const flipped = await reconcileStaleLaunchClaim(data, observed, async () => {
+      cancelled = true;
+    });
+    assertEquals(flipped, false, "the stale snapshot no longer matches → no flip");
+    assertEquals(cancelled, false, "the cancel is fenced: it never runs against a claim the poller did not win");
+    assertEquals((await runs.get("rk"))?.status, "running", "the renewed claim is left running");
+  });
+});
+
+test("#853 reconcileStaleLaunchClaim: the lease holder's finalize cancel DOES run, and the row is retired to failed only for the lease owner", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    const observed = { ...claimRow("running"), updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) };
+    await runs.insert(observed);
+    let cancelled = false;
+    const flipped = await reconcileStaleLaunchClaim(data, observed, async () => {
+      cancelled = true;
+    });
+    assertEquals(flipped, true, "the unchanged stale claim is flipped (lease acquired)");
+    assertEquals(cancelled, true, "the lease owner's finalize cancel runs");
+    assertEquals((await runs.get("rk"))?.status, "failed", "the row is retired to failed");
+  });
+});
+
+test("#852 reconcileStaleLaunchClaim: the retire-to-failed flip is a CAS on the OBSERVED snapshot — a claim a concurrent dispatch re-claimed (refreshing updated_at) between the poller's read and the write is NOT clobbered", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    // The poller read this row as a stale launch claim …
+    const observed = { ...claimRow("running"), updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) };
+    await runs.insert(observed);
+    // … but before it writes, a concurrent dispatch atomically RE-CLAIMS the stale row, refreshing
+    // `updated_at` to a fresh in-flight claim (still running, still no process key yet).
+    assertEquals(await claimRunForLaunch(data, true, claimRow("running")), true, "concurrent dispatch re-claims the stale row");
+    const reclaimedAt = (await runs.get("rk"))?.updated_at;
+    // The poller now reconciles off its STALE snapshot. The CAS must find no matching row and leave the
+    // fresh claim untouched — otherwise a third dispatch could re-claim mid-layout and double-launch.
+    assertEquals(await reconcileStaleLaunchClaim(data, observed), false, "stale snapshot no longer matches → no flip");
+    const row = await runs.get("rk");
+    assertEquals(row?.status, "running", "the renewed claim is left running, not clobbered to failed");
+    assertEquals(row?.process_key, null);
+    assertEquals(row?.updated_at, reclaimedAt, "the renewed claim's updated_at is preserved");
+  });
+});
+
+test("#852 reconcileStaleLaunchClaim: a genuinely stale claim whose snapshot still matches IS retired to failed exactly once", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    const observed = { ...claimRow("running"), updated_at: ago(LAUNCH_CLAIM_TTL_MS + 1000) };
+    await runs.insert(observed);
+    assertEquals(await reconcileStaleLaunchClaim(data, observed), true, "the unchanged stale claim is flipped");
+    assertEquals((await runs.get("rk"))?.status, "failed");
+    assertEquals((await runs.get("rk"))?.phase, DELIVERY_PHASE.FAILED);
+    assertEquals(await reconcileStaleLaunchClaim(data, observed), false, "already failed → no second flip");
+  });
+});
+
+test("#852 reconcileStaleLaunchClaim: a non-stale claim (launched, or within TTL) is never retired", async () => {
+  await withData(async (data) => {
+    const runs = deliveryGraphRuns(data);
+    const fresh = { ...claimRow("running"), updated_at: ago(1000) };
+    await runs.insert(fresh);
+    assertEquals(await reconcileStaleLaunchClaim(data, fresh), false, "an in-flight launch is not stale");
+    assertEquals((await runs.get("rk"))?.status, "running");
+  });
+});
+
+// ── #852 review: reconcile-before-relaunch (the at-most-once fence a timeout-only reclaim lacks) ──
+// A stale launch claim can hide a LIVE original instance (the dispatch died after createInstance but
+// before stamping the process key). `reconcileOriginalInstanceBeforeRelaunch` cancels that original —
+// matched by its seeded `runKey` variable — before a relaunch starts a second instance of the same
+// side-effecting graph (thread deliveryGraphRun.ts:180).
+
+/** A minimal engine stub for the reconcile seam: serves the configured ACTIVE instances, their `runKey`
+ * variables, and records cancels. Each instance's `runKey` is seeded at the RUN-ROOT scope (scopeKey ===
+ * processInstanceKey); an optional `childRunKey` seeds a SAME-NAMED variable at a DIFFERENT (child) scope
+ * to exercise the scope-restriction guard. */
+function reconcileEngine(instances: { processInstanceKey: string; runKey?: string; childRunKey?: string; truncated?: boolean }[]) {
+  const cancelled: string[] = [];
+  const engine = {
+    searchProcessInstances: async (filter?: { processDefinitionId?: string; state?: string }) =>
+      filter?.state === "ACTIVE" || filter?.state === undefined
+        ? instances.map((i) => ({ processInstanceKey: i.processInstanceKey, state: "ACTIVE" }))
+        : [],
+    searchVariables: async (filter?: { processInstanceKey?: string; name?: string }) => {
+      const inst = instances.find((i) => i.processInstanceKey === filter?.processInstanceKey);
+      if (!inst || filter?.name !== "runKey") return [];
+      const out: { variableKey: string; name: string; value: string; scopeKey: string; processInstanceKey: string; isTruncated: boolean }[] = [];
+      if (inst.runKey !== undefined) {
+        out.push({ variableKey: "v", name: "runKey", value: JSON.stringify(inst.runKey), scopeKey: inst.processInstanceKey, processInstanceKey: inst.processInstanceKey, isTruncated: inst.truncated ?? false });
+      }
+      if (inst.childRunKey !== undefined) {
+        out.push({ variableKey: "vc", name: "runKey", value: JSON.stringify(inst.childRunKey), scopeKey: `${inst.processInstanceKey}-child`, processInstanceKey: inst.processInstanceKey, isTruncated: false });
+      }
+      return out;
+    },
+    cancelInstance: async (req: { processInstanceKey: string }) => {
+      cancelled.push(req.processInstanceKey);
+    },
+  };
+  return { engine, cancelled };
+}
+
+test("#852 reconcileOriginalInstanceBeforeRelaunch: a live original carrying THIS run key is cancelled before relaunch", async () => {
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-orig", runKey: "rk" }]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, ["PI-orig"]);
+  assertEquals(cancelled, ["PI-orig"], "the live original is cancelled so the relaunch stays at-most-once");
+});
+
+test("#852 reconcileOriginalInstanceBeforeRelaunch: an ACTIVE instance of the SAME definition but a DIFFERENT run key is left running", async () => {
+  // Two distinct runs of one graph share the content-addressed processDefinitionId; the definition
+  // filter alone would cancel a DIFFERENT run's live instance. The runKey variable is the proof.
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-other", runKey: "other-run" }]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, []);
+  assertEquals(cancelled, [], "a different run's instance is never cancelled");
+});
+
+test("#853 reconcileOriginalInstanceBeforeRelaunch: an ACTIVE same-definition candidate with NO run-root runKey (pre-upgrade legacy) fails CLOSED — throws rather than waiving a blind relaunch", async () => {
+  // A legacy instance launched before this PR seeded the correlation variable has no readable run-root
+  // `runKey`. Treating that as "not ours → relaunch" would double-run its side-effecting nodes during
+  // the upgrade window (thread deliveryGraphRun.ts:328). We can neither cancel it (unproven) nor rule it
+  // out, so we fail closed: throw ReconcileConflictError so the stale claim stays reconcile-pending.
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-legacy" }]);
+  await assertRejects(
+    () => reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" }),
+    ReconcileConflictError,
+    "PI-legacy",
+  );
+  assertEquals(cancelled, [], "never cancel an instance we cannot prove belongs to this run");
+});
+
+test("#853 reconcileOriginalInstanceBeforeRelaunch: a TRUNCATED run-root runKey value also fails CLOSED (unreadable ≠ not ours)", async () => {
+  // The engine capped the returned root value, so a byte compare would spuriously differ and fail OPEN.
+  // Same class as the absent case: unreadable must not be treated as a different run.
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-trunc", runKey: "rk", truncated: true }]);
+  await assertRejects(
+    () => reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" }),
+    ReconcileConflictError,
+    "PI-trunc",
+  );
+  assertEquals(cancelled, [], "a truncated root value is unclassifiable, not a cancel");
+});
+
+test("#853 reconcileOriginalInstanceBeforeRelaunch: an unclassifiable candidate does NOT block once we positively matched and cancelled our own original", async () => {
+  // We found and cancelled our run-root match, so we KNOW which instance is ours; an unclassifiable
+  // sibling (a different legacy run of the same graph) must not block our relaunch.
+  const { engine, cancelled } = reconcileEngine([
+    { processInstanceKey: "PI-legacy" },
+    { processInstanceKey: "PI-orig", runKey: "rk" },
+  ]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, ["PI-orig"]);
+  assertEquals(cancelled, ["PI-orig"], "a positive match resolves the reconcile; the unclassifiable sibling is a different run");
+});
+
+test("#852 reconcileOriginalInstanceBeforeRelaunch: no ACTIVE instances → nothing to cancel", async () => {
+  const { engine, cancelled } = reconcileEngine([]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, []);
+  assertEquals(cancelled, []);
+});
+
+test("#853 reconcileOriginalInstanceBeforeRelaunch: a CHILD-scoped runKey equal to ours does NOT mark an unrelated run as ours (scope-restricted match)", async () => {
+  // The candidate's ROOT runKey is a DIFFERENT run; it only happens to carry a same-named `runKey`
+  // variable in a CHILD scope (a graph-emitted fact, or an agent job-scope field) whose value equals
+  // our run key. Matching that child value would cancel an unrelated live run (thread
+  // deliveryGraphRun.ts:318). The match is restricted to the run-root scope, so this is left running.
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-other", runKey: "other-run", childRunKey: "rk" }]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, []);
+  assertEquals(cancelled, [], "a child-scoped coincidental runKey never triggers a cancel of another run");
+});
+
+test("#853 reconcileOriginalInstanceBeforeRelaunch: our run is still cancelled when it ALSO carries a differing child-scoped runKey", async () => {
+  // The run-root scope carries OUR run key (a true original) while a child scope carries a different
+  // value. The scope-restricted match must still find the root variable and cancel the original.
+  const { engine, cancelled } = reconcileEngine([{ processInstanceKey: "PI-orig", runKey: "rk", childRunKey: "something-else" }]);
+  const cancelledKeys = await reconcileOriginalInstanceBeforeRelaunch(engine as never, { runKey: "rk", processDefinitionId: "delivery-graph-d" });
+  assertEquals(cancelledKeys, ["PI-orig"]);
+  assertEquals(cancelled, ["PI-orig"], "a true root-scope match is still cancelled despite a noisy child variable");
+});
+
 // ── pollDeliveryGraphPhase: engine-key coercion ───────────────────────────────
 test("pollDeliveryGraphPhase: a numeric engine processInstanceKey still matches the string process_key, so a COMPLETED instance reconciles to done", async () => {
   await withData(async (data) => {

package/app/deliveryGraphRun.ts

@@ -23,7 +23,7 @@
 // there is no agent-facing approval gate to park a run at. The old replayable `approvalToken` and the
 // approval-park write were removed with the agent `start` door.
 
-import type { DataLayer, ProcessInstanceState } from "@nanobpm/urban";
+import type { DataLayer, EngineClient, ProcessInstanceState } from "@nanobpm/urban";
 import type { CompileDeliveryGraphResult } from "../nano-generated/api-io.d.ts";
 import { isUniqueConstraintFence } from "./dbFence.ts";
 import { redactFreeText } from "./deliveryGraphCompiler.ts";
@@ -116,6 +116,26 @@ export interface DeliveryGraphRunIdentity {
 export const deliveryGraphRunIdentities = (data: DataLayer) =>
   data.table<DeliveryGraphRunIdentity>("delivery_graph_run_identity", "run_key");
 
+/** How long a launch claim (`running` with no `process_key` yet) may stay unlaunched before it is
+ * presumed dead (issue #852). A dispatch claims the row BEFORE the CPU-bound layout + deploy; a large
+ * graph legitimately spends minutes there (54 nodes ≈ 105 s on merlin), but a process that dies
+ * mid-launch never runs `markClaimFailed`, stranding a phantom `running` row that short-circuited every
+ * re-dispatch onto a run that does not exist. Past this TTL the claim is re-claimable by a dispatch and
+ * reconciled to `failed` by the poller (`pollDeliveryGraphPhase`). Generous on purpose: re-claiming a
+ * launch that is in fact still in flight would double-launch. */
+export const LAUNCH_CLAIM_TTL_MS = 15 * 60_000;
+
+/** True when `run` is a launch claim that never got an instance key within {@link LAUNCH_CLAIM_TTL_MS}
+ * (issue #852). A launched run (process key set) or a non-`running` row is never stale. */
+export function isStaleLaunchClaim(
+  run: Pick<DeliveryGraphRun, "status" | "process_key" | "updated_at">,
+  nowMs: number = Date.now(),
+): boolean {
+  if (run.status !== "running" || run.process_key) return false;
+  const at = Date.parse(run.updated_at);
+  return Number.isFinite(at) && nowMs - at > LAUNCH_CLAIM_TTL_MS;
+}
+
 /** Atomically claim a run for LAUNCH — the at-most-once dispatch fence. Returns `true` iff THIS caller
  * won the claim and must proceed to `runDeliveryGraph`; `false` iff a concurrent submit already claimed
  * it (the caller must short-circuit as `alreadyRunning` instead of double-launching). Two fences, one
@@ -157,7 +177,7 @@ export async function claimRunForLaunch(
   }
   const res = await data.open().tx(async (t) => {
     const flip = await t.exec(
-      `UPDATE "delivery_graph_runs" SET "status" = ?, "process_key" = ?, "process_definition_id" = ?, "phase" = ?, "phase_node_id" = ?, "updated_at" = ? WHERE "run_key" = ? AND "status" <> 'running'`,
+      `UPDATE "delivery_graph_runs" SET "status" = ?, "process_key" = ?, "process_definition_id" = ?, "phase" = ?, "phase_node_id" = ?, "updated_at" = ? WHERE "run_key" = ? AND ("status" <> 'running' OR ("process_key" IS NULL AND "updated_at" < ?))`,
       [
         claim.status,
         claim.process_key,
@@ -166,6 +186,8 @@ export async function claimRunForLaunch(
         claim.phase_node_id,
         claim.updated_at,
         claim.run_key,
+        // #852: a stale launch claim (running, never got an instance key) is re-claimable.
+        new Date(Date.now() - LAUNCH_CLAIM_TTL_MS).toISOString(),
       ],
     );
     if (flip.changed !== 1) return false;
@@ -185,6 +207,175 @@ export async function claimRunForLaunch(
   return res;
 }
 
+/** Atomically reconcile a STALE launch claim to `failed` — the poller's mirror of
+ * {@link claimRunForLaunch}, and the single canonical writer of that transition (the poller calls ONLY
+ * this; it does not run its own unconditional update). A stale claim (`running`, NULL `process_key`,
+ * past {@link LAUNCH_CLAIM_TTL_MS}) died mid-launch and leaves a phantom in-flight row, so the poller
+ * retires it to `failed`. But the poller observed the row on a PRIOR `find()` read, and between that
+ * read and this write a concurrent dispatch may have atomically RE-CLAIMED the same stale row
+ * (`claimRunForLaunch`), refreshing `updated_at` (and, once it stamps, `process_key`). An UNCONDITIONAL
+ * `update(run_key, {status:'failed'})` would clobber that fresh, live claim — and a third dispatch
+ * could then re-claim it mid-layout and double-launch. So the flip is a compare-and-swap GUARDED on the
+ * exact snapshot the caller observed (`status='running'`, `process_key IS NULL`, the SAME `updated_at`):
+ * a row a concurrent dispatch has touched no longer matches and is left untouched. Returns `true` iff
+ * THIS caller flipped the row (issue #852 review — thread service.ts:3065).
+ *
+ * RECONCILIATION LEASE (`finalize`, issue #853 review — thread service.ts:3077): the poller must cancel
+ * a still-running ORIGINAL engine instance BEFORE the row is retired, but that engine cancellation must
+ * be FENCED by this same stale-row CAS — otherwise two races open up:
+ *   • a FRESH `running`/NULL-key claim (still within its TTL) would otherwise enter the cancel branch
+ *     and cancel a legitimate instance whose `createInstance` just succeeded but whose key stamp has not
+ *     landed yet; and
+ *   • a concurrent dispatch could RE-CLAIM the stale row and start its replacement BEFORE the cancel
+ *     runs, so the poller would cancel the live replacement and only afterwards discover its CAS lost.
+ * So the caller passes the engine cancellation as `finalize`, and it is run INSIDE the CAS transaction,
+ * ONLY after the lease flip succeeds (`changed === 1`). The flip atomically acquires a reconciliation
+ * lease on the observed snapshot: once it lands the row is `failed`, so a concurrent dispatch's
+ * `claimRunForLaunch` guard (`status <> 'running'`) can no longer reclaim it — the lease is
+ * non-reclaimable for the duration of the cancel. And because `finalize` runs only when THIS caller won
+ * the flip, a concurrent dispatch that already reclaimed the row (refreshing `updated_at`) makes the CAS
+ * match zero rows, so `finalize` never runs and the live replacement is never cancelled. The retire to
+ * `failed` is thus finalized only for the lease owner. If `finalize` throws, the transaction rolls back
+ * (the row stays stale-claim `running`) so the pass retries next tick rather than waving a re-dispatch
+ * through into a possible double-launch. */
+export async function reconcileStaleLaunchClaim(
+  data: DataLayer,
+  run: Pick<DeliveryGraphRun, "run_key" | "status" | "process_key" | "updated_at">,
+  finalize?: () => Promise<void>,
+  nowMs: number = Date.now(),
+): Promise<boolean> {
+  // Non-stale rows (launched, terminal, or a claim still within its TTL) are never retired here — the
+  // staleness decision is owned HERE, not re-derived by each caller, so the CAS guard below and this
+  // predicate can never drift apart. This also fences the engine cancel: a fresh in-flight claim never
+  // reaches the flip, so `finalize` (the cancellation) never runs against it.
+  if (!isStaleLaunchClaim(run, nowMs)) return false;
+  const res = await data.open().tx(async (t) => {
+    const flip = await t.exec(
+      `UPDATE "delivery_graph_runs" SET "status" = 'failed', "phase" = ?, "phase_node_id" = NULL, "updated_at" = ? WHERE "run_key" = ? AND "status" = 'running' AND "process_key" IS NULL AND "updated_at" = ?`,
+      [DELIVERY_PHASE.FAILED, new Date(nowMs).toISOString(), run.run_key, run.updated_at],
+    );
+    if (flip.changed !== 1) return false;
+    // We hold the reconciliation lease (the row is now `failed`, non-reclaimable by a dispatch). Run the
+    // fenced engine cancellation BEFORE committing, so a concurrent dispatch can neither be cancelled by
+    // us nor reclaim the row out from under the cancel.
+    await finalize?.();
+    return true;
+  });
+  return res;
+}
+
+/** The engine seam {@link reconcileOriginalInstanceBeforeRelaunch} needs — instance search by
+ * definition, per-instance variable read, and cancel. */
+type ReconcileEngine = Pick<EngineClient, "searchProcessInstances" | "searchVariables" | "cancelInstance">;
+
+/** An ACTIVE instance of this run's content-addressed definition carried no readable RUN-ROOT `runKey`
+ * (absent or truncated), so {@link reconcileOriginalInstanceBeforeRelaunch} could neither cancel it as
+ * ours nor rule it out as a different run. Thrown (only when no candidate positively matched) to fail
+ * CLOSED: the callers leave the stale claim reconcile-pending and retry rather than relaunching a
+ * possible duplicate past an unclassifiable (e.g. pre-upgrade legacy) original. */
+export class ReconcileConflictError extends Error {
+  readonly runKey: string;
+  readonly unclassifiable: string[];
+  constructor(runKey: string, unclassifiable: string[]) {
+    super(`reconcile-before-relaunch: ${unclassifiable.length} ACTIVE instance(s) of run ${runKey}'s definition carry no readable run-root runKey (${unclassifiable.join(", ")}) — cannot prove they are not a live original; failing closed`);
+    this.name = "ReconcileConflictError";
+    this.runKey = runKey;
+    this.unclassifiable = unclassifiable;
+  }
+}
+
+/** Reconcile a still-running ORIGINAL instance of this run before a stale-claim relaunch starts a
+ * second one — the at-most-once recovery the Copilot review on issue #852 asked for (thread
+ * deliveryGraphRun.ts:180).
+ *
+ * A timeout-only reclaim cannot distinguish "the dispatch host died BEFORE `createInstance`" (safe to
+ * reclaim) from "died AFTER `createInstance` succeeded but BEFORE the process key was stamped" (a live
+ * instance is already executing the graph's side-effecting nodes — reclaiming and relaunching would run
+ * them TWICE). Now that the engine seam exposes it (@nanobpm/urban 0.96), close that window: every
+ * launched instance carries its `runKey` as a run-root process variable (`runDeliveryGraph`), so a
+ * relaunch can FIND a live original of THIS run and cancel it before starting its replacement.
+ *
+ * Returns the process keys of any live original instances it cancelled (normally ≤1). A candidate is a
+ * live duplicate iff it is an ACTIVE instance of this content-addressed `processDefinitionId` whose
+ * seeded RUN-ROOT `runKey` variable equals ours (a child-scoped same-named variable is ignored — see
+ * the match below). We never CANCEL an instance we cannot prove belongs to this run.
+ *
+ * Fail CLOSED on an UNCLASSIFIABLE candidate (thread deliveryGraphRun.ts:328): an ACTIVE instance of
+ * this exact content-addressed definition whose RUN-ROOT `runKey` is absent or truncated (unreadable)
+ * cannot be proven to be a DIFFERENT run — a PRE-UPGRADE (legacy) instance launched before this PR
+ * started seeding the correlation variable has exactly this shape, and so does a run whose root value
+ * came back truncated. Treating "unreadable" as "not ours" would fail OPEN: during the upgrade window a
+ * stale claim whose live original is a legacy instance would be relaunched alongside it, double-running
+ * the side-effecting nodes — the very duplicate this reconcile exists to prevent. So when NO candidate
+ * positively matches our run key AND at least one is unclassifiable, we throw {@link
+ * ReconcileConflictError} instead of returning a clean "nothing to cancel". Both callers (dispatch +
+ * poller) treat that throw exactly like a reconcile read failure — the stale claim is left
+ * reconcile-pending and retried — so recovery waits until the unclassifiable original terminates rather
+ * than launching a duplicate past it. When we DID positively identify and cancel our own original, an
+ * unclassifiable sibling is necessarily a DIFFERENT (legacy) run and never blocks our relaunch.
+ *
+ * Accepted limitation (nanobpm/nano-ide#588, PR #853 review): the candidate search below is a
+ * definition+state query, and the `@nanobpm/urban` 0.96 seam returns only a single finite default page
+ * for it — it paginates key-based searches, exposes no cursor, and offers no server-side
+ * variable-correlation filter. So if MORE simultaneously-ACTIVE instances of one byte-identical
+ * content-addressed graph exist than that page size, this run's live original can fall outside the page
+ * and be missed, permitting a duplicate launch. That tail needs BOTH a dispatch crash mid-launch (a
+ * stale claim) AND >page-size concurrent active runs of the exact same graph. Accepted rather than
+ * fixed here because closing it requires extending the engine seam/adapter (cursor pagination or
+ * correlation filtering) — tracked upstream at nanobpm/nano-ide#588. */
+export async function reconcileOriginalInstanceBeforeRelaunch(
+  engine: ReconcileEngine,
+  run: { runKey: string; processDefinitionId: string },
+): Promise<string[]> {
+  const candidates = await engine.searchProcessInstances({
+    processDefinitionId: run.processDefinitionId,
+    state: "ACTIVE",
+  });
+  const cancelled: string[] = [];
+  const unclassifiable: string[] = [];
+  for (const candidate of candidates) {
+    const processInstanceKey = String(candidate.processInstanceKey);
+    // Match the seeded `runKey` variable, not just the definition: two DISTINCT runs of the same graph
+    // (different explicit idempotencyKeys) share one content-addressed `processDefinitionId`, so the
+    // definition filter alone would cancel a DIFFERENT run's live instance. The variable read is the
+    // per-instance proof that this candidate is THIS run.
+    const vars = await engine.searchVariables({ processInstanceKey, name: "runKey" });
+    // Only the RUN-ROOT scope's `runKey` proves ownership. `searchVariables` returns same-named
+    // variables from EVERY scope inside the instance — a delivery graph may legally emit a fact named
+    // `runKey`, and an agent may surface a top-level `runKey` field in its job scope — so a CHILD scope
+    // whose value coincidentally equals this run key would otherwise mark an unrelated live run as ours
+    // and cancel it (thread deliveryGraphRun.ts:318). The run-root variable (seeded by
+    // `runDeliveryGraph`) is the one whose `scopeKey` IS the process instance key; restrict the match
+    // to it so a child-scoped coincidence can never trigger a cancel.
+    const rootVar = vars.find((v) => String(v.scopeKey) === processInstanceKey);
+    // Unreadable run-root `runKey`: absent (a pre-upgrade legacy instance, or a non-run-root candidate)
+    // or TRUNCATED (the engine capped the returned value, so a byte compare would spuriously differ).
+    // Either way we cannot prove this ACTIVE same-definition candidate is a DIFFERENT run, so record it
+    // as unclassifiable and fail closed below rather than fail OPEN by treating it as "not ours" (thread
+    // deliveryGraphRun.ts:328).
+    if (rootVar === undefined || rootVar.isTruncated) {
+      unclassifiable.push(processInstanceKey);
+      continue;
+    }
+    let match: boolean;
+    try {
+      match = JSON.parse(rootVar.value) === run.runKey;
+    } catch {
+      match = rootVar.value === run.runKey;
+    }
+    if (!match) continue;
+    await engine.cancelInstance({ processInstanceKey });
+    cancelled.push(processInstanceKey);
+  }
+  // Fail closed ONLY when we did not positively identify our own original: an unclassifiable candidate
+  // could BE our (legacy) live original, so refuse the blind relaunch. If we cancelled a positive match
+  // we KNOW which instance is ours, so an unclassifiable sibling is a different run and must not block.
+  if (cancelled.length === 0 && unclassifiable.length > 0) {
+    throw new ReconcileConflictError(run.runKey, unclassifiable);
+  }
+  return cancelled;
+}
+
 /** The idempotency key for a submitted graph: a caller-supplied `idempotencyKey` (trimmed) when
  * present and non-blank, else the graph's content `digest`. So two dispatches of the SAME graph (no
  * explicit key) collapse onto one run, and a caller can force a fresh run with an explicit key. */
@@ -212,7 +403,7 @@ function firstLine(text: string | undefined | null): string {
  * Tasks-inbox "Decision context" (issue #813). Storing the full prompt (not a clamped first line) is
  * what lets the operator read the entire instruction, and lets a URL embedded in the prompt reach the
  * task's clickable link. Falls back to the node id when a human node declares no prompt. */
-export function buildHumanLabels(compiled: CompileDeliveryGraphResult): Record<string, string> {
+export function buildHumanLabels(compiled: Pick<CompileDeliveryGraphResult, "resolved" | "humanNodes">): Record<string, string> {
   const elementByNodeId = new Map(compiled.resolved.nodes.map((n) => [n.id, n.element]));
   const labels: Record<string, string> = {};
   for (const stop of compiled.humanNodes) {

package/app/deliveryRunner.test.ts

@@ -485,6 +485,28 @@ test("runDeliveryGraph coerces a numeric engine processInstanceKey to a string h
   assertEquals(typeof r.handle.processInstanceKey, "string");
 });
 
+test("runDeliveryGraph seeds the prepared runKey as the createInstance correlation variable (stale-claim recovery handle)", async () => {
+  // Regression guard (Copilot review 5408224485, "previously missed"): the stale-claim recovery in
+  // `reconcileOriginalInstanceBeforeRelaunch` finds a still-running ORIGINAL instance by correlating on
+  // the engine `runKey` variable. The reconciliation tests stub `searchVariables` independently, so
+  // deleting this seed would leave every recovery test green while production silently misses the
+  // original instance and relaunches a duplicate. Capture the `variables` argument and assert the
+  // seeded `runKey` IS the prepared/returned run key.
+  let captured: Record<string, unknown> | undefined;
+  const engine = {
+    deployResources: async () => [],
+    createInstance: async (args: { variables: Record<string, unknown> }) => {
+      captured = args.variables;
+      return { processInstanceKey: "555" };
+    },
+  };
+  const r = await runDeliveryGraph(engine, GRAPH, { repoless: true });
+  assert(r.ok, `expected ok:true, got ${JSON.stringify(r)}`);
+  assert(captured, "createInstance was called with a variables argument");
+  assert(r.handle.runKey, "the prepared run key is non-empty");
+  assertEquals(captured.runKey, r.handle.runKey, "the engine correlation variable carries the prepared run key");
+});
+
 // Per-node repository isolation (issue #739): the delivery-graph runner seeds the canonical
 // `io.nanobpm.agentTask.repository` isolation envelope PER agent cell as flattened `<zeebe:taskHeaders>`
 // (`io.nanobpm.agentTask.repository.url`, …) injected into the deployable BPMN — NOT as a single