@nanobpm/nano-workforce 0.190.0 → 0.191.1

package/app/deliveryGraphDispatch.test.ts

@@ -6,15 +6,27 @@
 // graph's side effects. These tests drive it against an in-memory app/data/engine faithful to the run
 // aggregate's PRIMARY KEY fence and the guarded raw UPDATE the claim issues.
 import { test } from "node:test";
-import { assert, assertEquals } from "#test-assert";
+import { assert, assertEquals, assertRejects } from "#test-assert";
 import type { AppApi } from "@nanobpm/urban";
 import { dispatchDeliveryGraphRun } from "./deliveryGraphDispatch.ts";
 import { noopLog } from "../test/log.ts";
 
-function makeApp() {
+function makeApp(opts: { failIdentityWrite?: boolean } = {}) {
   const tables = new Map<string, Record<string, unknown>[]>();
   const started: { processDefinitionId: string }[] = [];
   const table = (name: string, key: string) => {
+    if (opts.failIdentityWrite && name === "delivery_graph_run_identity") {
+      // Simulate a transient side-table write error (e.g. a SQLite `disk I/O error`) — NOT a UNIQUE
+      // collision, so `upsertRunIdentity` rethrows it rather than folding it into an UPDATE.
+      return {
+        get: () => Promise.resolve(null),
+        find: () => Promise.resolve([]),
+        all: () => Promise.resolve([]),
+        insert: () => Promise.reject(new Error("disk I/O error")),
+        update: () => Promise.reject(new Error("disk I/O error")),
+        delete: () => Promise.resolve(),
+      };
+    }
     const rows =
       tables.get(name) ??
       (() => {
@@ -101,6 +113,39 @@ test("dispatchDeliveryGraphRun: a human-only graph launches straight away (runni
   assertEquals(runs()[0].status, "running");
 });
 
+test("#778 dispatchDeliveryGraphRun: a post-claim identity-write failure flips the claimed run to `failed`, not a stranded null-process_key `running` — so a phantom run never short-circuits a later dispatch (thread deliveryGraphDispatch.ts:229)", async () => {
+  const { app, started, runs } = makeApp({ failIdentityWrite: true });
+  // The identity side-table write throws AFTER the durable `running` claim; dispatch must propagate it.
+  await assertRejects(() => dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { repoless: true }), Error, "disk I/O error");
+  // The claimed row was flipped to `failed` (never left `running` with a null process key), so a later
+  // same-key dispatch does NOT short-circuit onto a phantom run — it can retry cleanly.
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0].status, "failed");
+  assertEquals(runs()[0].process_key ?? null, null);
+  // The side effect never launched.
+  assertEquals(started.length, 0);
+});
+
+test("#778 dispatchDeliveryGraphRun: a still-running run whose identity row is MISSING (null, pre-migration) short-circuits with identityConfirmed:false instead of throwing (thread deliveryGraphDispatch.ts:175)", async () => {
+  const { app, started } = makeApp();
+  const first = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { repoless: true });
+  assert(first.ok);
+  if (!first.ok) return;
+  assertEquals(first.status, "running");
+  // Simulate a run launched BEFORE the identity side-table existed: drop its identity row so
+  // `identities.get(runKey)` resolves to null (the test table — like the real `table.get()` — returns
+  // null, NOT undefined, for a missing row).
+  await app.data.table("delivery_graph_run_identity", "run_key").delete(first.runKey);
+  // A second same-key dispatch hits the already-running short-circuit; the null identity row must not
+  // throw while building the response — it is unprovable, so identityConfirmed is false.
+  const second = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { repoless: true });
+  assert(second.ok);
+  if (!second.ok) return;
+  assertEquals(second.alreadyRunning, true);
+  assertEquals(second.identityConfirmed, false);
+  assertEquals(started.length, 1); // no double launch
+});
+
 test("dispatchDeliveryGraphRun: a side-effecting graph dispatches with NO approval token — the operator seam IS the approval", async () => {
   const { app, started, runs } = makeApp();
   const res = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { repoless: true });
@@ -130,6 +175,35 @@ test("dispatchDeliveryGraphRun: an explicit idempotency key forces a distinct ru
   assertEquals(started.length, 2);
 });
 
+test("#778 dispatchDeliveryGraphRun: a recompiled digest that drifts from `expectedDigest` is REFUSED before any launch — no run, no engine instance (thread :1605)", async () => {
+  // A staged proposal is keyed by its stage-time digest. If the compiler ships a digest-affecting change
+  // (this PR adds labels/`<bpmn:documentation>`), the stored graph recompiles to a DIFFERENT digest. The
+  // door hands the stage-time digest as `expectedDigest`; a mismatch must refuse cleanly BEFORE the
+  // durable launch claim, so we never strand a live run against a proposal left `staged`.
+  const { app, started, runs } = makeApp();
+  const res = await dispatchDeliveryGraphRun(app, SIDE_EFFECTING, { repoless: true, expectedDigest: "sha256-stale-address" });
+  assertEquals(res.ok, false);
+  if (res.ok) return;
+  assert(res.errors.some((e) => e.path === "digest"), "the refusal is a digest-address error");
+  assertEquals(started.length, 0); // nothing launched
+  assertEquals(runs().length, 0); // no run row claimed
+});
+
+test("#778 dispatchDeliveryGraphRun: a matching `expectedDigest` dispatches normally (the same-compiler no-op path)", async () => {
+  // Compute the graph's true digest first, then pass it as `expectedDigest` — the normal deterministic
+  // recompile matches, so the address check is a no-op and the run launches.
+  const { app, started } = makeApp();
+  const probe = await dispatchDeliveryGraphRun(app, HUMAN_ONLY, { repoless: true });
+  assert(probe.ok);
+  if (!probe.ok) return;
+  const { app: app2, started: started2 } = makeApp();
+  const res = await dispatchDeliveryGraphRun(app2, HUMAN_ONLY, { repoless: true, expectedDigest: probe.digest });
+  assertEquals(res.ok, true);
+  if (!res.ok) return;
+  assertEquals(res.status, "running");
+  assertEquals(started2.length, 1);
+});
+
 test("dispatchDeliveryGraphRun: a malformed graph → ok:false with path-qualified errors, nothing launched", async () => {
   const { app, started } = makeApp();
   const res = await dispatchDeliveryGraphRun(app, { name: "empty", nodes: [] });
@@ -141,3 +215,45 @@ test("dispatchDeliveryGraphRun: a malformed graph → ok:false with path-qualifi
   }
   assertEquals(started.length, 0);
 });
+
+// Option C (issue #778): a graph carrying redacted-away credential material has a LOSSY content digest
+// (a sibling differing only in the secret would collide), so a keyless dispatch — which defaults the
+// run identity to that digest — is refused; an explicit `idempotencyKey` disambiguates it.
+const SECRET_BEARING = {
+  name: "deploy with a secret",
+  nodes: [{ id: "deploy", kind: "agent", agent: { jobType: "senior:demo", prompt: "push to https://user:s3cr3t@host.example/repo" } }],
+};
+
+test("dispatchDeliveryGraphRun: a secret-bearing graph dispatched KEYLESS is refused, points at idempotencyKey, launches nothing", async () => {
+  const { app, started, runs } = makeApp();
+  const res = await dispatchDeliveryGraphRun(app, SECRET_BEARING, { repoless: true });
+  assertEquals(res.ok, false);
+  if (res.ok) return;
+  assertEquals(res.errors.length, 1);
+  assertEquals(res.errors[0].path, "idempotencyKey");
+  assert(res.errors[0].message.includes("idempotencyKey"));
+  assertEquals(started.length, 0);
+  assertEquals(runs().length, 0); // nothing claimed
+});
+
+test("dispatchDeliveryGraphRun: the SAME secret-bearing graph WITH an explicit idempotencyKey launches", async () => {
+  const { app, started, runs } = makeApp();
+  const res = await dispatchDeliveryGraphRun(app, SECRET_BEARING, { runKey: "deploy-2024-06-a", repoless: true });
+  assertEquals(res.ok, true);
+  if (!res.ok) return;
+  assertEquals(res.status, "running");
+  assertEquals(res.runKey, "deploy-2024-06-a");
+  assertEquals(started.length, 1);
+  assertEquals(runs()[0].run_key, "deploy-2024-06-a");
+});
+
+test("dispatchDeliveryGraphRun: two credential-differing secret graphs with DISTINCT keys get distinct runs (no collision)", async () => {
+  const { app, started, runs } = makeApp();
+  const graphA = { name: "deploy", nodes: [{ id: "d", kind: "agent", agent: { jobType: "senior:demo", prompt: "push to https://user:AAA@host.example/repo" } }] };
+  const graphB = { name: "deploy", nodes: [{ id: "d", kind: "agent", agent: { jobType: "senior:demo", prompt: "push to https://user:BBB@host.example/repo" } }] };
+  const a = await dispatchDeliveryGraphRun(app, graphA, { runKey: "run-a", repoless: true });
+  const b = await dispatchDeliveryGraphRun(app, graphB, { runKey: "run-b", repoless: true });
+  assert(a.ok && b.ok);
+  assertEquals(started.length, 2);
+  assertEquals(new Set(runs().map((r) => r.run_key)).size, 2);
+});

package/app/deliveryGraphDispatch.ts

@@ -11,16 +11,19 @@
 // idempotency short-circuit, so a double-click or a re-dispatch never double-launches a graph's side
 // effects.
 
+import { createHash } from "node:crypto";
 import type { AppApi } from "@nanobpm/urban";
 import type { DeliveryGraph } from "../nano-generated/api-io.d.ts";
-import { validateDeliveryGraph } from "./deliveryGraph.ts";
-import { compileDeliveryGraph } from "./deliveryGraphCompiler.ts";
+import { canonicalJson, validateDeliveryGraph } from "./deliveryGraph.ts";
+import { compileDeliveryGraph, digestInvisibleRawValues, graphCarriesRedactedSecrets } from "./deliveryGraphCompiler.ts";
 import {
   buildDeliveryGraphRunRow,
   buildHumanLabels,
   claimRunForLaunch,
   computeRunKey,
   DELIVERY_PHASE,
+  type DeliveryGraphRunIdentity,
+  deliveryGraphRunIdentities,
   deliveryGraphRuns,
 } from "./deliveryGraphRun.ts";
 import type { DeliveryRunTimeouts } from "./deliveryRunner.ts";
@@ -37,11 +40,47 @@ export type DispatchDeliveryGraphResult =
       digest: string;
       sideEffecting: boolean;
       alreadyRunning: boolean;
+      /** Whether the returned run is PROVABLY this exact graph. Always true for a fresh launch. For an
+       *  `alreadyRunning` short-circuit it is true iff the running row's persisted lossless
+       *  {@link deliveryGraphIdentityFingerprint} matches the incoming graph's — i.e. a same-payload
+       *  retry, not a credential-different graph re-staged under the same explicit `idempotencyKey` (and
+       *  false for a pre-migration-113 row whose fingerprint is NULL, an unprovable identity). The
+       *  dispatch door refuses a secret-bearing short-circuit only when this is false (issue #778 review
+       *  — thread dispatchDeliveryGraph.ts:332). */
+      identityConfirmed: boolean;
       processInstanceKey?: string;
       processDefinitionId?: string;
     }
   | { ok: false; errors: { path: string; message: string }[] };
 
+/** The LOSSLESS content-identity fingerprint of a graph — `sha256(digest \0
+ * canonicalJson(digestInvisibleRawValues(graph)))`. The `digest` is content-addressed over the REDACTED
+ * `semanticBpmn`, so it alone collapses secret-differing graphs onto one identity; folding in
+ * {@link digestInvisibleRawValues} (the exact raw content the digest CANNOT see) restores a FULL identity
+ * that distinguishes them — the SAME `(digest, invisible-values)` pair `stableProposalRunKey` keys the
+ * keyless run key on. Persisted on the run row so an explicit-`idempotencyKey` short-circuit can prove
+ * the running run is THIS exact graph (issue #778 review — thread dispatchDeliveryGraph.ts:332). */
+export function deliveryGraphIdentityFingerprint(graph: DeliveryGraph, digest: string): string {
+  const invisible = canonicalJson(digestInvisibleRawValues(graph));
+  return createHash("sha256").update(`${digest}\u0000${invisible}`).digest("hex");
+}
+
+/** Persist a run's lossless identity fingerprint (upsert on `run_key`). A relaunch off a persisted row
+ * under an explicit `idempotencyKey` may carry a DIFFERENT graph, so a PRIMARY-KEY fence collision folds
+ * into an UPDATE that overwrites the stored fingerprint rather than surfacing — the same intended
+ * outcome as the insert branch (mirrors `DurableResumeRegistry.recordEnrolment`). */
+async function upsertRunIdentity(
+  identities: ReturnType<typeof deliveryGraphRunIdentities>,
+  row: DeliveryGraphRunIdentity,
+): Promise<void> {
+  try {
+    await identities.insert(row);
+  } catch (err) {
+    if (!(err instanceof Error) || !/UNIQUE constraint failed/i.test(err.message)) throw err;
+    await identities.update(row.run_key, { graph_fingerprint: row.graph_fingerprint });
+  }
+}
+
 /** Dispatch a delivery graph as a running engine-native process — the operator action. Re-validates
  * and re-compiles the (already-staged) graph to derive its content digest + run-row shape, then
  * launches it through the durable at-most-once fence. Idempotent: a re-dispatch of an already-running
@@ -49,7 +88,7 @@ export type DispatchDeliveryGraphResult =
 export async function dispatchDeliveryGraphRun(
   app: Pick<AppApi, "data" | "engine" | "log">,
   graph: unknown,
-  options: { runKey?: string | null; title?: string | null; repository?: string | null; baseBranch?: string | null; repoless?: boolean } & DeliveryRunTimeouts = {},
+  options: { runKey?: string | null; title?: string | null; repository?: string | null; baseBranch?: string | null; repoless?: boolean; expectedDigest?: string | null } & DeliveryRunTimeouts = {},
 ): Promise<DispatchDeliveryGraphResult> {
   const validationErrors = validateDeliveryGraph(graph);
   if (validationErrors.length > 0) {
@@ -64,12 +103,82 @@ export async function dispatchDeliveryGraphRun(
   }
 
   const digest = deliveryGraphDigest(compiled.semanticBpmn);
+  // ADDRESS-BEFORE-LAUNCH (issue #778 review — thread deliveryGraphCompiler.ts:1605): a staged proposal
+  // is keyed by the digest computed AT STAGE TIME. The labels/`<bpmn:documentation>` this PR adds are
+  // covered by that content digest, so after this compiler ships every proposal staged by the PREVIOUS
+  // compiler RECOMPILES to a different digest. The caller (the dispatch door) hands us that stage-time
+  // digest as `expectedDigest`; if the fresh recompile no longer matches it, the stored graph pre-dates
+  // this compiler and its content address has drifted. REFUSE here — BEFORE the durable launch claim —
+  // so we never launch the run and THEN report a post-launch 409 mismatch, which stranded a live run
+  // against a proposal left `staged`. A clean refusal leaves nothing running; the operator recompiles to
+  // re-stage under the current compiler. (A normal, same-compiler dispatch recompiles deterministically
+  // to the same digest, so this is a no-op there.)
+  const expectedDigest = typeof options.expectedDigest === "string" && options.expectedDigest.trim() !== "" ? options.expectedDigest.trim() : "";
+  if (expectedDigest !== "" && digest !== expectedDigest) {
+    app.log.warn("dispatch-delivery-graph refused: recompiled digest drifted from the staged address", { expectedDigest, digest });
+    return {
+      ok: false,
+      errors: [
+        {
+          path: "digest",
+          message:
+            `this staged proposal was compiled under a different compiler version — it now recompiles to digest ${digest}, ` +
+            `not the staged ${expectedDigest} its content address was pinned to, so it was NOT launched; recompile the graph to ` +
+            "re-stage it under the current compiler before dispatching (issue #778)",
+        },
+      ],
+    };
+  }
+  const explicitKey = typeof options.runKey === "string" && options.runKey.trim() !== "";
+  // Option C (issue #778): the graph's identity is content-addressed over the REDACTED `semanticBpmn`
+  // (issue #716), so two graphs differing ONLY in a redacted-away credential (a URL secret/`?query`/
+  // `#fragment`, a `command` target, a `verifyCommand`/`bodyIncludes`/`stdoutIncludes` match secret, or
+  // a free-form connector `payload`) share one digest — and a KEYLESS dispatch defaults `runKey` to that
+  // digest, silently collapsing the second onto the first's still-running instance and reusing its
+  // config. We accept the collision as by-design but REQUIRE an explicit `idempotencyKey` to
+  // disambiguate such a secret-bearing graph, rather than launch it under an ambiguous identity.
+  if (!explicitKey && graphCarriesRedactedSecrets(typedGraph)) {
+    app.log.warn("dispatch-delivery-graph refused: secret-bearing graph needs an explicit idempotencyKey", { digest });
+    return {
+      ok: false,
+      errors: [
+        {
+          path: "idempotencyKey",
+          message:
+            "this graph carries runtime values that redaction/normalisation strips from its content-addressed " +
+            "digest — a URL credential/`?query`/`#fragment`, a `command` target, a `verifyCommand`/`bodyIncludes`/" +
+            "`stdoutIncludes` match secret, a free-form connector `payload`, an untrimmed-whitespace difference in " +
+            "an agent/human `prompt`, or a value that loses characters to XML sanitisation — so its digest cannot " +
+            "distinguish it from another graph differing only in those values; supply an explicit `idempotencyKey` " +
+            "to dispatch it (issue #778)",
+        },
+      ],
+    };
+  }
   const runKey = computeRunKey(options.runKey, digest);
   const sideEffecting = compiled.sideEffects.length > 0;
   const explicitTitle = typeof options.title === "string" && options.title.trim() !== "" ? options.title.trim() : "";
   const graphName = typeof typedGraph.name === "string" && typedGraph.name.trim() !== "" ? typedGraph.name.trim() : "";
   const title = explicitTitle || graphName || runKey;
   const runs = deliveryGraphRuns(app.data);
+  const identities = deliveryGraphRunIdentities(app.data);
+  // The lossless identity of THIS graph — persisted at launch in the `delivery_graph_run_identity` side
+  // table, compared on a short-circuit so the door can distinguish a same-payload retry from a
+  // credential-different graph re-staged under the same explicit key (issue #778 review — thread
+  // dispatchDeliveryGraph.ts:332).
+  const graphFingerprint = deliveryGraphIdentityFingerprint(typedGraph, digest);
+  // Whether the run at `runKey` is PROVABLY this exact graph: a stored identity row that MATCHES ours. A
+  // MISSING row (a run launched before this table existed, or a not-yet-stamped concurrent claim) is
+  // unprovable → false, so the door refuses a secret-bearing short-circuit (the safe pre-existing 409).
+  const identityConfirmed = async (): Promise<boolean> => {
+    const identity = await identities.get(runKey);
+    // `table.get()` resolves to `null` (NOT `undefined`) for a MISSING row — a run launched before this
+    // side table existed, or a not-yet-stamped concurrent claim. A bare `!== undefined` admits that
+    // `null` and then throws on `.graph_fingerprint`; `!= null` treats both absences as unprovable →
+    // false, so the door reaches the safe pre-existing 409 instead of an incident (issue #778 review —
+    // thread deliveryGraphDispatch.ts:175).
+    return identity != null && identity.graph_fingerprint === graphFingerprint;
+  };
 
   // Idempotency short-circuit — a re-dispatch onto a still-running run does NOT double-launch.
   const existing = await runs.get(runKey);
@@ -82,6 +191,7 @@ export async function dispatchDeliveryGraphRun(
       digest: existing.digest,
       sideEffecting: existing.side_effecting === 1,
       alreadyRunning: true,
+      identityConfirmed: await identityConfirmed(),
       processInstanceKey: existing.process_key ?? undefined,
       processDefinitionId: existing.process_definition_id ?? undefined,
     };
@@ -113,22 +223,39 @@ export async function dispatchDeliveryGraphRun(
       digest: won?.digest ?? digest,
       sideEffecting: won ? won.side_effecting === 1 : sideEffecting,
       alreadyRunning: true,
+      identityConfirmed: await identityConfirmed(),
       processInstanceKey: won?.process_key ?? undefined,
       processDefinitionId: won?.process_definition_id ?? undefined,
     };
   }
-  if (existing) {
-    const { run_key, created_at, ...patch } = claim;
-    await runs.update(runKey, patch);
-  }
-
-  // Launch — deploy + start the compiled definition. On ANY launch failure flip the claimed row to
-  // `failed` so no null-process_key `running` row is ever stranded.
+  // On ANY post-claim failure — the identity/patch persistence just below OR the launch further down —
+  // flip the claimed row to `failed` so no null-process_key `running` row is ever stranded and later
+  // dispatches never short-circuit onto a phantom run.
   const markClaimFailed = async () => {
     const failed = buildDeliveryGraphRunRow({ ...rowBase, status: "failed", phase: DELIVERY_PHASE.FAILED, processKey: null });
     const { run_key, created_at, ...patch } = failed;
     await runs.update(runKey, patch);
   };
+  // The claim is ours — persist THIS run's lossless identity so a later same-key short-circuit can prove
+  // the running run is this exact graph. Upsert: a relaunch off a persisted (e.g. failed) row under an
+  // explicit key may carry a different graph, so overwrite the stored fingerprint. This runs AFTER the
+  // durable `running` claim but is a SEPARATE write, so a transient failure here would otherwise strand
+  // the claimed row as `running` with a null process key — later same-key dispatches would then
+  // short-circuit onto that phantom run. Route the failure through the same claim-failure cleanup before
+  // rethrowing so the phantom self-heals (issue #778 review — thread deliveryGraphDispatch.ts:229).
+  try {
+    await upsertRunIdentity(identities, { run_key: runKey, graph_fingerprint: graphFingerprint, created_at: claim.created_at });
+    if (existing) {
+      const { run_key, created_at, ...patch } = claim;
+      await runs.update(runKey, patch);
+    }
+  } catch (err) {
+    await markClaimFailed();
+    app.log.error("dispatch-delivery-graph identity persistence threw", { runKey });
+    throw err;
+  }
+
+  // Launch — deploy + start the compiled definition.
   let launched: Awaited<ReturnType<typeof runDeliveryGraph>>;
   try {
     // Thread the operator-supplied run-level timeouts (#505) so a submission override reaches every
@@ -180,6 +307,7 @@ export async function dispatchDeliveryGraphRun(
     digest,
     sideEffecting,
     alreadyRunning: false,
+    identityConfirmed: true,
     processInstanceKey: launched.handle.processInstanceKey,
     processDefinitionId: launched.handle.processDefinitionId,
   };

package/app/deliveryGraphProposals.test.ts

@@ -17,6 +17,7 @@ import {
   getStagedProposal,
   isLiveStaged,
   isProposalExpired,
+  listStagedProposals,
   markProposalDismissed,
   markProposalDispatched,
   markProposalExpired,
@@ -157,6 +158,35 @@ test("stageProposal: a re-stage of an EXPIRED digest RE-ANCHORS the TTL so it is
   });
 });
 
+test("stageProposal: a re-stage derives the TTL from the row COMMITTED at write time, so a stale caller cannot clobber a concurrently re-anchored live proposal back to an expired horizon (issue #778 review — thread :247)", async () => {
+  await withData(async (data) => {
+    const table = deliveryGraphProposals(data);
+    // Model the committed state a CONCURRENT re-stage left behind: digest d1 is LIVE with a FRESH TTL
+    // (`expires_at` far in the future) but its `created_at` anchor is old — exactly the row a rival stage
+    // produced when it re-anchored the TTL. The stale caller below prepared its write from an EARLIER read
+    // and now commits LAST (grabbing the newest `stage_seq`). The pre-transaction snapshot bug recomputed
+    // `expires_at` from the stale `created_at` (`created_at + TTL`, long past), so the last writer resurrected
+    // an EXPIRED horizon over the fresh one and left the newest-`stage_seq` row non-dispatchable. The TTL must
+    // instead be derived from the row as committed AT WRITE TIME, inside the same atomic fence as `stage_seq`.
+    const seed = row({ digest: "d1", createdAt: "2020-01-01T00:00:00.000Z" });
+    seed.expires_at = "2999-01-01T00:00:00.000Z"; // fresh TTL a concurrent re-anchor committed
+    seed.stage_seq = 1;
+    await table.insert(seed);
+
+    await stageProposal(data, row({ digest: "d1", createdAt: "2020-01-01T00:00:00.000Z" }));
+
+    const after = await table.get("d1");
+    assert(after, "the re-staged row must still exist");
+    assert(
+      !isProposalExpired(after?.expires_at),
+      "a re-stage must not clobber a live proposal's fresh TTL with an expired horizon recomputed from a stale created_at snapshot",
+    );
+    const live = await getStagedProposal(data, "d1");
+    assert(live, "the re-staged live proposal remains dispatchable");
+    assertEquals(live?.status, "staged");
+  });
+});
+
 test("stageProposal: a new digest for the SAME logical key supersedes the prior staged proposal", async () => {
   await withData(async (data) => {
     await stageProposal(data, row({ digest: "d1" }));
@@ -241,6 +271,60 @@ test("stageProposal: reconciles to EXACTLY ONE live proposal — an older stage
   });
 });
 
+test("stageProposal: a RE-STAGE wins a same-millisecond `updated_at` tie over an untouched sibling — the monotonic `stage_seq` reflects the re-stage that a frozen `rowid` does not (issue #778 review — thread :245)", async (t) => {
+  await withData(async (data) => {
+    const table = deliveryGraphProposals(data);
+    // Freeze wall-clock time so every `now()` inside `stageProposal` stamps the SAME `updated_at`,
+    // reproducing the same-millisecond concurrent-stage race deterministically (only the `Date` API is
+    // mocked, so async DB I/O on real timers is unaffected).
+    t.mock.timers.enable({ apis: ["Date"], now: Date.parse("2025-06-01T00:00:00.000Z") });
+    // Two staged siblings under logical_key "runbook", both stamped the frozen instant. d2 is staged
+    // AFTER d1, so it is (correctly) the live winner and d1 is superseded — `rowid` and `stage_seq` agree
+    // so far (insertion order == stage order).
+    await stageProposal(data, row({ digest: "d1" }));
+    await stageProposal(data, row({ digest: "d2" }));
+    assertEquals((await table.get("d1"))?.status, "superseded");
+    assertEquals((await table.get("d2"))?.status, "staged");
+    // Now RE-STAGE d1 (an UPDATE) at the SAME frozen instant. This is the newest write, so it MUST become
+    // the sole live proposal. But a re-stage is an `UPDATE`: d1's `rowid` stays frozen BELOW d2's, so the
+    // old `rowid` tie-break wrongly keeps d2 live and leaves the just-re-staged d1 superseded. The
+    // monotonic `stage_seq` (reassigned MAX+1 on every write, including this re-stage) advances past d2,
+    // so the reconcile correctly makes d1 live and supersedes d2.
+    await stageProposal(data, row({ digest: "d1" }));
+    assertEquals(
+      (await table.get("d1"))?.status,
+      "staged",
+      "the just-re-staged digest must be the live proposal, even against an untouched higher-rowid sibling",
+    );
+    assertEquals((await table.get("d2"))?.status, "superseded", "the untouched older sibling is superseded by the re-stage");
+    const live = (await table.all()).filter((r) => r.status === "staged" && r.logical_key === "runbook");
+    assertEquals(live.length, 1, "exactly one live proposal per logical graph after the re-stage");
+    assertEquals(live[0]?.digest, "d1", "the re-staged digest wins the same-`updated_at` tie");
+  });
+});
+
+test("stageProposal: assigns `stage_seq` ATOMICALLY in the write — a fresh stage is strictly > 0 and monotonic, so there is never a DEFAULT-0 window a concurrent reconcile could mis-order (issue #778 review — thread :237/:239)", async () => {
+  await withData(async (data) => {
+    const seqOf = async (digest: string): Promise<number> => {
+      const rows = await data.open().query<{ stage_seq: number }>(
+        `SELECT "stage_seq" FROM "delivery_graph_proposals" WHERE "digest" = ?`,
+        [digest],
+      );
+      return rows.length ? Number(rows[0].stage_seq) : -1;
+    };
+    // A first stage: the seq is assigned in the INSERT itself (MAX+1), never left at the column DEFAULT 0.
+    await stageProposal(data, row({ digest: "d1", logicalKey: "k-a" }));
+    const s1 = await seqOf("d1");
+    assert(s1 > 0, `a fresh stage carries a strictly-positive stage_seq (atomic, no 0 window): ${s1}`);
+    // A second, independent stage is strictly greater — the monotonic secondary ordering.
+    await stageProposal(data, row({ digest: "d2", logicalKey: "k-b" }));
+    assert((await seqOf("d2")) > s1, "a later stage's seq is strictly greater");
+    // A RE-STAGE (UPDATE) also advances the seq — it is reassigned on every write, not frozen like rowid.
+    await stageProposal(data, row({ digest: "d1", logicalKey: "k-a" }));
+    assert((await seqOf("d1")) > (await seqOf("d2")), "a re-stage advances stage_seq past every prior write");
+  });
+});
+
 test("stageProposal outcome: `row` reflects the POST-reconcile status — a stage immediately superseded by a newer sibling reports its own row as `superseded`, not the pre-reconcile `staged`", async () => {
   await withData(async (data) => {
     const table = deliveryGraphProposals(data);
@@ -280,6 +364,88 @@ test("markProposalDispatched: a dispatched proposal is no longer live", async ()
   });
 });
 
+test("markProposalDispatched: flipping a live `staged` row returns true; an already-terminal (dispatched) row returns false (guarded, mirrors markProposalDismissed/Expired)", async () => {
+  await withData(async (data) => {
+    await stageProposal(data, row());
+    assertEquals(await markProposalDispatched(data, "d1"), true);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "dispatched");
+    // A second consume of the same digest (already dispatched) must no-op and report the lost race.
+    assertEquals(await markProposalDispatched(data, "d1"), false);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "dispatched");
+  });
+});
+
+test("markProposalDispatched: a concurrent RE-STAGE (bumping stage_seq) after the dispatch snapshot makes the seq-guarded mark a no-op — the newer, never-launched graph stays `staged` (#778 review)", async () => {
+  await withData(async (data) => {
+    // Dispatch snapshots the proposal's stage_seq when it loads the row.
+    await stageProposal(data, row());
+    const snapshot = await getStagedProposal(data, "d1");
+    assert(snapshot, "the freshly staged proposal is live");
+    const snapSeq = snapshot.stage_seq;
+    // A concurrent re-stage overwrites the SAME digest in place with a newer graph (two credential-
+    // differing graphs share a semantic digest), bumping stage_seq past the dispatch's snapshot.
+    await stageProposal(data, row());
+    const reStaged = await deliveryGraphProposals(data).get("d1");
+    assert(reStaged && reStaged.stage_seq > snapSeq, "the re-stage bumped stage_seq past the snapshot");
+    // The dispatch marks with the STALE snapshot seq → the `stage_seq` guard fails → no row flips, so the
+    // newer graph is NOT retired to `dispatched`; it stays `staged` for its own dispatch.
+    assertEquals(await markProposalDispatched(data, "d1", snapSeq), false);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "staged");
+    // Marking with the CURRENT seq consumes it as normal.
+    assertEquals(await markProposalDispatched(data, "d1", reStaged.stage_seq), true);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "dispatched");
+  });
+});
+
+test("stageProposal: two concurrent FIRST stages of the SAME digest do NOT primary-key-conflict — the raced insert collapses into an idempotent UPSERT (#778 review — thread :245)", async () => {
+  await withData(async (data) => {
+    // Reproduce the race deterministically: our stage reads `existing = null` (no row yet), then — in the
+    // window BEFORE our transaction commits — a COMPETING first-stage of the same digest lands its row.
+    // The old read-then-branch INSERT would make our write hit a UNIQUE(digest) conflict and THROW,
+    // dropping our reconcile; the `ON CONFLICT("digest") DO UPDATE` upsert must instead collapse it.
+    let injected = false;
+    const racyData = new Proxy(data, {
+      get(target, prop) {
+        if (prop === "open") {
+          return () => {
+            const src = target.open();
+            return new Proxy(src, {
+              get(s, p) {
+                if (p === "tx") {
+                  return async (fn: (t: unknown) => Promise<void>) => {
+                    if (!injected) {
+                      injected = true;
+                      // A competing first-stage commits its row before ours does (real data layer).
+                      await stageProposal(data, row());
+                    }
+                    // @ts-expect-error the wrapped tx forwards the original callback unchanged
+                    return s.tx(fn);
+                  };
+                }
+                const v = Reflect.get(s, p, s);
+                return typeof v === "function" ? v.bind(s) : v;
+              },
+            });
+          };
+        }
+        const v = Reflect.get(target, prop, target);
+        return typeof v === "function" ? v.bind(target) : v;
+      },
+    });
+
+    // Must NOT throw despite our `existing = null` read racing the competitor's committed insert.
+    await stageProposal(racyData as DataLayer, row());
+    assert(injected, "the competing first-stage should have fired");
+    const finalRow = await deliveryGraphProposals(data).get("d1");
+    assert(finalRow, "exactly one row exists for the digest");
+    assertEquals(finalRow.status, "staged");
+    // The upsert collapsed onto the competitor's row (an UPDATE, not a second INSERT), so stage_seq
+    // advanced past the competitor's — proving no PK conflict and a single reconciled row.
+    assert(finalRow.stage_seq >= 2, `the raced insert collapsed into an upsert (stage_seq advanced): ${finalRow.stage_seq}`);
+    assertEquals((await listStagedProposals(data)).length, 1);
+  });
+});
+
 test("sweepExpiredProposals: flips aged-out staged proposals to `expired` so they drop out of the cockpit grid", async () => {
   await withData(async (data) => {
     // Two staged proposals with different logical keys so neither supersedes the other.
@@ -351,6 +517,59 @@ test("sweepExpiredProposals: a dispatch racing between the read and the write is
   });
 });
 
+test("sweepExpiredProposals: a re-stage racing between the read and the write is NOT expired — the `stage_seq` guard no-ops the stale flip so the fresh revision survives (#778 review — thread deliveryGraphProposals.ts:417)", async () => {
+  await withData(async (data) => {
+    const seqOf = async (digest: string): Promise<number> => {
+      const rows = await data.open().query<{ stage_seq: number }>(`SELECT "stage_seq" FROM "delivery_graph_proposals" WHERE "digest" = ?`, [digest]);
+      return rows.length ? Number(rows[0].stage_seq) : -1;
+    };
+    // One aged-out staged proposal — the sweep's `find()` will see it as `staged` at revision seq0.
+    await stageProposal(data, row({ digest: "d1", createdAt: "2024-01-01T00:00:00.000Z" }));
+    const seq0 = await seqOf("d1");
+    const at = new Date(Date.parse("2024-01-01T00:00:00.000Z") + DELIVERY_PROPOSAL_TTL_MS + 1000);
+
+    // Wrap the data layer so that, in the window between the sweep's `find()` and its per-row guarded
+    // `exec`, a concurrent RE-STAGE overwrites d1 in place with a fresh, future-TTL revision (bumping
+    // stage_seq past seq0). A blind `WHERE digest=? AND status='staged'` flip would expire that never-swept
+    // fresh revision; the `stage_seq`-guarded UPDATE must instead no-op and leave the row live `staged`.
+    let raced = false;
+    const racyData = new Proxy(data, {
+      get(target, prop, receiver) {
+        if (prop === "open") {
+          return () => {
+            const src = target.open();
+            return new Proxy(src, {
+              get(s, p) {
+                if (p === "exec") {
+                  return async (sql: string, params?: unknown[]) => {
+                    if (!raced) {
+                      raced = true;
+                      await stageProposal(data, row({ digest: "d1", createdAt: "2030-01-01T00:00:00.000Z" }));
+                    }
+                    return s.exec(sql, params);
+                  };
+                }
+                const v = Reflect.get(s, p, s);
+                return typeof v === "function" ? v.bind(s) : v;
+              },
+            });
+          };
+        }
+        const v = Reflect.get(target, prop, target);
+        return typeof v === "function" ? v.bind(target) : v;
+      },
+    });
+
+    const swept = await sweepExpiredProposals(racyData as DataLayer, at);
+    assert(raced, "the racing re-stage should have fired");
+    assertEquals(swept, 0);
+    assert((await seqOf("d1")) > seq0, "the injected re-stage bumped stage_seq past the sweep's snapshot");
+    const d1 = await deliveryGraphProposals(data).get("d1");
+    assertEquals(d1?.status, "staged");
+    assert(!isProposalExpired(d1?.expires_at ?? ""), "the fresh revision retains its future TTL — it was not expired by the stale sweep");
+  });
+});
+
 test("markProposalDismissed: flipping a live `staged` row returns true and lands it `dismissed`", async () => {
   await withData(async (data) => {
     await stageProposal(data, row());
@@ -484,3 +703,27 @@ test("markProposalExpired: a dismiss racing between the dispatch door's liveness
     assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "dismissed");
   });
 });
+
+test("markProposalExpired: an `expectedStageSeq` guard NO-OPs when a concurrent re-stage bumped the revision (corrupt-graph retirement does not retire a newer graph)", async () => {
+  await withData(async (data) => {
+    const seqOf = async (digest: string): Promise<number> => {
+      const rows = await data.open().query<{ stage_seq: number }>(`SELECT "stage_seq" FROM "delivery_graph_proposals" WHERE "digest" = ?`, [digest]);
+      return rows.length ? Number(rows[0].stage_seq) : -1;
+    };
+    // The dispatch door reads a `staged` proposal at revision `seq0` (its graph is corrupt).
+    await stageProposal(data, row());
+    const seq0 = await seqOf("d1");
+    // Before the corrupt-graph retirement fires, a concurrent re-stage overwrites the SAME digest with a
+    // fresh (valid) graph, bumping `stage_seq`.
+    await stageProposal(data, row());
+    const seq1 = await seqOf("d1");
+    assert(seq1 > seq0, "the re-stage bumped stage_seq");
+    // The retirement guarded on the STALE revision must no-op (return false) and leave the newer graph staged.
+    const flipped = await markProposalExpired(data, "d1", seq0);
+    assertEquals(flipped, false);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "staged");
+    // Guarded on the CURRENT revision, it retires as usual.
+    assertEquals(await markProposalExpired(data, "d1", seq1), true);
+    assertEquals((await deliveryGraphProposals(data).get("d1"))?.status, "expired");
+  });
+});