@nanobpm/nano-workforce 0.115.0 → 0.117.0

package/e2e/delivery-graph-start.e2e.ts

@@ -0,0 +1,145 @@
+// End-to-end proof of the S5 DISPATCH DOOR (ADR 0005 Decision 7) driven through its REAL ingress — the
+// `startDeliveryGraph` OpenAPI operation (`POST /app/api/actions/start/delivery-graph`), the one
+// contract all three ingress paths (agent POST, raw REST, UI JSON-paste) share. Hermetic: deterministic
+// virtual clock, no network. It proves the acceptance the slice hinges on:
+//
+//   • APPROVAL GATE: a side-effecting graph submitted WITHOUT approval is refused + PARKED (400,
+//     awaiting-approval, no engine instance, no agent job fired) — and is VISIBLE in the cockpit's
+//     `delivery_graph_runs` aggregate so an operator can see it waiting.
+//   • DISPATCH: re-submitting the same graph WITH its content-addressed approval token dispatches — the
+//     graph deploys + runs engine-natively (the agent side effect fires), and the run's derived phase
+//     shows WHERE it is parked ("Parked on human node: …") via the same `pollDeliveryGraphPhase`
+//     projection the cockpit reads.
+//   • IDEMPOTENCY: a duplicate submit short-circuits (`alreadyRunning`) — the agent side effect fires
+//     exactly ONCE, never twice.
+//   • COMPLETION: when the instance ends, the poller reconciles the run to `done` (instanceTracking's
+//     onTerminated reconciles only TERMINATED, so this poller owns COMPLETED→done).
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { after, describe, test } from "node:test";
+import assert from "node:assert/strict";
+import { bootTestApp, type TestApp } from "@nanobpm/urban-testkit";
+import { deliveryGraphRuns } from "../app/deliveryGraphRun.ts";
+import { pollDeliveryGraphPhase } from "../app/service.ts";
+import type { DeliveryGraph } from "../nano-generated/api-io.d.ts";
+
+const APP_ROOT = resolve(import.meta.dirname, "..");
+const GITHUB_ENV: Record<string, string> = { NANO_PR_GITHUB_TRANSPORT: "token", GITHUB_TOKEN: "" };
+
+interface StartResult {
+  ok: boolean;
+  status: string;
+  runKey: string;
+  digest: string;
+  sideEffecting: boolean;
+  alreadyRunning?: boolean;
+  processInstanceKey?: string;
+  approvalToken?: string;
+}
+
+// A side-effecting graph: an `agent` side effect gated ahead of a `human` stop. Approval is required
+// (the agent + the human-facing merge/publish class of graphs Decision 7 protects).
+const GRAPH: DeliveryGraph = {
+  name: "release runbook e2e",
+  nodes: [
+    { id: "open", kind: "agent", agent: { jobType: "senior:demo", prompt: "open + prep" } },
+    { id: "publish", kind: "human", human: { prompt: "run the manual OTP publish" } },
+  ],
+  edges: [{ from: "open", to: "publish" }],
+};
+
+describe("startDeliveryGraph dispatch door — submit → approve → dispatch, idempotent (S5)", () => {
+  const dirs: string[] = [];
+  const apps: TestApp[] = [];
+  after(async () => {
+    for (const app of apps) await app.stop?.();
+    for (const d of dirs) rmSync(d, { recursive: true, force: true });
+  });
+  const boot = async (): Promise<TestApp> => {
+    const d = mkdtempSync(join(tmpdir(), "nwf-delivery-start-e2e-"));
+    dirs.push(d);
+    const app = await bootTestApp(APP_ROOT, { env: { ...GITHUB_ENV, NANO_APP_DB_URL: `file:${join(d, "app.db")}` } });
+    apps.push(app);
+    return app;
+  };
+
+  test("a side-effecting graph is parked at approval, then dispatches once approved; a duplicate never double-launches", async () => {
+    const app = await boot();
+    assert.ok(app.api, "app declares an `api` binding");
+    const api = app.api;
+
+    let agentFired = 0;
+    await app.engine.registerWorker("senior:demo", async () => {
+      agentFired++;
+      return {};
+    });
+
+    // ── Approval gate: submit WITHOUT approval → refused + parked, nothing launched ────────────────
+    const parked = await api.call<StartResult>("startDeliveryGraph", { body: { graph: GRAPH } });
+    assert.equal(parked.status, 400, "an unapproved side-effecting graph is refused");
+    assert.equal(parked.body.status, "awaiting-approval");
+    assert.equal(parked.body.sideEffecting, true);
+    assert.ok(parked.body.approvalToken, "the response hands back the approval token to re-submit with");
+    await app.settle();
+    assert.equal(agentFired, 0, "a parked graph never dispatched its side effect");
+
+    // The parked run is durable + visible in the cockpit aggregate.
+    const runKey = parked.body.runKey;
+    const parkedRow = await deliveryGraphRuns(app.db).get(runKey);
+    assert.ok(parkedRow, "a delivery_graph_runs row exists for the parked graph");
+    assert.equal(parkedRow?.status, "awaiting-approval");
+    assert.equal(parkedRow?.process_key, null, "no engine instance while parked");
+
+    // ── Dispatch: re-submit WITH the token → deploys + runs engine-natively ───────────────────────
+    const token = parked.body.approvalToken;
+    const dispatched = await api.call<StartResult>("startDeliveryGraph", { body: { graph: GRAPH, approvalToken: token } });
+    assert.equal(dispatched.status, 202, "an approved graph dispatches");
+    assert.equal(dispatched.body.status, "running");
+    assert.equal(dispatched.body.alreadyRunning, false);
+    assert.ok(dispatched.body.processInstanceKey, "the run carries the started engine instance key");
+    await app.settle();
+    assert.equal(agentFired, 1, "the agent side effect fired exactly once");
+
+    // The run transitioned parked → running IN PLACE (one row, not a duplicate), carrying its instance.
+    const runningRows = await deliveryGraphRuns(app.db).find({ status: "running" });
+    assert.equal(runningRows.length, 1, "exactly one running run");
+    assert.equal(runningRows[0]?.run_key, runKey, "the SAME run row was approved, not a new one");
+    assert.equal(runningRows[0]?.process_key, dispatched.body.processInstanceKey);
+
+    // ── Cockpit phase: the poller derives WHERE the run is parked (the human node) ─────────────────
+    await pollDeliveryGraphPhase(app.db, app.engine);
+    const phased = await deliveryGraphRuns(app.db).get(runKey);
+    assert.equal(phased?.status, "running");
+    assert.match(String(phased?.phase), /^Parked on human node:/, `phase shows the parked human node, got ${phased?.phase}`);
+
+    // ── Idempotency: a duplicate submit short-circuits — the side effect never fires twice ────────
+    const dup = await api.call<StartResult>("startDeliveryGraph", { body: { graph: GRAPH, approvalToken: token } });
+    assert.equal(dup.status, 202);
+    assert.equal(dup.body.alreadyRunning, true, "the re-submit short-circuited the already-running run");
+    await app.settle();
+    assert.equal(agentFired, 1, "the agent side effect STILL fired only once (no double-launch)");
+
+    // ── Completion: complete the human stop → the instance ends → the poller reconciles to done ───
+    const open = await app.engine.searchUserTasks({ state: "CREATED" });
+    const human = open.find((t) => t.elementId?.startsWith("delivery-human-task__") && !t.elementId?.endsWith("__esc"));
+    assert.ok(human, `a human user task is open, got ${JSON.stringify(open.map((t) => t.elementId))}`);
+    await app.engine.completeUserTask(human.userTaskKey, { humanOutcome: "completed" });
+    await app.settle();
+    await pollDeliveryGraphPhase(app.db, app.engine);
+    const done = await deliveryGraphRuns(app.db).get(runKey);
+    assert.equal(done?.status, "done", "the completed instance reconciled the run to done");
+    assert.equal(done?.phase, "Completed");
+  });
+
+  test("a non-side-effecting (human-only) graph dispatches with NO approval", async () => {
+    const app = await boot();
+    assert.ok(app.api);
+    const graph: DeliveryGraph = { name: "manual gate", nodes: [{ id: "ack", kind: "human", human: { prompt: "click done" } }] };
+    const res = await app.api.call<StartResult>("startDeliveryGraph", { body: { graph } });
+    assert.equal(res.status, 202, "a graph with no side effects needs no approval");
+    assert.equal(res.body.status, "running");
+    assert.equal(res.body.sideEffecting, false);
+    assert.ok(res.body.processInstanceKey);
+  });
+});

package/nano.app.json

@@ -81,6 +81,20 @@
         }
       },
       "pollMs": 5000
+    },
+    {
+      "table": "delivery_graph_runs",
+      "keyField": "process_key",
+      "statusField": "status",
+      "activeStatuses": [
+        "running"
+      ],
+      "onTerminated": {
+        "set": {
+          "status": "failed"
+        }
+      },
+      "pollMs": 5000
     }
   ],
   "workers": [
@@ -214,7 +228,8 @@
     "senior:feature",
     "senior:trial-merge",
     "senior:retro",
-    "senior:conformance"
+    "senior:conformance",
+    "senior:scope-classify"
   ],
   "surfaces": {
     "taskInbox": {

package/openapi.yaml

@@ -1713,6 +1713,85 @@ components:
           items:
             $ref: "#/components/schemas/DeliveryCompileError"
           description: The path-qualified validation/compile failures (at least one).
+    DeliveryGraphStart:
+      description: >-
+        The `startDeliveryGraph` request body (ADR 0005 slice S5) — the ONE gated dispatch door that
+        turns an agent-authored `DeliveryGraph` into a RUNNING engine-native process. This is the OUTER
+        action, distinct from S1's pure `compileDeliveryGraph`: there is deliberately no `dryRun` flag
+        (Decision 5/7) — compile and start are separate operations. The door re-validates (S0), compiles
+        (S1), and launches the S4 runner. Three ingress paths — an agent-ergonomic POST, a raw REST call,
+        and a UI JSON-paste — all hit this ONE contract.
+      type: object
+      additionalProperties: false
+      required:
+        - graph
+      properties:
+        graph:
+          $ref: "#/components/schemas/DeliveryGraph"
+        approvalToken:
+          type: string
+          description: >-
+            The approval OF the rendered preview (Decision 7). Because a delivery graph merges PRs and
+            publishes packages, a graph with any SIDE-EFFECTING node (`agent`/`connector`) dispatches
+            ONLY when the caller presents the graph's content-addressed approval token (returned as
+            `approvalToken` on a prior unapproved submit's `awaiting-approval` response). A graph with
+            no side effects (only `wait`/`human` nodes) needs none and dispatches straight away.
+        idempotencyKey:
+          type: string
+          maxLength: 255
+          description: >-
+            OPTIONAL caller-supplied idempotency key. A re-POST with the SAME key (or, when omitted, the
+            SAME graph — the default key is the graph's content digest) does NOT double-launch: an
+            in-flight run short-circuits with `alreadyRunning: true`. Blank/whitespace is treated as
+            absent.
+    StartDeliveryGraphResult:
+      description: >-
+        The `startDeliveryGraph` outcome. `status` is the run's lifecycle position: `running` (the graph
+        dispatched — or was already running, see `alreadyRunning`) or `awaiting-approval` (a
+        side-effecting graph was submitted without a valid `approvalToken` — it is PARKED, visible in the
+        cockpit, and refused pending approval; re-POST with the returned `approvalToken` to dispatch).
+      type: object
+      additionalProperties: false
+      required:
+        - ok
+        - status
+        - runKey
+        - digest
+        - sideEffecting
+      properties:
+        ok:
+          type: boolean
+          description: True when the graph dispatched (or was already running); false when parked at approval.
+        status:
+          type: string
+          enum: [running, awaiting-approval]
+          description: The run's lifecycle position after this call.
+        runKey:
+          type: string
+          description: The idempotency key the run is stored under (the caller key, else the content digest).
+        digest:
+          type: string
+          description: The graph's content digest — the content-address of the compiled definition.
+        sideEffecting:
+          type: boolean
+          description: Whether the graph has any side-effecting (`agent`/`connector`) node — i.e. whether approval is required.
+        alreadyRunning:
+          type: boolean
+          description: True when a re-submit short-circuited onto an already-running run (no second launch).
+        processInstanceKey:
+          type: string
+          description: The started engine instance key. Present when `status` is `running`; absent while parked at approval.
+        processDefinitionId:
+          type: string
+          description: The content-addressed deployed process id (`delivery-graph-<digest>`). Present when dispatched.
+        approvalToken:
+          type: string
+          description: >-
+            The token to present as `approvalToken` on a re-POST to dispatch this exact graph. Present
+            when `status` is `awaiting-approval` (equals `digest`).
+        message:
+          type: string
+          description: A human-readable summary of the outcome (e.g. the approval-required reason).
     FeatureStart:
       description: The start-feature request body — a SINGLE-issue feature run. Names the target issue
         by EXACTLY ONE of `issue` (an `owner/repo#123` reference) or `url` (a bare issue URL), plus a
@@ -2495,6 +2574,47 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/CompileDeliveryGraphErrors"
+  /actions/start/delivery-graph:
+    post:
+      operationId: startDeliveryGraph
+      summary: Validate + compile + DISPATCH a delivery graph as a running engine-native process (gated, idempotent). (ADR 0005 slice S5)
+      description: >-
+        The delivery-graph DISPATCH door (ADR 0005 Decision 7) — the single gated OUTER action that
+        turns an agent-authored `DeliveryGraph` into a RUNNING engine-native process. Distinct from S1's
+        pure `compileDeliveryGraph`: compile and start are SEPARATE operations (Decision 5/7), so there
+        is deliberately no `dryRun` flag. The door re-validates the graph (`validateDeliveryGraph`, S0),
+        compiles it (`compileDeliveryGraph`, S1), and LAUNCHES the S4 runner (`runDeliveryGraph`).
+        Because these graphs merge PRs and publish packages, dispatch is GATED: a graph with any
+        side-effecting (`agent`/`connector`) node must present the content-addressed `approvalToken` of
+        its rendered preview, else it is PARKED at approval (a 400 carrying the token, plus a visible
+        `awaiting-approval` run in the cockpit) rather than dispatched. Idempotent: a re-POST of the same
+        graph (or the same `idempotencyKey`) short-circuits an already-running run instead of
+        double-launching. Three ingress paths — an agent-ergonomic POST, a raw REST call, and a UI
+        JSON-paste — all hit this ONE contract.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/DeliveryGraphStart"
+      responses:
+        "202":
+          description: The graph dispatched (or a re-submit short-circuited an already-running run).
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/StartDeliveryGraphResult"
+        "400":
+          description: >-
+            The graph failed validation/compilation (path-qualified errors), OR a side-effecting graph
+            was submitted without a valid approval token — it is refused and PARKED at approval (the
+            body carries the `approvalToken` to re-submit with).
+          content:
+            application/json:
+              schema:
+                oneOf:
+                  - $ref: "#/components/schemas/CompileDeliveryGraphErrors"
+                  - $ref: "#/components/schemas/StartDeliveryGraphResult"
   /actions/start/feature:
     post:
       operationId: startFeature

package/operations/startDeliveryGraph.integration.test.ts

@@ -0,0 +1,316 @@
+// Integration coverage for the S5 DISPATCH door (ADR 0005 Decision 7) driven through the operation
+// EDGE — `startDeliveryGraph` composing S0 validate → S1 compile → approval gate → S4 launch. The unit
+// tests in app/deliveryGraphRun.test.ts prove the pure decision helpers in isolation; this file proves
+// the COMPOSED behaviour at the door: each path maps to the correct HTTP status and the correct
+// durable-run / launch effect. It runs the real delegate against an in-memory app/data/engine — no
+// network, deterministic on a single run.
+import { test } from "node:test";
+import { assertEquals } from "#test-assert";
+import type { AppApi } from "@nanobpm/urban";
+import { noopLog } from "../test/log.ts";
+import startDeliveryGraph from "./startDeliveryGraph.ts";
+
+// ── in-memory app (data + engine) ────────────────────────────────────────────
+// A generic table over an array (the DataLayer surface the run aggregate uses: get/find/insert/update)
+// plus a fake engine recording each deploy + start so an accept path can assert exactly-once launch.
+function makeApp(opts: { failCreate?: boolean } = {}) {
+  const tables = new Map<string, Record<string, unknown>[]>();
+  const started: { processDefinitionId: string; variables?: Record<string, unknown> }[] = [];
+  const deployed: unknown[][] = [];
+  const table = (name: string, key: string) => {
+    const rows = tables.get(name) ?? (() => {
+      const fresh: Record<string, unknown>[] = [];
+      tables.set(name, fresh);
+      return fresh;
+    })();
+    return {
+      get: (k: unknown) => Promise.resolve(rows.find((r) => r[key] === k) ?? null),
+      find: (q: Record<string, unknown>) =>
+        Promise.resolve(rows.filter((r) => Object.entries(q).every(([f, v]) => r[f] === v))),
+      insert: (r: Record<string, unknown>) => {
+        // Faithful to the durable table's PRIMARY KEY: a duplicate-key insert is rejected with the
+        // SQLite fence message `isUniqueConstraintFence` classifies, so the door's claim-before-launch
+        // fence is exercised the same way it is against the real store.
+        if (rows.some((existing) => existing[key] === r[key])) {
+          return Promise.reject(new Error(`UNIQUE constraint failed: ${name}.${key}`));
+        }
+        rows.push(r);
+        return Promise.resolve(r);
+      },
+      update: (k: unknown, patch: Record<string, unknown>) => {
+        const row = rows.find((r) => r[key] === k);
+        if (row) Object.assign(row, patch);
+        return Promise.resolve(row);
+      },
+      delete: (k: unknown) => {
+        const i = rows.findIndex((r) => r[key] === k);
+        if (i >= 0) rows.splice(i, 1);
+        return Promise.resolve();
+      },
+    };
+  };
+  const app = {
+    data: {
+      table,
+      // Faithful model of the guarded raw UPDATEs the door issues via the DataSource gateway, BOTH of
+      // which fence on `WHERE "run_key" = ? AND "status" <> 'running'`: the launch-claim compare-and-swap
+      // (`SET status=?,updated_at=?`) and the approval-park write (`SET process_key=?,…,updated_at=?`).
+      // Columns are parsed from the SQL so either statement is applied faithfully. Deferred to a microtask
+      // to model the real async DataSource — the guard-and-write is NOT visible synchronously at call
+      // time, so a concurrently-scheduled delegate can still read the row pre-flip and reach its OWN
+      // claim/park (the exact interleave that made the unfenced writes double-launch / clobber a claim).
+      // The single `status <> 'running'` guard then lets only the first writer win: a launched `running`
+      // row is never flipped back, and a losing claim matches zero rows (`changed: 0`).
+      open: () => ({
+        exec: (sql: string, params: unknown[]) =>
+          Promise.resolve().then(() => {
+            // `"col" = ?` matches every SET assignment plus the WHERE `"run_key" = ?` (the `<> 'running'`
+            // guard uses `<>`, not `=`, so it is excluded); the last param is therefore the run_key.
+            const cols = [...sql.matchAll(/"(\w+)"\s*=\s*\?/g)].map((m) => m[1]);
+            const runKey = params[params.length - 1];
+            const rows = tables.get("delivery_graph_runs") ?? [];
+            const row = rows.find((r) => r["run_key"] === runKey);
+            if (row && row["status"] !== "running") {
+              for (let i = 0; i < cols.length - 1; i++) row[cols[i]] = params[i];
+              return { changed: 1 };
+            }
+            return { changed: 0 };
+          }),
+      }),
+    },
+    engine: {
+      deployResources: (res: unknown[]) => {
+        deployed.push(res);
+        return Promise.resolve([]);
+      },
+      createInstance: (req: { processDefinitionId: string; variables?: Record<string, unknown> }) => {
+        started.push(req);
+        if (opts.failCreate) return Promise.reject(new Error("engine unavailable"));
+        return Promise.resolve({ processInstanceKey: "PI-1" });
+      },
+    },
+    log: noopLog(),
+  } as unknown as AppApi;
+  return { app, started, deployed, runs: () => tables.get("delivery_graph_runs") ?? [] };
+}
+
+function input(body: unknown) {
+  return {
+    req: { method: "POST", path: "/", query: new URLSearchParams(), headers: new Headers(), text: async () => "" } as never,
+    params: {},
+    query: {},
+    body,
+  };
+}
+
+// A SIDE-EFFECTING graph (an `agent` node → approval required) and a NON-side-effecting one
+// (`human`-only → dispatches without approval).
+const SIDE_EFFECTING = {
+  name: "release runbook",
+  nodes: [
+    { id: "open-b", kind: "agent", agent: { jobType: "senior:feature", prompt: "un-draft + merge #B" } },
+    { id: "publish", kind: "human", human: { prompt: "run the manual OTP publish" } },
+  ],
+  edges: [{ from: "open-b", to: "publish" }],
+};
+const HUMAN_ONLY = {
+  name: "manual gate",
+  nodes: [{ id: "ack", kind: "human", human: { prompt: "click done when the release is out" } }],
+};
+
+test("missing graph → 400, nothing launched", async () => {
+  const { app, started } = makeApp();
+  const res = (await startDeliveryGraph(input({}), app)) as { status: number; body: { ok: boolean } };
+  assertEquals(res.status, 400);
+  assertEquals(res.body.ok, false);
+  assertEquals(started.length, 0);
+});
+
+test("a malformed graph fails S0 validation → 400, nothing compiled or launched", async () => {
+  const { app, started } = makeApp();
+  // Duplicate node ids — a semantic error `validateDeliveryGraph` catches (shape alone is fine).
+  const dup = { nodes: [{ id: "a", kind: "agent", agent: { jobType: "j" } }, { id: "a", kind: "agent", agent: { jobType: "j" } }] };
+  const res = (await startDeliveryGraph(input({ graph: dup }), app)) as { status: number; body: { ok: boolean; errors?: unknown[] } };
+  assertEquals(res.status, 400);
+  assertEquals(res.body.ok, false);
+  assertEquals(Array.isArray(res.body.errors), true);
+  assertEquals(started.length, 0);
+});
+
+test("a side-effecting graph WITHOUT approval is refused + PARKED at approval (400, awaiting-approval row, no launch)", async () => {
+  const { app, started, runs } = makeApp();
+  const res = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING }), app)) as {
+    status: number;
+    body: { ok: boolean; status: string; approvalToken: string; sideEffecting: boolean };
+  };
+  assertEquals(res.status, 400);
+  assertEquals(res.body.ok, false);
+  assertEquals(res.body.status, "awaiting-approval");
+  assertEquals(res.body.sideEffecting, true);
+  assertEquals(typeof res.body.approvalToken, "string");
+  assertEquals(started.length, 0); // parked, never launched
+  // The parked run is durable + visible (cockpit reads this table).
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0]?.["status"], "awaiting-approval");
+});
+
+test("re-submitting the SAME side-effecting graph WITH its approval token dispatches (202, running, launched once)", async () => {
+  const { app, started, runs } = makeApp();
+  // First submit parks + hands back the token.
+  const parked = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING }), app)) as { body: { approvalToken: string } };
+  const token = parked.body.approvalToken;
+  // Second submit approves → dispatch. The SAME run row transitions parked → running (not a new row).
+  const res = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING, approvalToken: token }), app)) as {
+    status: number;
+    body: { ok: boolean; status: string; processInstanceKey?: string };
+  };
+  assertEquals(res.status, 202);
+  assertEquals(res.body.ok, true);
+  assertEquals(res.body.status, "running");
+  assertEquals(res.body.processInstanceKey, "PI-1");
+  assertEquals(started.length, 1);
+  assertEquals(runs().length, 1); // still ONE row — approval updated it, did not duplicate
+  assertEquals(runs()[0]?.["status"], "running");
+});
+
+test("a non-side-effecting (human-only) graph dispatches WITHOUT approval (202, running)", async () => {
+  const { app, started } = makeApp();
+  const res = (await startDeliveryGraph(input({ graph: HUMAN_ONLY }), app)) as {
+    status: number;
+    body: { ok: boolean; status: string; sideEffecting: boolean };
+  };
+  assertEquals(res.status, 202);
+  assertEquals(res.body.ok, true);
+  assertEquals(res.body.status, "running");
+  assertEquals(res.body.sideEffecting, false);
+  assertEquals(started.length, 1);
+});
+
+test("a duplicate submit of an already-running graph short-circuits — no second launch", async () => {
+  const { app, started } = makeApp();
+  await startDeliveryGraph(input({ graph: HUMAN_ONLY }), app); // launch #1
+  const res = (await startDeliveryGraph(input({ graph: HUMAN_ONLY }), app)) as {
+    status: number;
+    body: { alreadyRunning: boolean; status: string };
+  };
+  assertEquals(res.status, 202);
+  assertEquals(res.body.alreadyRunning, true);
+  assertEquals(res.body.status, "running");
+  assertEquals(started.length, 1); // still ONE launch — the re-POST did not double-launch
+});
+
+test("a caller idempotencyKey scopes the run — the same key short-circuits, a different key launches again", async () => {
+  const { app, started } = makeApp();
+  await startDeliveryGraph(input({ graph: HUMAN_ONLY, idempotencyKey: "run-1" }), app);
+  const same = (await startDeliveryGraph(input({ graph: HUMAN_ONLY, idempotencyKey: "run-1" }), app)) as { body: { alreadyRunning: boolean } };
+  assertEquals(same.body.alreadyRunning, true);
+  assertEquals(started.length, 1);
+  const other = (await startDeliveryGraph(input({ graph: HUMAN_ONLY, idempotencyKey: "run-2" }), app)) as { body: { status: string; runKey: string } };
+  assertEquals(other.body.status, "running");
+  assertEquals(other.body.runKey, "run-2");
+  assertEquals(started.length, 2); // a distinct key is a distinct run
+});
+
+test("two SIMULTANEOUS submits of the same graph launch it exactly ONCE — the loser hits the run_key fence and short-circuits, no double side effect", async () => {
+  const { app, started, runs } = makeApp();
+  // Fire both before awaiting either: both read `existing === null`, then race to claim the run_key.
+  // The claim-before-launch fence means the loser's insert collides on the PK and it NEVER launches.
+  const [a, b] = (await Promise.all([
+    startDeliveryGraph(input({ graph: HUMAN_ONLY }), app),
+    startDeliveryGraph(input({ graph: HUMAN_ONLY }), app),
+  ])) as { status: number; body: { ok: boolean; status: string; alreadyRunning: boolean } }[];
+  assertEquals(a.status, 202);
+  assertEquals(b.status, 202);
+  assertEquals(a.body.ok, true);
+  assertEquals(b.body.ok, true);
+  // Exactly ONE launch and ONE durable row — no double-dispatch of side effects, no duplicate row.
+  assertEquals(started.length, 1);
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0]?.["status"], "running");
+  // Exactly one racer is the short-circuited loser (alreadyRunning); the other is the fresh winner.
+  assertEquals([a, b].filter((r) => r.body.alreadyRunning === true).length, 1);
+});
+
+test("two SIMULTANEOUS APPROVED re-submits of an already-PARKED graph launch it exactly ONCE — the parked→running claim is a compare-and-swap, not an unfenced update", async () => {
+  const { app, started, runs } = makeApp();
+  // Park the side-effecting graph first (unapproved), then grab its approval token.
+  const parked = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING }), app)) as { body: { approvalToken: string } };
+  const token = parked.body.approvalToken;
+  assertEquals(runs()[0]?.["status"], "awaiting-approval");
+  // Fire two APPROVED submits before awaiting either: both read `existing` as the SAME parked row.
+  // Without a fence on the parked→running transition both would `update` then both launch. The
+  // compare-and-swap (`WHERE status <> 'running'`) lets exactly one flip the row and launch.
+  const [a, b] = (await Promise.all([
+    startDeliveryGraph(input({ graph: SIDE_EFFECTING, approvalToken: token }), app),
+    startDeliveryGraph(input({ graph: SIDE_EFFECTING, approvalToken: token }), app),
+  ])) as { status: number; body: { ok: boolean; status: string; alreadyRunning: boolean } }[];
+  assertEquals(a.status, 202);
+  assertEquals(b.status, 202);
+  // Exactly ONE launch of the side-effecting graph and still ONE durable row (no double-dispatch).
+  assertEquals(started.length, 1);
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0]?.["status"], "running");
+  // Exactly one racer is the short-circuited loser (alreadyRunning); the other is the fresh winner.
+  assertEquals([a, b].filter((r) => r.body.alreadyRunning === true).length, 1);
+});
+
+test("an APPROVED launch racing a concurrent UNAPPROVED re-submit of an already-PARKED graph is NOT clobbered — the park write is fenced `WHERE status <> 'running'`, so the launched claim (and its process_key) survives", async () => {
+  const { app, started, runs } = makeApp();
+  // Park the side-effecting graph first (unapproved) + grab its token — both racers read THIS row.
+  const parked = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING }), app)) as { body: { approvalToken: string } };
+  const token = parked.body.approvalToken;
+  assertEquals(runs()[0]?.["status"], "awaiting-approval");
+  // Fire an APPROVED submit (which claims → running → launches) SIMULTANEOUSLY with another UNAPPROVED
+  // submit (which re-parks). Both read `existing` as the parked row. A blind park `update` would flip
+  // the launched `running` claim back to `awaiting-approval` and null its process_key — breaking the
+  // at-most-once fence. The guarded park write refuses to touch a `running` row instead.
+  const [approved, unapproved] = (await Promise.all([
+    startDeliveryGraph(input({ graph: SIDE_EFFECTING, approvalToken: token }), app),
+    startDeliveryGraph(input({ graph: SIDE_EFFECTING }), app),
+  ])) as { status: number; body: { status: string } }[];
+  assertEquals(approved.status, 202);
+  assertEquals(approved.body.status, "running"); // the approved submit dispatched
+  assertEquals(unapproved.status, 400); // the unapproved submit is refused (needs approval)
+  // Exactly ONE launch and ONE durable row, still `running` with its instance key — NOT clobbered.
+  assertEquals(started.length, 1);
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0]?.["status"], "running");
+  assertEquals(runs()[0]?.["process_key"], "PI-1");
+});
+
+test("a launch failure rolls the claimed run to `failed` — no stranded null-process_key `running` row", async () => {
+  const { app, started, runs } = makeApp({ failCreate: true });
+  let threw = false;
+  try {
+    await startDeliveryGraph(input({ graph: HUMAN_ONLY }), app);
+  } catch {
+    threw = true; // a thrown engine error propagates (framework maps it to a 500) — but only after rollback
+  }
+  assertEquals(threw, true);
+  assertEquals(started.length, 1); // the launch was attempted once
+  // The claim was written, then rolled back to a TERMINAL `failed` — the reconciler/poller skip null-
+  // key rows, so leaving it `running` would strand it forever; `failed` lets it drop out cleanly.
+  assertEquals(runs().length, 1);
+  assertEquals(runs()[0]?.["status"], "failed");
+  assertEquals(runs()[0]?.["process_key"], null);
+});
+
+test("a reused idempotencyKey short-circuits with the RUNNING run's persisted digest/sideEffecting — not the new submission's", async () => {
+  const { app, started } = makeApp();
+  // Launch a human-only (non-side-effecting) run under an explicit key.
+  const first = (await startDeliveryGraph(input({ graph: HUMAN_ONLY, idempotencyKey: "shared" }), app)) as {
+    body: { digest: string; sideEffecting: boolean };
+  };
+  assertEquals(first.body.sideEffecting, false);
+  // Re-POST the SAME key with a DIFFERENT (side-effecting) graph. The response must describe the run
+  // that is actually running — the human-only one — not this mismatched submission.
+  const second = (await startDeliveryGraph(input({ graph: SIDE_EFFECTING, idempotencyKey: "shared" }), app)) as {
+    status: number;
+    body: { alreadyRunning: boolean; digest: string; sideEffecting: boolean };
+  };
+  assertEquals(second.status, 202);
+  assertEquals(second.body.alreadyRunning, true);
+  assertEquals(second.body.sideEffecting, false); // the RUNNING run's value, not the side-effecting resubmit's
+  assertEquals(second.body.digest, first.body.digest);
+  assertEquals(started.length, 1); // still one launch
+});