automategraph 1.5.1 → 1.5.3

package/tests/complex.test.ts

@@ -0,0 +1,1123 @@
+/* ============================================================
+ * 20 COMPLEX / INTEGRATION TESTS
+ * ------------------------------------------------------------
+ * These exercise the whole engine end-to-end: multi-branch
+ * pipelines, failure semantics, concurrency & scale, events and
+ * listener isolation, plugin lifecycle, cancellation and
+ * checkpoint integrity — several of them documenting observed
+ * (and in some cases surprising) engine semantics.
+ * ============================================================ */
+
+import {
+  GraphExecutionEngine,
+  NodeContext,
+  TimeoutError
+} from "../src/automategraph";
+
+import {
+  capture,
+  constNode,
+  g,
+  linear,
+  logCollector,
+  node,
+  plugin,
+  sleep,
+  SpyStorage,
+  tick
+} from "./helpers";
+
+import { suite } from "./harness";
+
+const e2e = suite("integration", "E2E pipelines");
+const sem = suite("integration", "Failure semantics");
+const con = suite("integration", "Concurrency & scale");
+const evt = suite("integration", "Events & listeners");
+const lif = suite("integration", "Lifecycle & plugins");
+const can = suite("integration", "Cancellation & cycles");
+
+const BIG = { timeoutMs: 20_000 };
+
+/* =========================================================
+ * C-01 — multi-branch order pipeline, end to end
+ * ========================================================= */
+
+e2e.t(
+  "multi-branch order pipeline completes with a checkpoint per successful node",
+  async t => {
+    const storage = new SpyStorage();
+    const order: string[] = [];
+
+    const definition = g({
+      id: "order-pipeline",
+      version: "2.1.0",
+      nodes: [
+        node("validate", () => { order.push("validate"); return { orderId: 7, amount: 120 }; }, {
+          next: ["fraud", "audit"]
+        }),
+        node("fraud", context => {
+          const input = context.outputs.validate as { amount: number };
+          order.push("fraud");
+          return input.amount < 1000 ? "clean" : "suspicious";
+        }, { next: ["charge"] }),
+        node("charge", context => {
+          order.push("charge");
+          return { charged: (context.outputs.validate as { amount: number }).amount };
+        }, { next: ["ship"] }),
+        node("ship", () => { order.push("ship"); return "SHIPPED"; }, { next: ["notify"] }),
+        constNode("notify", "EMAIL_SENT"),
+        constNode("audit", "AUDITED")
+      ],
+      entryNodes: ["validate"],
+      terminalNodes: ["notify"]
+    });
+
+    const engine = new GraphExecutionEngine(definition, { storage });
+    const events = capture(engine, ["graph.ready", "node.succeeded", "graph.logicallyCompleted", "graph.finished"]);
+    const report = await engine.run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (!report.logicallyCompleted) throw new Error("not logically completed");
+    if (report.graphId !== "order-pipeline" || report.graphVersion !== "2.1.0") throw new Error("metadata wrong");
+    if (report.errors.length !== 0) throw new Error(`errors=${report.errors.length}`);
+
+    const ran = report.results.map(result => result.nodeId);
+    for (const expected of ["validate", "fraud", "audit", "charge", "ship", "notify"]) {
+      if (!ran.includes(expected)) throw new Error(`missing node ${expected}`);
+    }
+    for (const expected of ["charge", "ship", "notify"]) {
+      const count = ran.filter(id => id === expected).length;
+      if (count !== 1) throw new Error(`${expected} ran ${count} times (expected exactly once)`);
+    }
+
+    // required-order subsequence on the critical path
+    const criticalPath = ran.filter(id => ["validate", "fraud", "charge", "ship", "notify"].includes(id));
+    if (criticalPath.join(",") !== "validate,fraud,charge,ship,notify") {
+      throw new Error(`critical path=${criticalPath.join(",")}`);
+    }
+
+    const logical = events.find(entry => entry.event === "graph.logicallyCompleted");
+    if (!logical) throw new Error("logicallyCompleted event missing");
+    if ((logical.payload as { terminalNodeId: string }).terminalNodeId !== "notify") {
+      throw new Error("wrong terminal node in event");
+    }
+
+    const finished = events.find(entry => entry.event === "graph.finished");
+    if ((finished!.payload as { status: string }).status !== "COMPLETED") throw new Error("finished status");
+
+    // exactly one checkpoint per successful node, no extras, no gaps
+    const succeeded = report.results.filter(result => result.status === "SUCCEEDED").length;
+    if (storage.calls !== succeeded) throw new Error(`checkpoints=${storage.calls} vs succeeded=${succeeded}`);
+    const last = storage.snapshots[storage.snapshots.length - 1];
+    if (Object.keys(last.outputs).length !== succeeded) throw new Error("snapshot outputs incomplete");
+    if (last.outputs.ship !== "SHIPPED") throw new Error("ship output lost");
+
+    t.log("checkpoints written", storage.calls);
+    t.log("critical path", criticalPath.join(" -> "));
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-02 — chaos: flaky nodes + ONTHROW repair inside one graph
+ * ========================================================= */
+
+e2e.t(
+  "chaos graph: retry recovery plus ONTHROW repair still completes",
+  async t => {
+    let riskyAttempts = 0;
+    // Gate: the steady branch must not reach the terminal before the
+    // repair path has run, otherwise "logically completed" would stop
+    // the repair activation from being scheduled (see C-15 scenario 4).
+    let releaseSteady!: () => void;
+    const repairDone = new Promise<void>(resolve => { releaseSteady = resolve; });
+
+    const definition = g({
+      nodes: [
+        node("root", () => "go", { next: ["risky", "steady"] }),
+        node("risky", () => {
+          riskyAttempts += 1;
+          if (riskyAttempts === 1) throw new Error("transient-1");
+          if (riskyAttempts === 2) throw new Error("transient-2");
+          return "risky-ok";
+        }, { retry: { maxAttempts: 3, delayMs: 5 }, errorPolicy: "RETRY", next: ["check"] }),
+        node("check", () => { throw new Error("data-corrupt"); }, { onThrow: "repair" }),
+        node("repair", () => { releaseSteady(); return "repaired"; }, { next: ["done"] }),
+        node("steady", async () => {
+          await Promise.race([repairDone, sleep(2000)]);
+          return "steady-ok";
+        }, { next: ["done"] }),
+        constNode("done", "DONE")
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["done"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (riskyAttempts !== 3) throw new Error(`riskyAttempts=${riskyAttempts}`);
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+
+    const risky = report.results.find(result => result.nodeId === "risky")!;
+    if (risky.attempt !== 3 || risky.status !== "SUCCEEDED") throw new Error("risky outcome wrong");
+
+    const check = report.results.find(result => result.nodeId === "check")!;
+    if (check.status !== "FAILED") throw new Error(check.status);
+
+    if (report.errors.length !== 1) throw new Error(`errors=${report.errors.length}`);
+    const error = report.errors[0];
+    if (!error.handled) throw new Error("repair error should be handled");
+    if (error.nodeId !== "check") throw new Error(error.nodeId);
+    if (error.message !== "data-corrupt") throw new Error(error.message);
+    if (error.path.join("/") !== "root/risky/check") throw new Error(error.path.join("/"));
+
+    if (report.outputs.repair !== "repaired") throw new Error("repair output missing");
+    if (!report.results.some(result => result.nodeId === "done")) throw new Error("done never ran");
+
+    t.log("error path", error.path);
+    t.log("handled", error.handled);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-20 — full resilience scenario (mixed outcomes)
+ * ========================================================= */
+
+e2e.t(
+  "resilience scenario: timeout is routed to a fallback so the run completes",
+  async t => {
+    const storage = new SpyStorage();
+    const custom: unknown[] = [];
+    // The resilient path must finish before another branch can reach the
+    // terminal, otherwise logical completion drops the fallback activation.
+    let releaseFast!: () => void;
+    const fallbackDone = new Promise<void>(resolve => { releaseFast = resolve; });
+
+    const definition = g({
+      nodes: [
+        node("start", () => 1, { next: ["slow-vendor", "fast-vendor"] }),
+        node("slow-vendor", async () => { await sleep(80); return "vendor-result"; }, {
+          timeoutMs: 25,
+          onThrow: "vendor-fallback"
+        }),
+        node("vendor-fallback", () => { releaseFast(); return "cached-price"; }, { next: ["decide"] }),
+        node("fast-vendor", async () => {
+          await Promise.race([fallbackDone, sleep(2000)]);
+          return "fresh-price";
+        }, { next: ["decide"] }),
+        node("decide", async context => {
+          await context.emit("pricing.decided", { fallback: context.outputs["vendor-fallback"] });
+          return "decided";
+        }, { next: ["finish"] }),
+        constNode("finish", "DONE")
+      ],
+      entryNodes: ["start"],
+      terminalNodes: ["finish"]
+    });
+
+    const engine = new GraphExecutionEngine(definition, { storage });
+    engine.events.on("pricing.decided", payload => { custom.push(payload); });
+    const report = await engine.run();
+    await sleep(20);
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.errors.length !== 1) throw new Error(`errors=${report.errors.length}`);
+
+    const timeoutError = report.errors[0];
+    if (timeoutError.code !== "NODE_TIMEOUT") throw new Error(timeoutError.code);
+    if (!timeoutError.handled) throw new Error("timeout error not handled by fallback");
+    if (timeoutError.nodeId !== "slow-vendor") throw new Error(timeoutError.nodeId);
+
+    const slow = report.results.find(result => result.nodeId === "slow-vendor")!;
+    if (slow.status !== "TIMED_OUT") throw new Error(slow.status);
+
+    if (report.outputs["vendor-fallback"] !== "cached-price") throw new Error("fallback output missing");
+    if (!report.results.some(result => result.nodeId === "finish")) throw new Error("finish never ran");
+
+    const decideRuns = report.results.filter(result => result.nodeId === "decide").length;
+    if (decideRuns < 1) throw new Error("decide never ran");
+    if (custom.length !== decideRuns) {
+      throw new Error(`custom events=${custom.length} vs decide runs=${decideRuns}`);
+    }
+    if ((custom[0] as { fallback: string }).fallback !== "cached-price") {
+      throw new Error("decide did not observe the fallback output");
+    }
+
+    // every successful node produced exactly one checkpoint
+    const succeeded = report.results.filter(result => result.status === "SUCCEEDED").length;
+    if (storage.calls !== succeeded) throw new Error(`checkpoints=${storage.calls} vs succeeded=${succeeded}`);
+
+    t.log("timeout details", timeoutError.details);
+    t.log("succeeded nodes", succeeded);
+    t.log("finding: fan-in runs the join node once per incoming edge (decide runs = " + decideRuns + ")");
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-03 — FAIL_FAST at graph level: unscheduled work is dropped
+ * ========================================================= */
+
+sem.t(
+  "graph-level FAIL_FAST stops not-yet-scheduled successors on independent branches",
+  async t => {
+    const definition = g({
+      errorPolicy: "FAIL_FAST",
+      nodes: [
+        node("root", () => "go", { next: ["failing", "slow-branch"] }),
+        node("failing", () => { throw new Error("policy-violation"); }),
+        node("slow-branch", async () => { await sleep(30); return "still-here"; }, { next: ["slow-child"] }),
+        constNode("slow-child", "should-not-run")
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["slow-child"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+    const ran = report.results.map(result => result.nodeId);
+
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (!ran.includes("failing")) throw new Error("failing node did not run");
+    if (!ran.includes("slow-branch")) throw new Error("in-flight sibling was wrongly aborted");
+    if (ran.includes("slow-child")) throw new Error("FAIL_FAST failed to stop pending work");
+
+    const failures = report.errors.filter(error => error.code === "NODE_EXECUTION_ERROR");
+    if (failures.length !== 1) throw new Error(`errors=${failures.length}`);
+    if (failures[0].message !== "policy-violation") throw new Error(failures[0].message);
+
+    t.log(
+      "finding: an already-started sibling still observes its own completion " +
+      "before the graph settles as FAILED"
+    );
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-04 — ONTHROW is a transition, not a retry
+ * ========================================================= */
+
+sem.t(
+  "ONTHROW routes to a repair node and the failing node is never re-attempted",
+  async t => {
+    let failures = 0;
+
+    const definition = g({
+      nodes: [
+        node("root", () => "go", { next: ["risky"] }),
+        node("risky", () => { failures += 1; throw new Error("explode"); }, { onThrow: "repair" }),
+        node("repair", () => "fixed", { next: ["verify"] }),
+        node("verify", () => "verified", { next: ["finish"] }),
+        constNode("finish", "END")
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["finish"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (failures !== 1) throw new Error(`risky executed ${failures} times (ONTHROW must not retry)`);
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+
+    const risky = report.results.find(result => result.nodeId === "risky")!;
+    if (risky.attempt !== 1 || risky.status !== "FAILED") throw new Error("risky outcome wrong");
+
+    if (report.errors[0].path.join("/") !== "root/risky") throw new Error(report.errors[0].path.join("/"));
+    if (!report.errors[0].handled) throw new Error("error should be marked handled");
+
+    const order = report.results.map(result => result.nodeId).join(",");
+    if (!/root,risky,repair,verify,finish/.test(order)) throw new Error(order);
+
+    t.log("activation order", order);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-05 — diamond fan-out/fan-in: activations are per-edge
+ * ========================================================= */
+
+con.t(
+  "diamond fan-in executes the join node once per incoming activation (no dedup)",
+  async t => {
+    let joinActivations = 0;
+    let releaseJoin!: () => void;
+    const bothArrived = new Promise<void>(resolve => { releaseJoin = resolve; });
+
+    const definition = g({
+      nodes: [
+        node("a", () => "A", { next: ["b", "c"] }),
+        node("b", () => "B", { next: ["d"] }),
+        node("c", () => "C", { next: ["d"] }),
+        node("d", async context => {
+          joinActivations += 1;
+          if (joinActivations === 2) releaseJoin();
+          // Hold until both incoming activations exist, so the count is
+          // deterministic instead of racing logical completion.
+          await Promise.race([bothArrived, sleep(2000)]);
+          return `d(seen b=${String(context.outputs.b)}, c=${String(context.outputs.c)})`;
+        }, { next: ["e"] }),
+        constNode("e", "E")
+      ],
+      entryNodes: ["a"],
+      terminalNodes: ["e"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+    const dResults = report.results.filter(result => result.nodeId === "d");
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (joinActivations !== 2) throw new Error(`join activated ${joinActivations} times (expected 2)`);
+    if (dResults.length !== 2) throw new Error(`join results=${dResults.length}`);
+    if (new Set(dResults.map(result => result.activationId)).size !== 2) {
+      throw new Error("activationIds not distinct");
+    }
+    if (dResults.some(result => result.status !== "SUCCEEDED")) throw new Error("join activation failed");
+    if (!report.results.some(result => result.nodeId === "e")) throw new Error("terminal never ran");
+
+    const finalValue = String(report.outputs.d);
+    if (!finalValue.startsWith("d(seen b=B, c=C)")) throw new Error(finalValue);
+
+    t.log("finding: no join/dedup semantics — the join node ran once per incoming edge");
+    t.log("final outputs.d (last writer wins)", finalValue);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-06 — cycles terminate through ONTHROW (the only static branch)
+ * ========================================================= */
+
+can.t(
+  "cyclic graph iterates via next + ONTHROW exit and keeps a growing path",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("loop", (context: NodeContext) => {
+          const iteration = (context.outputs.loop as number | undefined) ?? 0;
+          if (iteration >= 5) throw new Error("loop-exit");
+          return iteration + 1;
+        }, { next: ["loop"], onThrow: "done" }),
+        constNode("done", "LOOP_FINISHED")
+      ],
+      entryNodes: ["loop"],
+      terminalNodes: ["done"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+
+    const loopRuns = report.results.filter(result => result.nodeId === "loop");
+    if (loopRuns.length !== 6) throw new Error(`loop ran ${loopRuns.length} times`);
+    if (report.outputs.loop !== 5) throw new Error(String(report.outputs.loop));
+
+    const exitError = report.errors[0];
+    if (exitError.message !== "loop-exit" || !exitError.handled) throw new Error("exit not handled");
+    if (exitError.path.length !== 6) throw new Error(`path length ${exitError.path.length}`);
+    if (exitError.path.some(step => step !== "loop")) throw new Error(exitError.path.join(","));
+
+    t.log("finding: cycles are legal (no validation) and terminate only via ONTHROW");
+    t.log("loop error path", exitError.path.join(" -> "));
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-07 — wide fan-out with a hard concurrency barrier
+ * ========================================================= */
+
+con.t(
+  "250-node fan-out satisfies a full-concurrency barrier (no hidden cap)",
+  async t => {
+    const LEAVES = 250;
+    let started = 0;
+    let releaseBarrier!: () => void;
+    const barrier = new Promise<void>(resolve => { releaseBarrier = resolve; });
+
+    const leaves = Array.from({ length: LEAVES }, (_, index) =>
+      node(`leaf-${index}`, async () => {
+        started += 1;
+        if (started === LEAVES) releaseBarrier();
+        await barrier;
+        return index;
+      })
+    );
+
+    const definition = g({
+      nodes: [node("root", () => "root", { next: leaves.map(leaf => leaf.id) }), ...leaves],
+      entryNodes: ["root"],
+      terminalNodes: [`leaf-${LEAVES - 1}`]
+    });
+
+    const startedAt = Date.now();
+    const report = await new GraphExecutionEngine(definition).run();
+    const durationMs = Date.now() - startedAt;
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (started !== LEAVES) throw new Error(`barrier deadlock: only ${started} leaves started`);
+    if (report.results.length !== LEAVES + 1) throw new Error(`results=${report.results.length}`);
+    if (report.results.some(result => result.status !== "SUCCEEDED")) throw new Error("some leaf failed");
+    if (report.outputs[`leaf-${LEAVES - 1}`] !== LEAVES - 1) throw new Error("leaf output wrong");
+
+    t.log("finding: scheduler concurrency is unbounded (MAX_SAFE_INTEGER)");
+    t.log(
+      "throughput proof: all 250 nodes were simultaneously in flight (the barrier could only be " +
+      "released once every leaf had started), so wall time is ~1 node, not 250"
+    );
+    t.log("wall time", `${durationMs}ms (serial lower bound would be >= 1250ms if capped)`);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-19 — long chain: checkpoint monotonicity and ordering
+ * ========================================================= */
+
+con.t(
+  "40-node chain produces monotone, gap-free checkpoints in execution order",
+  async t => {
+    const LENGTH = 40;
+    const ids = Array.from({ length: LENGTH }, (_, index) => `s${index}`);
+    const storage = new SpyStorage();
+    const report = await new GraphExecutionEngine(linear(ids, id => `${id}-out`), { storage }).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (storage.calls !== LENGTH) throw new Error(`checkpoints=${storage.calls}`);
+
+    const order = report.results.map(result => result.nodeId).join(",");
+    if (order !== ids.join(",")) throw new Error("chain ran out of order");
+
+    let previousCount = 0;
+    let previousTime = 0;
+    for (const [index, snapshot] of storage.snapshots.entries()) {
+      const count = Object.keys(snapshot.outputs).length;
+      if (count !== index + 1) throw new Error(`snapshot ${index} has ${count} outputs`);
+      if (count <= previousCount) throw new Error("outputs not monotonically increasing");
+      previousCount = count;
+
+      const time = Date.parse(snapshot.timestamp);
+      if (time < previousTime) throw new Error("snapshot timestamps went backwards");
+      previousTime = time;
+
+      if (snapshot.status !== "RUNNING") throw new Error(`snapshot status ${snapshot.status}`);
+      if (snapshot.executionId !== report.executionId) throw new Error("snapshot executionId mismatch");
+    }
+
+    t.log("snapshots", storage.calls);
+    t.log("final outputs", previousCount);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-13 — event ordering with required listeners
+ * ========================================================= */
+
+evt.t(
+  "required listeners observe a deterministic lifecycle event order",
+  async t => {
+    const engine = new GraphExecutionEngine(linear(["a", "b"]));
+    const sequence: string[] = [];
+    const succeededListeners: string[] = [];
+
+    for (const name of [
+      "graph.ready",
+      "graph.cancellationRequested",
+      "node.started",
+      "node.succeeded",
+      "node.failed",
+      "graph.logicallyCompleted",
+      "graph.finished"
+    ]) {
+      engine.events.on(name, (payload: unknown) => {
+        const info = payload as { nodeId?: string };
+        sequence.push(info.nodeId ? `${name}:${info.nodeId}` : name);
+      }, { required: true });
+    }
+
+    engine.events.on("node.succeeded", () => { succeededListeners.push("first"); }, { required: true });
+    engine.events.on("node.succeeded", async () => {
+      await sleep(1);
+      succeededListeners.push("second");
+    }, { required: true });
+    engine.events.on("node.succeeded", () => { succeededListeners.push("third"); }, { required: true });
+
+    const startedAt = Date.now();
+    const report = await engine.run();
+    const runMs = Date.now() - startedAt;
+
+    const expected = [
+      "graph.ready",
+      "node.started:a",
+      "node.succeeded:a",
+      "node.started:b",
+      "node.succeeded:b",
+      "graph.logicallyCompleted",
+      "graph.finished"
+    ];
+    if (sequence.join("|") !== expected.join("|")) {
+      throw new Error(`sequence=${sequence.join("|")}`);
+    }
+
+    if (succeededListeners.join(",") !== "first,second,third,first,second,third") {
+      throw new Error(`required listener FIFO broken: ${succeededListeners.join(",")}`);
+    }
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+
+    t.log("event sequence", sequence.join(" -> "));
+    t.log("run duration", `${runMs}ms`);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-17 — listener isolation & resilience
+ * ========================================================= */
+
+evt.t(
+  "faulty non-required listeners are isolated; a failing required listener is contained",
+  async t => {
+    const { lines, onLog } = logCollector();
+    const engine = new GraphExecutionEngine(linear(["a", "b"]), { onLog });
+
+    engine.events.on("node.succeeded", () => { throw new Error("sync-listener-boom"); });
+    engine.events.on("node.started", async () => { throw new Error("async-listener-boom"); });
+    engine.events.on("graph.finished", () => { throw new Error("required-finished-boom"); }, { required: true });
+
+    const report = await engine.run();
+    await tick();
+    await sleep(20);
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.errors.length !== 0) {
+      throw new Error(`listener faults leaked into node errors: ${report.errors.length}`);
+    }
+
+    const isolated = lines.filter(line => /^Non-critical event listener failed: /.test(line.message));
+    const eventsLogged = new Set(isolated.map(line => line.message.split(": ")[1]));
+    if (!eventsLogged.has("node.succeeded") || !eventsLogged.has("node.started")) {
+      throw new Error(`isolation logging incomplete: ${[...eventsLogged].join(",")}`);
+    }
+
+    const contained = lines.find(line => line.message === "graph.finished listener failed");
+    if (!contained) throw new Error("required graph.finished failure not contained");
+
+    t.log("isolated listener failures", isolated.length);
+    t.log(
+      "finding: graph.finished is the only emit whose rejection is caught by the engine; " +
+      "a failing required listener elsewhere (e.g. node.failed) escapes into the node error path"
+    );
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-08 — plugin lifecycle & shutdown error collection
+ * ========================================================= */
+
+lif.t(
+  "plugins close in reverse order and shutdown failures are collected, not fatal",
+  async t => {
+    const closes: string[] = [];
+
+    const engine = new GraphExecutionEngine(linear(["a"]), {
+      plugins: [
+        plugin({ id: "audit", onClose: async () => { await sleep(5); closes.push("audit"); } }),
+        plugin({
+          id: "telemetry",
+          onClose: () => { throw new Error("socket already closed"); }
+        }),
+        plugin({ id: "cache", onClose: () => { closes.push("cache"); } })
+      ]
+    });
+
+    const report = await engine.run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (closes.join(",") !== "cache,audit") throw new Error(`close order=${closes.join(",")}`);
+    if (report.pluginShutdownErrors.length !== 1) {
+      throw new Error(`shutdown errors=${report.pluginShutdownErrors.length}`);
+    }
+    if (!report.pluginShutdownErrors[0].includes('Plugin "telemetry" shutdown failed')) {
+      throw new Error(report.pluginShutdownErrors[0]);
+    }
+    if (!report.pluginShutdownErrors[0].includes("socket already closed")) {
+      throw new Error("cause message missing");
+    }
+
+    t.log("shutdown errors", report.pluginShutdownErrors);
+  },
+  BIG
+);
+
+lif.t(
+  "plugin shutdown is time-boxed by cancellationTimeoutMs",
+  async t => {
+    const closes: string[] = [];
+    const engine = new GraphExecutionEngine(linear(["a"]), {
+      cancellationTimeoutMs: 30,
+      plugins: [
+        plugin({ id: "slow-close", onClose: async () => { await sleep(400); closes.push("slow"); } }),
+        plugin({ id: "fast-close", onClose: () => { closes.push("fast"); } })
+      ]
+    });
+
+    const startedAt = Date.now();
+    const report = await engine.run();
+    const durationMs = Date.now() - startedAt;
+
+    if (durationMs > 3000) throw new Error(`shutdown hung for ${durationMs}ms`);
+    if (!report.pluginShutdownErrors.some(message => message.includes('Plugin "slow-close"'))) {
+      throw new Error("slow close was not flagged");
+    }
+    // fast-close still ran: a failure/timeout in one plugin does not skip the rest
+    if (!closes.includes("fast")) throw new Error("remaining plugin skipped after timeout");
+
+    t.log("run duration", `${durationMs}ms`);
+    t.log("shutdown errors", report.pluginShutdownErrors.length);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-14 — metadata isolation between concurrent executions
+ * ========================================================= */
+
+lif.t(
+  "two concurrent executions keep ids, snapshots and results isolated",
+  async t => {
+    const makeDefinition = () =>
+      g({
+        nodes: [
+          node("a", async () => { await sleep(15); return "A"; }, { next: ["b"] }),
+          node("b", () => { throw new Error("branch-boom"); }, { onThrow: "c" }),
+          constNode("c", "C")
+        ],
+        entryNodes: ["a"],
+        terminalNodes: ["c"]
+      });
+
+    const storageOne = new SpyStorage();
+    const storageTwo = new SpyStorage();
+    const one = new GraphExecutionEngine(makeDefinition(), { storage: storageOne });
+    const two = new GraphExecutionEngine(makeDefinition(), { storage: storageTwo });
+
+    const [reportOne, reportTwo] = await Promise.all([one.run(), two.run()]);
+
+    if (reportOne.executionId === reportTwo.executionId) throw new Error("executionIds collided");
+    if (reportOne.status !== "COMPLETED" || reportTwo.status !== "COMPLETED") throw new Error("status mismatch");
+
+    for (const [report, storage] of [
+      [reportOne, storageOne] as const,
+      [reportTwo, storageTwo] as const
+    ]) {
+      if (storage.calls === 0) throw new Error("no checkpoints captured");
+      if (storage.snapshots.some(snapshot => snapshot.executionId !== report.executionId)) {
+        throw new Error("snapshot crossed executions");
+      }
+      if (report.errors.some(error => error.executionId !== report.executionId)) {
+        throw new Error("error crossed executions");
+      }
+    }
+
+    const activationIds = new Set([
+      ...reportOne.results.map(result => result.activationId),
+      ...reportTwo.results.map(result => result.activationId)
+    ]);
+    if (activationIds.size !== reportOne.results.length + reportTwo.results.length) {
+      throw new Error("activationIds leaked between executions");
+    }
+
+    if (JSON.stringify(reportOne.outputs) !== JSON.stringify(reportTwo.outputs)) {
+      throw new Error("outputs diverged between identical runs");
+    }
+
+    t.log("execution ids", [
+      reportOne.executionId.slice(0, 8),
+      reportTwo.executionId.slice(0, 8)
+    ]);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-09 — checkpoint persistence failure mid-run
+ * ========================================================= */
+
+sem.t(
+  "checkpoint failure on the 2nd save blocks scheduling and freezes the last valid snapshot",
+  async t => {
+    const storage = new SpyStorage();
+    storage.failOn = callIndex => callIndex === 2;
+
+    const definition = g({
+      nodes: [
+        node("entry", () => "E", { next: ["b1", "b2"] }),
+        node("b1", async () => { await sleep(10); return "B1"; }, { next: ["tail"] }),
+        node("b2", async () => { await sleep(15); return "B2"; }, { next: ["tail"] }),
+        constNode("tail", "TAIL")
+      ],
+      entryNodes: ["entry"],
+      terminalNodes: ["tail"]
+    });
+
+    const report = await new GraphExecutionEngine(definition, { storage }).run();
+
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (storage.snapshots.length !== 1) throw new Error(`valid snapshots=${storage.snapshots.length}`);
+
+    const valid = storage.snapshots[0];
+    if (valid.outputs.entry !== "E") throw new Error("last valid snapshot lost entry output");
+
+    const checkpointErrors = report.errors.filter(error => error.code === "CHECKPOINT_SAVE_FAILED");
+    if (checkpointErrors.length === 0) throw new Error("CHECKPOINT_SAVE_FAILED never recorded");
+
+    if (report.results.some(result => result.nodeId === "tail")) {
+      throw new Error("successors scheduled after checkpoint failure");
+    }
+
+    const nodeFailures = report.results.filter(result => result.status === "FAILED").map(result => result.nodeId);
+    if (!nodeFailures.includes("b1") || !nodeFailures.includes("b2")) {
+      throw new Error(`node failures=${nodeFailures.join(",")}`);
+    }
+
+    t.log("checkpoint errors", checkpointErrors.length);
+    t.log(
+      "finding: the checkpoint failure surfaces twice per node (CHECKPOINT_SAVE_FAILED plus a " +
+      "NODE_EXECUTION_ERROR for the raw persistence error) and marks those nodes FAILED"
+    );
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-11 — node timeout is cooperative, not preemptive
+ * ========================================================= */
+
+sem.t(
+  "node timeout settles the graph while the abandoned node keeps running (cooperative abort)",
+  async t => {
+    let sawAbort = false;
+    let abortReason: unknown;
+
+    const definition = g({
+      nodes: [
+        node("stubborn", async context => {
+          await sleep(150);
+          sawAbort = context.signal.aborted;
+          abortReason = context.signal.reason;
+          return "late-arrival";
+        }, { timeoutMs: 40 })
+      ],
+      entryNodes: ["stubborn"],
+      terminalNodes: ["stubborn"]
+    });
+
+    const engine = new GraphExecutionEngine(definition);
+    const startedAt = Date.now();
+    const report = await engine.run();
+    const durationMs = Date.now() - startedAt;
+
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (report.results[0].status !== "TIMED_OUT") throw new Error(report.results[0].status);
+    if (durationMs >= 140) throw new Error(`timeout did not preempt the wait (${durationMs}ms)`);
+
+    const error = report.errors[0];
+    if (error.code !== "NODE_TIMEOUT") throw new Error(error.code);
+    if (!error.stack) throw new Error("missing stack trace");
+    const details = error.details as { nodeId: string; timeoutMs: number };
+    if (details.nodeId !== "stubborn" || details.timeoutMs !== 40) throw new Error("bad details");
+
+    // Wait for the abandoned node to finish on its own.
+    await sleep(200);
+    if (!sawAbort) throw new Error("signal never aborted");
+    if (!(abortReason instanceof TimeoutError)) throw new Error("abort reason is not a TimeoutError");
+    if (engine.getState() !== "FAILED") throw new Error("state mutated after settlement");
+
+    t.log("run duration", `${durationMs}ms (node keeps running for 150ms)`);
+    t.log("abort reason", (abortReason as Error).name);
+    t.log("finding: JavaScript cannot kill a running node; the engine reports TIMED_OUT and moves on");
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-12 — retry & error-policy semantics matrix
+ * ========================================================= */
+
+sem.t(
+  "policy matrix: RETRY loops, CONTINUE ignores maxAttempts, FALLBACK has no distinct path",
+  async t => {
+    // (a) RETRY loops until success
+    let attemptsA = 0;
+    const a = g({
+      nodes: [
+        node("flaky", () => { attemptsA += 1; if (attemptsA < 3) throw new Error(`f${attemptsA}`); return "ok"; },
+          { retry: { maxAttempts: 3, delayMs: 10 }, errorPolicy: "RETRY" })
+      ],
+      entryNodes: ["flaky"],
+      terminalNodes: ["flaky"]
+    });
+    const reportA = await new GraphExecutionEngine(a).run();
+    if (attemptsA !== 3) throw new Error(`(a) attempts=${attemptsA}`);
+    if (reportA.status !== "COMPLETED") throw new Error(`(a) ${reportA.status}`);
+
+    // (b) CONTINUE (default): retry policy present but policy does not loop
+    let attemptsB = 0;
+    const b = g({
+      nodes: [
+        node("once", () => { attemptsB += 1; throw new Error("single-shot"); },
+          { retry: { maxAttempts: 5 } })
+      ],
+      entryNodes: ["once"],
+      terminalNodes: ["once"]
+    });
+    const reportB = await new GraphExecutionEngine(b).run();
+    if (attemptsB !== 1) throw new Error(`(b) attempts=${attemptsB} — retry ignored without RETRY policy`);
+    if (reportB.results[0].attempt !== 1) throw new Error("(b) attempt recorded wrong");
+    if (reportB.status !== "FAILED") throw new Error(`(b) ${reportB.status}`);
+
+    // (c) FALLBACK: no fallback routing exists; behaves like CONTINUE
+    let attemptsC = 0;
+    const c = g({
+      nodes: [
+        node("primary", () => { attemptsC += 1; throw new Error("no-fallback-wired"); },
+          { errorPolicy: "FALLBACK", retry: { maxAttempts: 2 } })
+      ],
+      entryNodes: ["primary"],
+      terminalNodes: ["primary"]
+    });
+    const reportC = await new GraphExecutionEngine(c).run();
+    if (attemptsC !== 1) throw new Error(`(c) attempts=${attemptsC}`);
+    if (reportC.status !== "FAILED") throw new Error(`(c) ${reportC.status}`);
+
+    t.log("(a) RETRY attempts", attemptsA);
+    t.log("(b) CONTINUE attempts", attemptsB);
+    t.log("(c) FALLBACK attempts", attemptsC);
+    t.log("finding: FALLBACK is accepted but treated as CONTINUE — no alternative-route mechanism exists");
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-15 — final-state precedence matrix
+ * ========================================================= */
+
+sem.t(
+  "final-state precedence: terminal reached + unhandled errors still yields FAILED",
+  async t => {
+    // (1) work exhausted, terminal never reached
+    const dead = g({
+      nodes: [constNode("a", "dead-end"), constNode("sink", "never")],
+      entryNodes: ["a"],
+      terminalNodes: ["sink"]
+    });
+    const reportDead = await new GraphExecutionEngine(dead).run();
+    if (reportDead.status !== "FAILED") throw new Error(`(1) ${reportDead.status}`);
+    if (reportDead.logicallyCompleted) throw new Error("(1) wrongly logicallyCompleted");
+    if (reportDead.errors[0].code !== "GRAPH_TERMINAL_NOT_REACHED") throw new Error("(1) code");
+
+    // (2) terminal reached on one branch, other branch failed unhandled
+    const mixed = g({
+      nodes: [
+        node("entry", () => "E", { next: ["bad", "good-terminal"] }),
+        node("bad", () => { throw new Error("branch-failed"); }),
+        constNode("good-terminal", "REACHED")
+      ],
+      entryNodes: ["entry"],
+      terminalNodes: ["good-terminal"]
+    });
+    const reportMixed = await new GraphExecutionEngine(mixed).run();
+    if (!reportMixed.logicallyCompleted) throw new Error("(2) terminal not acknowledged");
+    if (reportMixed.status !== "FAILED") {
+      throw new Error(`(2) ${reportMixed.status} — unhandled errors must outweigh logical completion`);
+    }
+
+    // (3) clean run reaching the terminal
+    const clean = g({
+      nodes: [node("entry", () => "E", { next: ["finish"] }), constNode("finish", "F")],
+      entryNodes: ["entry"],
+      terminalNodes: ["finish"]
+    });
+    const reportClean = await new GraphExecutionEngine(clean).run();
+    if (reportClean.status !== "COMPLETED" || !reportClean.logicallyCompleted) {
+      throw new Error(`(3) ${reportClean.status}`);
+    }
+
+    // (4) another branch reaches the terminal while this branch is failing:
+    //     the ONTHROW recovery activation is pushed but silently dropped.
+    const dropped = g({
+      nodes: [
+        node("entry", () => "E", { next: ["fast-branch", "slow-branch"] }),
+        node("fast-branch", () => "FAST", { next: ["finish"] }),
+        node("slow-branch", async () => { await sleep(35); throw new Error("late-failure"); }, {
+          onThrow: "recovery"
+        }),
+        node("recovery", () => "RECOVERED", { next: ["finish"] }),
+        constNode("finish", "F")
+      ],
+      entryNodes: ["entry"],
+      terminalNodes: ["finish"]
+    });
+    const reportDropped = await new GraphExecutionEngine(dropped).run();
+    const ranDropped = reportDropped.results.map(result => result.nodeId);
+
+    if (reportDropped.status !== "COMPLETED") throw new Error(`(4) ${reportDropped.status}`);
+    if (ranDropped.includes("recovery")) {
+      throw new Error("(4) recovery ran — expected it to be dropped after logical completion");
+    }
+    if (reportDropped.outputs.recovery !== undefined) throw new Error("(4) recovery output present");
+    const lateError = reportDropped.errors.find(error => error.nodeId === "slow-branch");
+    if (!lateError) throw new Error("(4) late failure not recorded");
+    if (!lateError.handled) throw new Error("(4) late failure should be marked handled");
+
+    t.log("(1) dead-end status", reportDead.status);
+    t.log("(2) logical completion", reportMixed.logicallyCompleted, "status", reportMixed.status);
+    t.log("(4) recovery dropped but error marked handled", lateError.handled);
+    t.log(
+      "finding: settlement order is cancellation > checkpoint failure > unhandled errors > " +
+      "logical completion; FAILED silently wins over a completed terminal"
+    );
+    t.log(
+      "finding: once any branch reaches a terminal node, pending activations (including ONTHROW " +
+      "recovery paths) are cleared and never executed, yet their errors stay marked handled"
+    );
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-16 — policy resolution: node override vs graph default
+ * ========================================================= */
+
+sem.t(
+  "policy resolution: every FAIL_FAST is sticky and halts the scheduler",
+  async t => {
+    // (a) graph CONTINUE + node FAIL_FAST: v1.1 (A4) made node-level
+    //     FAIL_FAST sticky, so it halts scheduling like the graph-level policy.
+    const a = g({
+      nodes: [
+        node("root", () => "R", { next: ["b1", "b2"] }),
+        node("b1", async () => { await sleep(25); return "B1"; }, { next: ["b1-child"] }),
+        node("b2", () => { throw new Error("node-level-fast-fail"); }, { errorPolicy: "FAIL_FAST" }),
+        constNode("b1-child", "RAN_AFTER_OVERRIDE")
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["b1-child"]
+    });
+    const reportA = await new GraphExecutionEngine(a).run();
+    const ranA = reportA.results.map(result => result.nodeId);
+    if (reportA.status !== "FAILED") throw new Error(`(a) ${reportA.status}`);
+    if (ranA.includes("b1-child")) {
+      throw new Error("(a) node-level FAIL_FAST must stop the scheduler (sticky since v1.1)");
+    }
+    if (!ranA.includes("b1")) throw new Error("(a) in-flight sibling lost");
+
+    // (b) graph FAIL_FAST + node CONTINUE: the graph policy still halts the other branch
+    const b = g({
+      errorPolicy: "FAIL_FAST",
+      nodes: [
+        node("root", () => "R", { next: ["b1", "b2"] }),
+        node("b1", async () => { await sleep(25); return "B1"; }, { next: ["b1-child"] }),
+        node("b2", () => { throw new Error("node-continue"); }, { errorPolicy: "CONTINUE" }),
+        constNode("b1-child", "SHOULD_NOT_RUN")
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["b1-child"]
+    });
+    const reportB = await new GraphExecutionEngine(b).run();
+    const ranB = reportB.results.map(result => result.nodeId);
+    if (ranB.includes("b1-child")) {
+      throw new Error("(b) graph-level FAIL_FAST failed to halt the sibling branch");
+    }
+    if (!ranB.includes("b1")) throw new Error("(b) in-flight sibling lost");
+
+    // (c) node RETRY overrides graph CONTINUE
+    let attempts = 0;
+    const c = g({
+      errorPolicy: "CONTINUE",
+      nodes: [
+        node("flaky", () => { attempts += 1; if (attempts === 1) throw new Error("once"); return "ok"; },
+          { retry: { maxAttempts: 2 }, errorPolicy: "RETRY" })
+      ],
+      entryNodes: ["flaky"],
+      terminalNodes: ["flaky"]
+    });
+    const reportC = await new GraphExecutionEngine(c).run();
+    if (attempts !== 2) throw new Error(`(c) attempts=${attempts}`);
+    if (reportC.status !== "COMPLETED") throw new Error(`(c) ${reportC.status}`);
+
+    t.log(
+      "finding: FAIL_FAST is sticky at both levels since v1.1 (A4) — the old node-level behaviour " +
+      "was a one-off queue flush that did nothing under unbounded concurrency; a node-level CONTINUE " +
+      "still cannot shield other branches from a graph-level FAIL_FAST"
+    );
+    t.log("(a) node override FAIL_FAST -> sibling successor dropped", !ranA.includes("b1-child"));
+    t.log("(b) graph FAIL_FAST -> sibling successor dropped", !ranB.includes("b1-child"));
+    t.log("(c) retry attempts under node override", attempts);
+  },
+  BIG
+);
+
+/* =========================================================
+ * C-10 — cancellation of an in-flight fan-out
+ * ========================================================= */
+
+can.t(
+  "cancelling a fan-out aborts in-flight nodes and never starts their successors",
+  async t => {
+    const workers = Array.from({ length: 10 }, (_, index) =>
+      node(`worker-${index}`, async () => { await sleep(200); return index; }, { next: [`leaf-${index}`] })
+    );
+    const leaves = Array.from({ length: 10 }, (_, index) => constNode(`leaf-${index}`, index));
+
+    const definition = g({
+      nodes: [node("root", () => "R", { next: workers.map(worker => worker.id) }), ...workers, ...leaves],
+      entryNodes: ["root"],
+      terminalNodes: ["leaf-9"]
+    });
+
+    const engine = new GraphExecutionEngine(definition);
+    const captured = capture(engine, ["graph.cancellationRequested", "graph.finished"]);
+    const startedAt = Date.now();
+    const running = engine.run();
+    await sleep(20);
+
+    const cancelStarted = Date.now();
+    await engine.cancel();
+    const cancelMs = Date.now() - cancelStarted;
+
+    const report = await running;
+    const totalMs = Date.now() - startedAt;
+
+    if (report.status !== "CANCELLED") throw new Error(report.status);
+    if (engine.getState() !== "CANCELLED") throw new Error("engine state mismatch");
+
+    const ran = report.results.map(result => result.nodeId);
+    if (!ran.includes("root")) throw new Error("root should have completed");
+    if (ran.some(id => id.startsWith("leaf-"))) throw new Error("successors started after cancel");
+    if (report.results.filter(result => result.nodeId.startsWith("worker-")).length !== 10) {
+      throw new Error("expected all 10 in-flight workers to be recorded");
+    }
+    if (report.results.some(result => result.nodeId.startsWith("worker-") && result.status !== "CANCELLED")) {
+      throw new Error("workers not marked CANCELLED");
+    }
+    if (cancelMs > 100) throw new Error(`cancel() blocked for ${cancelMs}ms`);
+    if (totalMs < 150) throw new Error(`run() settled before in-flight work drained (${totalMs}ms)`);
+
+    const requestEvent = captured.find(entry => entry.event === "graph.cancellationRequested");
+    const finishedEvent = captured.find(entry => entry.event === "graph.finished");
+    if (!requestEvent) throw new Error("cancellation event missing");
+    if ((finishedEvent!.payload as { status: string }).status !== "CANCELLED") {
+      throw new Error("graph.finished status wrong");
+    }
+
+    t.log("cancel() latency", `${cancelMs}ms`);
+    t.log("run() drain time", `${totalMs}ms (workers sleep 200ms)`);
+    t.log("finding: cancel() returns immediately but the report waits for cooperative drain");
+  },
+  BIG
+);