automategraph 1.5.0 → 1.5.3

package/tests/async.test.ts

@@ -0,0 +1,668 @@
+/* ============================================================
+ * ASYNC NODE-CALLBACK PACK
+ * ------------------------------------------------------------
+ * The engine's contract is:
+ *
+ *   readonly run: (context: NodeContext) => unknown | Promise<unknown>
+ *
+ * so `async` node callbacks (and any thenable-returning callback)
+ * are first-class. This pack proves the async contract holds on
+ * every code path: success, retry, ONTHROW, timeout, failure,
+ * cancellation, fan-out, checkpoints, events and error capture.
+ *
+ * These 20 cases are counted separately from the canonical
+ * 100 unit + 20 complex suites.
+ * ============================================================ */
+
+import {
+  AutomateGraphError,
+  GraphExecutionEngine,
+  NodeContext,
+  TimeoutError
+} from "../src/automategraph";
+
+import { capture, constNode, g, linear, node, sleep, SpyStorage, tick } from "./helpers";
+import { suite } from "./harness";
+
+const asy = suite("async", "Node callbacks (async)");
+const BIG = { timeoutMs: 20_000 };
+
+/* A-01 — the baseline async contract */
+asy.t(
+  "an async node callback is awaited and its resolved value becomes the output",
+  async t => {
+    const definition = g({
+      nodes: [node("async-node", async () => { await sleep(10); return "async-result"; })],
+      entryNodes: ["async-node"],
+      terminalNodes: ["async-node"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.outputs["async-node"] !== "async-result") throw new Error(String(report.outputs["async-node"]));
+    if (report.results[0].status !== "SUCCEEDED") throw new Error(report.results[0].status);
+
+    t.log("finding: NodeContext signals are typed as AbortSignal even in older @types/node DOM-less libs, " +
+      "so async callbacks can consume context.signal without casting");
+  },
+  BIG
+);
+
+/* A-02 — sync callback returning a thenable */
+asy.t(
+  "a non-async callback that returns a thenable is also awaited",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("thenable", () => {
+          // deliberately NOT an async function
+          return { then: (resolve: (value: unknown) => void) => setTimeout(() => resolve("thenable-done"), 5) };
+        })
+      ],
+      entryNodes: ["thenable"],
+      terminalNodes: ["thenable"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.outputs.thenable !== "thenable-done") throw new Error(String(report.outputs.thenable));
+  },
+  BIG
+);
+
+/* A-03 — async without an explicit return yields undefined, not an error */
+asy.t(
+  "async callback with no return value produces an undefined output",
+  async t => {
+    const definition = g({
+      nodes: [node("void-async", async () => { await sleep(5); /* no return */ })],
+      entryNodes: ["void-async"],
+      terminalNodes: ["void-async"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (!("void-async" in report.outputs)) throw new Error("output key missing");
+    if (report.outputs["void-async"] !== undefined) throw new Error("expected undefined");
+  },
+  BIG
+);
+
+/* A-04 — rejection of async callbacks is captured as a GraphError */
+asy.t(
+  "a rejected async callback is captured with code, stack and path",
+  async t => {
+    const definition = g({
+      nodes: [
+        constNode("a", "A", { next: ["boom"] }),
+        node("boom", async () => {
+          await sleep(5);
+          throw new Error("async-rejection");
+        })
+      ],
+      entryNodes: ["a"],
+      terminalNodes: ["boom"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (report.results.find(result => result.nodeId === "boom")!.status !== "FAILED") throw new Error("status");
+    if (report.errors.length !== 1) throw new Error(`errors=${report.errors.length}`);
+
+    const error = report.errors[0];
+    if (error.message !== "async-rejection") throw new Error(error.message);
+    if (error.code !== "NODE_EXECUTION_ERROR") throw new Error(error.code);
+    if (!error.stack || !error.stack.includes("async-rejection")) throw new Error("stack missing");
+    if (error.path.join("/") !== "a/boom") throw new Error(error.path.join("/"));
+    if (error.handled) throw new Error("should be unhandled");
+
+    t.log("captured stack head", error.stack!.split("\n")[0]);
+  },
+  BIG
+);
+
+/* A-05 — AutomateGraphError shaped rejections keep their code and details */
+asy.t(
+  "async rejections with AutomateGraphError preserve code and details",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("app-async", async () => {
+          await sleep(5);
+          throw new AutomateGraphError("upstream unavailable", "UPSTREAM_DOWN", { service: "billing" });
+        })
+      ],
+      entryNodes: ["app-async"],
+      terminalNodes: ["app-async"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+    const error = report.errors[0];
+
+    if (error.code !== "UPSTREAM_DOWN") throw new Error(error.code);
+    if ((error.details as { service: string }).service !== "billing") throw new Error("details lost");
+
+    t.log("finding: TimeoutError is identified by class OR by a NODE_TIMEOUT-family code, so async " +
+      "rejections carrying that code are classified as TIMED_OUT");
+  },
+  BIG
+);
+
+/* A-06 — async retry: succeed on the final attempt */
+asy.t(
+  "async retry callback recovers on a later attempt inside one activation",
+  async t => {
+    let attempts = 0;
+    const definition = g({
+      nodes: [
+        node("async-flaky", async () => {
+          attempts += 1;
+          await sleep(5);
+          if (attempts < 3) throw new Error(`async-fail-${attempts}`);
+          return "async-recovered";
+        }, { retry: { maxAttempts: 3, delayMs: 5 }, errorPolicy: "RETRY" })
+      ],
+      entryNodes: ["async-flaky"],
+      terminalNodes: ["async-flaky"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (attempts !== 3) throw new Error(`attempts=${attempts}`);
+    if (report.results[0].attempt !== 3) throw new Error(`attempt=${report.results[0].attempt}`);
+    if (report.results[0].status !== "SUCCEEDED") throw new Error(report.results[0].status);
+    if (report.errors.length !== 0) throw new Error("recovered attempts must not record errors");
+    if (report.outputs["async-flaky"] !== "async-recovered") throw new Error("output missing");
+  },
+  BIG
+);
+
+/* A-07 — async retry with a custom errorPolicy fixture */
+asy.t(
+  "async callback can be retried by a custom node errorPolicy (RETRY)",
+  async t => {
+    let attempts = 0;
+    const definition = g({
+      nodes: [
+        node("policy-retry", async () => {
+          attempts += 1;
+          await sleep(3);
+          if (attempts === 1) throw new Error("first-async-fail");
+          return "second-async-ok";
+        }, { retry: { maxAttempts: 2 }, errorPolicy: "RETRY", next: ["tail"] }),
+        constNode("tail", "TAIL")
+      ],
+      entryNodes: ["policy-retry"],
+      terminalNodes: ["tail"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (attempts !== 2) throw new Error(`attempts=${attempts}`);
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.results[0].attempt !== 2) throw new Error("final attempt not recorded");
+    if (!report.results.some(result => result.nodeId === "tail")) throw new Error("successor skipped");
+  },
+  BIG
+);
+
+/* A-08 — async ONTHROW repair */
+asy.t(
+  "async rejection routes through ONTHROW into an async repair callback",
+  async t => {
+    let failures = 0;
+    const definition = g({
+      nodes: [
+        node("async-broken", async () => { failures += 1; await sleep(5); throw new Error("async-explode"); }, {
+          onThrow: "async-repair"
+        }),
+        node("async-repair", async () => { await sleep(5); return "ASYNC-REPAIRED"; }, { next: ["finish"] }),
+        constNode("finish", "END")
+      ],
+      entryNodes: ["async-broken"],
+      terminalNodes: ["finish"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (failures !== 1) throw new Error(`async-broken ran ${failures} times`);
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.outputs["async-repair"] !== "ASYNC-REPAIRED") throw new Error("repair output missing");
+    if (!report.errors[0].handled) throw new Error("repair error must be handled");
+    if (report.results.find(result => result.nodeId === "async-broken")!.status !== "FAILED") throw new Error("status");
+  },
+  BIG
+);
+
+/* A-09 — async timeout */
+asy.t(
+  "a slow async callback is preempted by timeoutMs and reported TIMED_OUT",
+  async t => {
+    let settledLate = false;
+    const definition = g({
+      nodes: [
+        node("async-slow", async () => {
+          await sleep(120);
+          settledLate = true;
+          return "too-late";
+        }, { timeoutMs: 25 })
+      ],
+      entryNodes: ["async-slow"],
+      terminalNodes: ["async-slow"]
+    });
+
+    const startedAt = Date.now();
+    const report = await new GraphExecutionEngine(definition).run();
+    const durationMs = Date.now() - startedAt;
+
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (report.results[0].status !== "TIMED_OUT") throw new Error(report.results[0].status);
+    if (report.results[0].output !== undefined) throw new Error("timed-out async output must be discarded");
+    if (durationMs >= 110) throw new Error(`timeout not enforced (${durationMs}ms)`);
+    if (report.errors[0].code !== "NODE_TIMEOUT") throw new Error(report.errors[0].code);
+
+    await sleep(140);
+    if (!settledLate) throw new Error("abandoned async callback never settled");
+
+    t.log("run duration", `${durationMs}ms (callback keeps running for 120ms)`);
+    t.log("finding: on timeout the async callback's later resolution is discarded — no output is recorded");
+  },
+  BIG
+);
+
+/* A-10 — async cancellation is cooperative via context.signal */
+asy.t(
+  "a cooperative async callback observes the abort and rejects; the run settles CANCELLED",
+  async t => {
+    let sawAbort: boolean | undefined;
+    let aborted = false;
+
+    const definition = g({
+      nodes: [
+        node("async-cooperative", async (context: NodeContext) => {
+          const guard = new Promise<never>((_, reject) => {
+            context.signal.addEventListener("abort", () => {
+              aborted = true;
+              const reason = context.signal.reason;
+              reject(reason instanceof Error ? reason : new AutomateGraphError("aborted mid-await", "GRAPH_CANCELLED"));
+            });
+          });
+          // A real async workload: fall back to the abort guard instead of idle waiting.
+          await Promise.race([sleep(500).then(() => "unreached"), guard]);
+          sawAbort = context.signal.aborted;
+          return "unreached";
+        })
+      ],
+      entryNodes: ["async-cooperative"],
+      terminalNodes: ["async-cooperative"]
+    });
+
+    const engine = new GraphExecutionEngine(definition);
+    const running = engine.run();
+    await sleep(15);
+    await engine.cancel();
+    const report = await running;
+
+    if (!aborted) throw new Error("abort event never reached the async callback");
+    if (report.status !== "CANCELLED") throw new Error(report.status);
+    if (sawAbort !== undefined) throw new Error("callback continued past the abort guard");
+    if (report.results.length !== 1) throw new Error("unexpected extra activations");
+
+    // v1.1 (A1) fixed the asymmetry: rejecting because of the abort now
+    // settles exactly like resolving after it — NodeStatus CANCELLED and a
+    // handled GRAPH_CANCELLED record (previously FAILED + unhandled).
+    const nodeResult = report.results[0];
+    if (nodeResult.status !== "CANCELLED") throw new Error(nodeResult.status);
+    if (report.errors.length !== 1) throw new Error(`errors=${report.errors.length}`);
+    if (report.errors[0].code !== "GRAPH_CANCELLED") throw new Error(report.errors[0].code);
+    if (!report.errors[0].handled) throw new Error("cancellation error must be recorded as handled");
+    if (nodeResult.output !== undefined) throw new Error("aborted callback must not record output");
+
+    // v1.1 (A6): the result carries its own error record.
+    if (!nodeResult.error) throw new Error("result.error missing");
+    if (nodeResult.error.id !== report.errors[0].id) throw new Error("result.error mismatch");
+
+    t.log("graph status", report.status, "| node status", nodeResult.status);
+    t.log("error record", `${report.errors[0].code} handled=${report.errors[0].handled}`);
+    t.log(
+      "finding: since v1.1 (A1) cooperative cancellation is symmetric — a callback that rejects " +
+      "because of the abort settles CANCELLED with a handled GRAPH_CANCELLED record, exactly like " +
+      "one that resolves after it; result.error now carries the record (A6)"
+    );
+  },
+  BIG
+);
+
+/* A-11 — async callback that ignores cancellation */
+asy.t(
+  "an async callback that ignores the signal delays settlement; successors stay unscheduled",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("stubborn-async", async () => { await sleep(60); return "finished-anyway"; }, { next: ["tail"] }),
+        constNode("tail", "TAIL")
+      ],
+      entryNodes: ["stubborn-async"],
+      terminalNodes: ["tail"]
+    });
+
+    const engine = new GraphExecutionEngine(definition);
+    const startedAt = Date.now();
+    const running = engine.run();
+    await sleep(10);
+    await engine.cancel();
+    const report = await running;
+    const durationMs = Date.now() - startedAt;
+
+    if (report.status !== "CANCELLED") throw new Error(report.status);
+    if (durationMs < 40) throw new Error(`settled too early (${durationMs}ms) — cooperative drain broken`);
+    if (report.results.some(result => result.nodeId === "tail")) throw new Error("successor ran after cancel");
+    if (report.outputs.tail !== undefined) throw new Error("successor output recorded");
+
+    t.log("drain time", `${durationMs}ms (callback sleeps 60ms and ignores the abort)`);
+  },
+  BIG
+);
+
+/* A-12 — async node whose completion triggers a checkpoint */
+asy.t(
+  "async completion triggers exactly one checkpoint per successful node",
+  async t => {
+    const storage = new SpyStorage();
+    const definition = linear(["a", "b", "c"], async id => { await sleep(5); return `${id}-async`; });
+
+    const report = await new GraphExecutionEngine(definition, { storage }).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (storage.calls !== 3) throw new Error(`checkpoints=${storage.calls}`);
+    const last = storage.snapshots[storage.snapshots.length - 1];
+    if (last.outputs.c !== "c-async") throw new Error("final snapshot incomplete");
+    if (Object.keys(last.outputs).length !== 3) throw new Error("snapshot outputs mismatch");
+  },
+  BIG
+);
+
+/* A-13 — async context.emit is awaited in order */
+asy.t(
+  "context.emit from an async callback is awaited and ordered",
+  async t => {
+    const seen: string[] = [];
+    const definition = g({
+      nodes: [
+        node("emitter", async (context: NodeContext) => {
+          await context.emit("async.first", { n: 1 });
+          await context.emit("async.second", { n: 2 });
+          return "emitted";
+        })
+      ],
+      entryNodes: ["emitter"],
+      terminalNodes: ["emitter"]
+    });
+
+    const engine = new GraphExecutionEngine(definition);
+    engine.events.on("async.first", () => { seen.push("first"); }, { required: true });
+    engine.events.on("async.second", async () => { await sleep(3); seen.push("second"); }, { required: true });
+
+    const report = await engine.run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (seen.join(",") !== "first,second") throw new Error(seen.join(","));
+    if (report.outputs.emitter !== "emitted") throw new Error("output missing");
+  },
+  BIG
+);
+
+/* A-14 — async parallelism */
+asy.t(
+  "async callbacks on independent branches execute in parallel",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("root", () => "root", { next: ["p1", "p2", "p3", "p4"] }),
+        ...["p1", "p2", "p3", "p4"].map(id =>
+          node(id, async () => { await sleep(40); return id; })
+        )
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["p4"]
+    });
+
+    const startedAt = Date.now();
+    const report = await new GraphExecutionEngine(definition).run();
+    const durationMs = Date.now() - startedAt;
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.results.filter(result => result.status === "SUCCEEDED").length !== 5) {
+      throw new Error("not all async branches succeeded");
+    }
+    if (durationMs >= 100) throw new Error(`serialized: ${durationMs}ms for 4x40ms in parallel`);
+
+    t.log("wall time", `${durationMs}ms (serial would be ~160ms)`);
+  },
+  BIG
+);
+
+/* A-15 — mixed sync/async pipeline keeps isolation */
+asy.t(
+  "mixed sync and async nodes run together in one pipeline",
+  async t => {
+    const definition = g({
+      id: "mixed-pipeline",
+      version: "1.0.0",
+      nodes: [
+        node("sync-validate", () => "valid", { next: ["async-enrich"] }),
+        node("async-enrich", async context => {
+          await sleep(5);
+          return `enriched(${context.outputs["sync-validate"]})`;
+        }, { next: ["sync-price"] }),
+        node("sync-price", context => `priced(${context.outputs["async-enrich"]})`, { next: ["async-store"] }),
+        node("async-store", async context => {
+          await sleep(5);
+          return `stored(${context.outputs["sync-price"]})`;
+        })
+      ],
+      entryNodes: ["sync-validate"],
+      terminalNodes: ["async-store"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    const final = String(report.outputs["async-store"]);
+    if (final !== "stored(priced(enriched(valid)))") throw new Error(final);
+    if (report.results.length !== 4) throw new Error(`results=${report.results.length}`);
+  },
+  BIG
+);
+
+/* A-16 — async callbacks composing the fast-path/retry/fatal branches */
+asy.t(
+  "async callback composes success, retry and failure across a three-branch graph",
+  async t => {
+    let fastAttempts = 0;
+    let retryAttempts = 0;
+
+    const definition = g({
+      nodes: [
+        node("root", () => "R", { next: ["async-fast", "async-retry", "async-fatal"] }),
+        node("async-fast", async () => { fastAttempts += 1; await sleep(5); return "fast-ok"; }),
+        node("async-retry", async () => {
+          retryAttempts += 1;
+          await sleep(5);
+          if (retryAttempts < 2) throw new Error("async-once");
+          return "retry-ok";
+        }, { retry: { maxAttempts: 2 }, errorPolicy: "RETRY" }),
+        node("async-fatal", async () => { await sleep(5); throw new Error("async-fatal-error"); })
+      ],
+      entryNodes: ["root"],
+      terminalNodes: ["async-fast"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (fastAttempts !== 1) throw new Error(`fastAttempts=${fastAttempts}`);
+    if (retryAttempts !== 2) throw new Error(`retryAttempts=${retryAttempts}`);
+    if (report.status !== "FAILED") throw new Error(report.status);
+    if (report.outputs["async-fast"] !== "fast-ok") throw new Error("fast output missing");
+    if (report.outputs["async-retry"] !== "retry-ok") throw new Error("retry output missing");
+    if (report.errors.length !== 1) throw new Error(`errors=${report.errors.length}`);
+    if (report.errors[0].nodeId !== "async-fatal") throw new Error(report.errors[0].nodeId);
+
+    t.log("fatal error path", report.errors[0].path.join("/"));
+  },
+  BIG
+);
+
+/* A-17 — async state-machine callback through a sync node */
+asy.t(
+  "an async callback returning a marker drives a synchronous state-machine successor",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("async-evaluate", async context => {
+          await sleep(5);
+          const marker = (context.outputs.manual as string | undefined) ?? "AUTO";
+          return marker;
+        }, { next: ["auto-path"] }),
+        node("auto-path", context => {
+          const marker = context.outputs["async-evaluate"];
+          if (marker !== "AUTO") throw new Error(`unexpected marker ${String(marker)}`);
+          return "auto-done";
+        })
+      ],
+      entryNodes: ["async-evaluate"],
+      terminalNodes: ["auto-path"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.outputs["auto-path"] !== "auto-done") throw new Error("state machine diverged");
+  },
+  BIG
+);
+
+/* A-18 — abandoned async work must not corrupt a later execution */
+asy.t(
+  "an unrelated execution started after an abandoned async node is unaffected",
+  async t => {
+    const abandoned: string[] = [];
+
+    const slow = g({
+      nodes: [
+        node("stubborn", async () => {
+          await sleep(150);
+          abandoned.push("late side effect");
+          return "late";
+        }, { timeoutMs: 20 })
+      ],
+      entryNodes: ["stubborn"],
+      terminalNodes: ["stubborn"]
+    });
+
+    const first = await new GraphExecutionEngine(slow).run();
+    if (first.status !== "FAILED") throw new Error(first.status);
+
+    const second = await new GraphExecutionEngine(linear(["x", "y"], async id => `${id}!`)).run();
+    if (second.status !== "COMPLETED") throw new Error(second.status);
+    if (second.outputs.y !== "y!") throw new Error("second execution corrupted");
+
+    if (second.errors.length !== 0) throw new Error("errors leaked between executions");
+    if (Object.keys(second.outputs).length !== 2) throw new Error("outputs leaked between executions");
+
+    await sleep(160);
+    if (abandoned.length !== 1) throw new Error("abandoned side effect did not run at all");
+
+    t.log("finding: the abandoned async node still performed its side effect after settlement — " +
+      "async cancellation is cooperative only; nothing can stop an in-flight promise");
+  },
+  BIG
+);
+
+/* A-19 — async node emitting into a non-required listener */
+asy.t(
+  "diverse async callbacks (timers, microtasks, thenables) are classified correctly",
+  async t => {
+    const definition = g({
+      nodes: [
+        node("timer-node", async () => { await sleep(1); return "timer"; }, { next: ["microtask-node"] }),
+        node("microtask-node", async () => { await Promise.resolve(); return "microtask"; }, { next: ["thenable-node"] }),
+        node("thenable-node", () =>
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          (globalThis as unknown as { Promise: PromiseConstructor }).Promise.resolve("thenable"),
+          { next: ["void-node"] }),
+        node("void-node", async () => { await sleep(1); })
+      ],
+      entryNodes: ["timer-node"],
+      terminalNodes: ["void-node"]
+    });
+
+    const report = await new GraphExecutionEngine(definition).run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if (report.outputs["timer-node"] !== "timer") throw new Error("timer");
+    if (report.outputs["microtask-node"] !== "microtask") throw new Error("microtask");
+    if (report.outputs["thenable-node"] !== "thenable") throw new Error("thenable");
+    if (report.outputs["void-node"] !== undefined) throw new Error("void");
+    if (report.results.every(result => result.status === "SUCCEEDED") !== true) throw new Error("some failed");
+  },
+  BIG
+);
+
+/* A-20 — async node used with the full plugin/storage stack */
+asy.t(
+  "async callbacks integrate with plugins, storage snapshots and the execution report",
+  async t => {
+    const events: string[] = [];
+    const storage = new SpyStorage();
+
+    const definition = g({
+      id: "async-integrated",
+      version: "3.0.0",
+      nodes: [
+        node("load", async () => { await sleep(5); return { rows: 3 }; }, { next: ["transform"] }),
+        node("transform", async context => {
+          await sleep(5);
+          const rows = (context.outputs.load as { rows: number }).rows;
+          return { rows: rows * 2 };
+        }, { next: ["persist"] }),
+        node("persist", async () => { await sleep(5); return "PERSISTED"; })
+      ],
+      entryNodes: ["load"],
+      terminalNodes: ["persist"]
+    });
+
+    const engine = new GraphExecutionEngine(definition, {
+      storage,
+      plugins: [
+        {
+          id: "async-plugin",
+          version: "1.2.0",
+          async initialize() { await sleep(3); events.push("init"); },
+          async close() { await sleep(3); events.push("close"); }
+        }
+      ]
+    });
+
+    const captured = capture(engine, ["node.started", "node.succeeded", "graph.finished"]);
+    const report = await engine.run();
+
+    if (report.status !== "COMPLETED") throw new Error(report.status);
+    if ((report.outputs.transform as { rows: number }).rows !== 6) throw new Error("async chain output wrong");
+    if (events.join(",") !== "init,close") throw new Error(events.join(","));
+    if (storage.calls !== 3) throw new Error(`checkpoints=${storage.calls}`);
+    if (captured.filter(entry => entry.event === "node.succeeded").length !== 3) throw new Error("events");
+    if (storage.snapshots.some(snapshot => snapshot.graphVersion !== "3.0.0")) throw new Error("snapshot version");
+
+    const firstSnapshot = storage.snapshots[0];
+    if (firstSnapshot.outputs.load === undefined) throw new Error("first checkpoint missing output");
+
+    t.log("checkpoints", storage.calls);
+    t.log("plugin lifecycle", events.join(" -> "));
+  },
+  BIG
+);