@alexkroman1/aai-cli 10.0.1 → 11.0.0

package/dist/templates/recap-workflow/agent.test.ts

@@ -7,8 +7,9 @@
  *
  * Three tiers, and the line between them is what this file is careful about:
  *
- * - **The tools**, against a stubbed `ctx.workflows`. That is the only honest
- *   way to unit-test them (the real client needs a Workflow DevKit world), and
+ * - **The tools**, against a stubbed `ctx.workflows`. That is the honest way to
+ *   unit-test a TOOL — what a tool owns is the call it makes, not what the run
+ *   does afterwards — and
  *   it is enough: what the agent half promises is that the handoff passes a
  *   correlation key, that a second request finds the live run instead of paying
  *   for a second transcription, and that a cancel says out loud what cancelling
@@ -20,24 +21,24 @@
  * - **The body's two helpers** — the poll loop and the compensation unwind —
  *   with `sleep` stubbed. What that asserts is ORDERING and BRANCHING, which is
  *   ordinary logic and worth pinning; it asserts nothing about durability,
- *   replay or suspension, and could not. `recapFlow` itself is deliberately not
- *   driven here for exactly that reason — a body test dressed up as a durability
- *   test would be the worse failure. `aai-cli`'s
- *   `dev-workflow.scenario.test.ts` is the tier that builds a project and
- *   runs a real one.
+ *   replay or suspension, and could not.
+ * - **`recapFlow` itself, durably.** The last block runs it on the real replay
+ *   engine over an in-memory journal (`runWorkflow`,
+ *   `@alexkroman1/aai-runtime/testing`), which this file used to say needed a
+ *   built world. It is the tier that reaches this desk's two most expensive
+ *   claims: that a resume does not re-transcribe, and that the RETENTION GATE's
+ *   unanswered window deletes. `aai-cli`'s `dev-workflow.scenario.test.ts` is
+ *   still the tier above it, with a project and a real queue.
  *
- * **One branch of the body is therefore UNPINNED here, and it is the most
- * expensive one in the file: `recapFlow`'s `isWorkflowSuspend(err)` re-throw.**
- * Its own comment says what its absence did — the first poll that had to wait
- * deleted the transcript the run was waiting for — so the missing test is worth
- * naming rather than leaving as a gap somebody assumes is covered.
- * `createWorkflowCtx` cannot produce a suspend by construction: its `sleep` is
+ * **The branch this file used to name as its biggest gap no longer exists.** It
+ * was `recapFlow`'s `if (isWorkflowSuspend(err)) throw err;` — the guard whose
+ * absence had once deleted the transcript the run was waiting for — and it was
+ * unpinnable here by construction, since `createWorkflowCtx`'s `sleep` is
  * RECORDED and its `waitFor` answers out of `hooks`, so no wait it serves ever
- * throws. Minting the engine's brand by hand would drive the branch against a
- * value no engine here produced — a test of this file's own fixture. The real
- * one is `aai-runtime`'s `workflow-replay.test.ts`, whose engine fails any run
- * whose body swallows a suspend, which is the guard under this rule for every
- * template at once.
+ * suspended. A wait now hands the body a promise that never settles, so a
+ * suspension cannot reach a `catch` at all and there is no branch left to test:
+ * `aai-runtime`'s `workflow-replay-suspend.ts` carries the mechanism and
+ * `workflow-replay.test.ts` the engine-level proof, for every template at once.
  */
 
 /** The def a DEPLOYED agent runs: authored, plus what `tools/` declares. */
@@ -49,6 +50,7 @@ import {
   createWorkflowCtx,
   parseSchemaInput,
   schemaInputIssues,
+  stubGatewayRoute,
   toolRunner,
 } from "@alexkroman1/aai/testing";
 import {
@@ -57,6 +59,7 @@ import {
   installStubGateway as stubGateway,
 } from "@alexkroman1/aai/testing/vitest";
 import type { WorkflowRunSnapshot } from "@alexkroman1/aai/workflow-api";
+import { runWorkflow } from "@alexkroman1/aai-runtime/testing";
 import { beforeEach, describe, expect, test, vi } from "vitest";
 import { recap } from "./shared.ts";
 import {
@@ -798,7 +801,7 @@ describe("awaitTranscript — the polling port, and the callback over it", () =>
     // Three reads: the one before the park, the one the delivery woke, and the
     // one after the ordinary fifteen-second wait that followed it.
     expect(polls()).toBe(3);
-    expect(ctx.slept).toEqual([{ until: 15_000, correlationId: undefined }]);
+    expect(ctx.slept).toEqual([{ label: "poll", until: 15_000, correlationId: undefined }]);
   });
 
   test("parks ONCE and then polls, because a token cannot be claimed twice", async () => {
@@ -834,7 +837,7 @@ describe("awaitTranscript — the polling port, and the callback over it", () =>
     expect(polls()).toBe(3);
     // It asked, and then it stopped counting on the answer.
     expect(ctx.waited).toEqual([NUDGE]);
-    expect(ctx.slept).toEqual([{ until: 15_000, correlationId: undefined }]);
+    expect(ctx.slept).toEqual([{ label: "poll", until: 15_000, correlationId: undefined }]);
   });
 
   test("says 'still transcribing' after ONE closed window, not after nine", async () => {
@@ -996,3 +999,186 @@ describe("compensate — the saga port", () => {
     ).resolves.toBeUndefined();
   });
 });
+
+/**
+ * `recapFlow` itself, on the real replay engine.
+ *
+ * `runWorkflow` starts the declared workflow on `createInProcessWorkflowEngine`
+ * over an in-memory journal and drives one delivery at a time, with a suspension
+ * RECORDED rather than waited out. So the poll cadence a deployed run spends
+ * fifteen seconds a turn on costs this file nothing, and the run really parks,
+ * really resumes off its journal, and really closes its window.
+ *
+ * Two of this desk's claims are only reachable here, and both are expensive when
+ * wrong:
+ *
+ * - **A resume does not re-transcribe.** Twenty minutes of provider time is
+ *   already paid for, which is why `summarize` is its own step; nothing below
+ *   this tier can show that a second walk did not submit the recording again.
+ * - **The retention gate's unanswered window DELETES.** "A gate whose no-answer
+ *   branch keeps the data is not a gate", and for a desk holding transcripts of
+ *   other people's meetings the default is what the whole pattern is about.
+ *   `ctx.workflows.wakeUp` deliberately cannot end an approval window — that is
+ *   `SleepRecord.kind`'s entire reason — so `expireWaits()` is the only thing
+ *   that reaches the branch.
+ *
+ * Note the body takes the POLL arm throughout: `callbackUrl` degrades to
+ * `undefined` in a spec (no minter is published), so `job.callback` is `false`
+ * and the first-turn callback park is never entered. That is the same arm a
+ * local `aai dev` run takes, and its own doc says so.
+ */
+describe("the run is DURABLE", () => {
+  const URL = "https://example.com/standup.mp3";
+  const INPUT = { url: URL, requestedBy: "sess_1" };
+  const RECAP = '{"headline":"Standup","points":["a","b"],"spoken":"They shipped it."}';
+
+  /**
+   * The provider and the model, behind ONE published `stepFetch`.
+   *
+   * Every step's HTTP goes through the slot, the model call included, so a
+   * gateway stub installed over `globalThis.fetch` beside a provider stub would
+   * be bypassed — `link-digest/agent.test.ts` carries the same note.
+   * `stubGatewayRoute` is the composition for it.
+   *
+   * `transcript` is a list of successive `GET` answers, so a spec says how many
+   * polls the job takes by how many entries it gives.
+   */
+  function stubWorld(transcript: readonly Record<string, unknown>[]) {
+    const model = stubGatewayRoute(RECAP);
+    const deletes: string[] = [];
+    let polls = 0;
+    const provider = installStubStepFetch((request) => {
+      const routed = model.route(request);
+      if (routed) return routed;
+      if (request.method === "DELETE") {
+        deletes.push(request.url);
+        return { status: 200, body: {} };
+      }
+      if (request.method === "POST") return { status: 200, body: { id: "t_1", status: "queued" } };
+      // The last answer repeats, so a spec that wants one more poll than it
+      // scripted gets the terminal state rather than an index error.
+      const at = Math.min(polls++, transcript.length - 1);
+      return { status: 200, body: transcript[at] ?? {} };
+    });
+    return { model, deletes, provider, polls: () => polls };
+  }
+
+  const PROCESSING = { id: "t_1", status: "processing" };
+  const DONE = {
+    id: "t_1",
+    status: "completed",
+    text: "We shipped it.",
+    audio_duration: 600,
+  };
+
+  beforeEach(() => {
+    vi.stubEnv("ASSEMBLYAI_API_KEY", "sk-test");
+  });
+
+  test("submits once, then parks on the poll cadence rather than blocking", async () => {
+    stubWorld([PROCESSING, DONE]);
+    const started = Date.now();
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+
+    expect(run.status).toBe("running");
+    // The 15s poll interval, journaled — the run is in progress and not
+    // executing, which is what a caller polling it sees.
+    expect(run.wakeAt).toBeGreaterThan(started);
+    expect(run.steps.map((step) => step.key)).toEqual(["checkTranscript#0", "submitRecording#0"]);
+  });
+
+  test("resumes past the poll WITHOUT re-submitting the recording", async () => {
+    // The claim `summarize` is a separate step for: twenty minutes of provider
+    // time is already paid for, and a resume must not spend it again.
+    const world = stubWorld([PROCESSING, DONE]);
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+    await run.advanceSleep();
+
+    // Parked again, now on the retention gate — a hook with a deadline.
+    expect(run.status).toBe("running");
+    expect(run.steps.map((step) => step.key)).toEqual([
+      "checkTranscript#0",
+      "checkTranscript#1",
+      "noteGate#0",
+      "submitRecording#0",
+      "summarize#0",
+    ]);
+    expect(run.deliveries).toBe(2);
+    // ONE submit across two walks — `submitRecording#0` came back out of the
+    // journal on the second — and one model call.
+    expect(
+      world.provider.calls.filter((call) => call.method === "POST" && !call.url.includes("chat")),
+    ).toHaveLength(1);
+    expect(world.model.calls).toHaveLength(1);
+  });
+
+  test("an answer of KEEP leaves the transcript on the account", async () => {
+    const world = stubWorld([DONE]);
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+    await run.signal(retentionToken("sess_1"), { keep: true });
+
+    expect(run.status).toBe("completed");
+    expect(run.output).toMatchObject({ kept: true, answered: true, requestedBy: "sess_1" });
+    // No discard step reached at all, and nothing deleted.
+    expect(run.steps.map((step) => step.name)).not.toContain("discardOnDecline");
+    expect(world.deletes).toEqual([]);
+  });
+
+  test("an UNANSWERED window deletes, which is the safe default the gate exists for", async () => {
+    // The branch nothing else can reach: `ctx.workflows.wakeUp` must not close
+    // an approval window, and the deadline carries no correlation id, so the
+    // only public route to this outcome is to wait out two real minutes.
+    const world = stubWorld([DONE]);
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+    expect(run.status).toBe("running");
+
+    await run.expireWaits();
+    expect(run.status).toBe("completed");
+    // `answered: false` is the distinction the type carries: the caller did not
+    // decline, they said nothing, and the desk deleted anyway.
+    expect(run.output).toMatchObject({ kept: false, answered: false });
+    expect(run.steps.map((step) => step.name)).toContain("discardOnDecline");
+    expect(world.deletes).toHaveLength(1);
+    expect(world.deletes[0]).toContain("t_1");
+  });
+
+  test("a signal that arrives after the window closed cannot reopen it", async () => {
+    // `closeHook` is a compare-and-set, so the walk that timed out and every
+    // later replay read the same branch — the divergence `HookRecord.closed`
+    // exists to prevent.
+    stubWorld([DONE]);
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+    await run.expireWaits();
+    expect(run.output).toMatchObject({ kept: false });
+
+    await run.signal(retentionToken("sess_1"), { keep: true });
+    expect(run.signalled).toBe(false);
+    expect(run.output).toMatchObject({ kept: false });
+  });
+
+  test("a failure after the transcript exists UNWINDS it, and the run still fails", async () => {
+    // The saga. `summarize` is given prose instead of JSON on every attempt, so
+    // the step exhausts its patience and the body's catch runs the compensation
+    // stack — which must delete the transcript the run acquired.
+    const model = stubGatewayRoute("Here is a recap, in prose, as you did not ask.");
+    const deletes: string[] = [];
+    installStubStepFetch((request) => {
+      const routed = model.route(request);
+      if (routed) return routed;
+      if (request.method === "DELETE") {
+        deletes.push(request.url);
+        return { status: 200, body: {} };
+      }
+      if (request.method === "POST") return { status: 200, body: { id: "t_1", status: "queued" } };
+      return { status: 200, body: DONE };
+    });
+
+    const run = await runWorkflow(recap, INPUT, { name: "recap" });
+    expect(run.status).toBe("failed");
+    expect(run.error).toMatch(/JSON/i);
+    // The undo ran, as a STEP — which is what makes a crash during the unwind
+    // resume with the finished ones replayed rather than run twice.
+    expect(run.steps.map((step) => step.name)).toContain("discardTranscript");
+    expect(deletes).toHaveLength(1);
+  });
+});

package/dist/templates/recap-workflow/workflows/recap.ts

@@ -111,7 +111,7 @@
  * just your shell.
  */
 
-import { isWorkflowSuspend, type WorkflowCtx } from "@alexkroman1/aai";
+import type { WorkflowCtx } from "@alexkroman1/aai";
 import { report, requireStepEnv, stepFetch, stepWebhookUrl } from "@alexkroman1/aai/step";
 import {
   FatalError,
@@ -333,18 +333,21 @@ export async function recapFlow(input: { url: string; requestedBy: string }, ctx
     const retention = await askWhetherToKeep(input.requestedBy, job.id, compensations, ctx);
     return { ...recap, ...retention, requestedBy: input.requestedBy };
   } catch (err) {
-    // **A suspend is not a failure, and this catch is why that matters.** The
-    // body above WAITS three ways now — `awaitTranscript` parks on the
-    // provider's callback, then sleeps between polls, and the gate waits for an
-    // answer — and every one of them suspends by throwing, so it lands here.
-    // Without this line the first poll that had to wait unwound the compensation
-    // stack, DELETED the transcript the run was waiting for, journaled the
-    // deletion as successful and re-threw; the engine saw its own signal come
-    // back out and recorded the run as healthily suspended. The data was gone and
-    // every signal said fine. `replayRun` now also fails a run that swallows one,
-    // so a body that forgets this is loud rather than silently destructive — but
-    // the body is the place it belongs.
-    if (isWorkflowSuspend(err)) throw err;
+    // **A suspension cannot arrive here, and it once could.** The body above
+    // WAITS three ways — `awaitTranscript` parks on the provider's callback,
+    // then sleeps between polls, and the gate waits for an answer — and each of
+    // those used to suspend by THROWING, so it landed in this catch. This saga
+    // is the code that paid for it: the first poll that had to wait unwound the
+    // compensation stack, DELETED the transcript the run was waiting for,
+    // journaled the deletion as successful and re-threw, and the engine saw its
+    // own signal come back out and recorded the run as healthily suspended. The
+    // data was gone and every signal said fine.
+    //
+    // The guard that lived here (`if (isWorkflowSuspend(err)) throw err;`) is
+    // gone because the hazard is: a wait now hands back a promise that never
+    // settles, so a parked body does not reach a `catch` at all. This block sees
+    // step failures, and nothing else.
+    //
     // The saga's whole point. Everything acquired above is released, in reverse,
     // before the failure is re-thrown — and because each undo is a STEP, a crash
     // during the unwind resumes with the finished ones replayed from the journal
@@ -441,7 +444,7 @@ export async function awaitTranscript(
       // read at the top of the next turn is what establishes the fact.
       await ctx.waitFor(nudge, { timeoutMs: CALLBACK_WINDOW_MS });
     } else {
-      await ctx.sleep(POLL_INTERVAL_MS);
+      await ctx.sleep("poll", POLL_INTERVAL_MS);
     }
   }
   throw new Error(`Gave up on that recording after ${MAX_POLLS} checks.`);

package/dist/templates/redline/agent.test.ts

@@ -1,25 +1,26 @@
 /**
  * Specs for the redline desk — the reflection port, as a workflow app.
  *
- * Same honest line as `link-digest`'s spec, for the same reason: the workflow
- * BODY is only durable once the Workflow DevKit's build has transformed it, so
- * testing it here would exercise a plain async function and prove nothing about
- * replay. What IS testable is the declaration (three things that are silent when
- * wrong — the `page: "static"` field, the workflow's name, and the input schema)
- * and the STEPS, which imported with no bundler in the path are ordinary async
+ * The declaration is testable (three things that are silent when wrong — the
+ * `page: "static"` field, the workflow's name, and the input schema) and so are
+ * the STEPS, which imported with no bundler in the path are ordinary async
  * functions: their JSON contract with the model, their `FatalError` guards, and
  * the pure helpers underneath them.
  *
- * The loop's EXIT is the one thing worth naming that a spec here cannot reach.
- * It is decided in the body, on a step's journaled verdict — see the module doc
- * in `workflows/redline.ts` — and what would prove it is a replay, which needs a
- * built world. The critique step's verdict handling is where the testable half
- * of that lives.
+ * **And so is the loop's EXIT**, which this file used to name as the one thing a
+ * spec here could not reach: "it is decided in the body, on a step's journaled
+ * verdict, and what would prove it is a replay, which needs a built world."
+ * There is no built world any more — `runWorkflow`
+ * (`@alexkroman1/aai-runtime/testing`) runs this body on the real replay engine
+ * over an in-memory journal. The last block below is that claim, asserted the
+ * only way it can be: crash the desk mid-round, hand the journal to a fresh
+ * engine, and count the model calls the resume did NOT make.
  */
 
 import { FatalError } from "@alexkroman1/aai/step-errors";
 import { parseSchemaInput, schemaInputIssues } from "@alexkroman1/aai/testing";
 import { installStubGateway as stubGateway } from "@alexkroman1/aai/testing/vitest";
+import { runWorkflow } from "@alexkroman1/aai-runtime/testing";
 import { beforeEach, describe, expect, test, vi } from "vitest";
 import agentDef, { MAX_ROUNDS, redline } from "./agent.ts";
 import {
@@ -209,3 +210,128 @@ describe("the steps", () => {
     });
   });
 });
+
+/**
+ * The loop, against a real durable engine.
+ *
+ * `runWorkflow` starts the declared workflow on the engine `aai dev` uses, over
+ * an in-memory journal, and drives one delivery at a time. The desk has no
+ * suspension, so what these assert is the OTHER durable property — the one this
+ * module doc says is the shape neither sibling template has: a loop whose exit
+ * is decided at run time, on a step's journaled verdict, and which therefore has
+ * to take the same branch on every walk.
+ *
+ * The model is the whole world here (`writeDraft`, `critiqueDraft` and
+ * `reviseDraft` are all `stepGenerate*Classified`), so `stubGateway`'s scripted
+ * replies ARE the run, and its call log is what proves a replay did not pay for
+ * a round twice. Scripted in body order, with the last reply repeating.
+ */
+describe("the run is DURABLE", () => {
+  const BRIEF = {
+    brief: "Explain why a 402 is the interesting status code for an agent to meet.",
+    audience: "engineers" as const,
+    mustCover: [],
+  };
+  const SHIP = '{"verdict":"ship","score":9,"notes":[]}';
+  const REVISE = '{"verdict":"revise","score":5,"notes":["thin in the middle"]}';
+
+  beforeEach(() => {
+    vi.stubEnv("ASSEMBLYAI_API_KEY", "sk-test");
+  });
+
+  test("stops on the CRITIC's verdict, and journals one step per call site reached", async () => {
+    const model = stubGateway(["A draft about 402s.", SHIP]);
+    const run = await runWorkflow(redline, { ...BRIEF, rounds: 3 }, { name: "redline" });
+
+    expect(run.status).toBe("completed");
+    expect(run.output?.shipped).toBe(true);
+    expect(run.output?.roundsRun).toBe(1);
+    // Two rounds of budget went unspent, so the journal holds one critique and
+    // no revision at all — `(name, occurrence)` identity means the entries are
+    // the record of which call sites the body actually reached.
+    expect(run.steps.map((step) => step.key)).toEqual(["critiqueDraft#0", "writeDraft#0"]);
+    expect(model).toHaveLength(2);
+  });
+
+  test("journals a round per iteration, so `critiqueDraft#1` is round two", async () => {
+    // Revise, revise, then ship: the loop runs to its budget of three.
+    const model = stubGateway([
+      "A draft about 402s.",
+      REVISE,
+      "A better draft.",
+      REVISE,
+      "A better draft still.",
+      SHIP,
+    ]);
+    const run = await runWorkflow(redline, { ...BRIEF, rounds: 3 }, { name: "redline" });
+
+    expect(run.status).toBe("completed");
+    expect(run.output?.roundsRun).toBe(3);
+    expect(run.steps.map((step) => step.key)).toEqual([
+      "critiqueDraft#0",
+      "critiqueDraft#1",
+      "critiqueDraft#2",
+      "reviseDraft#0",
+      "reviseDraft#1",
+      "writeDraft#0",
+    ]);
+    expect(model).toHaveLength(6);
+  });
+
+  test("a worker that dies mid-round replays the finished rounds instead of re-writing them", async () => {
+    // The claim the module doc makes and nothing could check: "a rate limit in
+    // round three replays rounds one and two from the journal for free and
+    // re-issues only the call that failed."
+    const model = stubGateway(["A draft about 402s.", REVISE, "A better draft.", SHIP]);
+    const run = await runWorkflow(
+      redline,
+      { ...BRIEF, rounds: 3 },
+      { name: "redline", crashAt: "reviseDraft" },
+    );
+
+    expect(run.crashed).toBe(true);
+    expect(run.steps.map((step) => step.key)).toEqual(["critiqueDraft#0", "writeDraft#0"]);
+    const spentBeforeTheCrash = model.length;
+    expect(spentBeforeTheCrash).toBe(2);
+
+    await run.restart();
+    expect(run.status).toBe("completed");
+    expect(run.output?.shipped).toBe(true);
+    // Four calls in total for a run that reached four call sites — so the
+    // resume paid for the revision and the second critique and NOT for the
+    // draft or the first critique, which came back out of the journal.
+    expect(model).toHaveLength(4);
+    expect(run.steps.map((step) => step.key)).toEqual([
+      "critiqueDraft#0",
+      "critiqueDraft#1",
+      "reviseDraft#0",
+      "writeDraft#0",
+    ]);
+  });
+
+  test("takes the SAME branch on the walk after a crash, because the verdict is journaled", async () => {
+    // The replay-stability claim itself. The critic said "ship" on the first
+    // walk; the model is then scripted to say "revise" to anything asked
+    // afterwards. A body that re-decided the loop on a fresh model call would
+    // carry on revising — a body that reads its journaled verdict cannot.
+    const model = stubGateway(["A draft about 402s.", SHIP, REVISE]);
+    const run = await runWorkflow(
+      redline,
+      { ...BRIEF, rounds: 3 },
+      { name: "redline", crashAt: "critiqueDraft" },
+    );
+    // Crashed BEFORE the critique's body ran, so nothing is journaled but the
+    // draft — and the resume is what reaches the verdict.
+    expect(run.crashed).toBe(true);
+    expect(run.steps.map((step) => step.key)).toEqual(["writeDraft#0"]);
+
+    await run.restart();
+    expect(run.status).toBe("completed");
+    expect(run.output?.shipped).toBe(true);
+    expect(run.output?.roundsRun).toBe(1);
+    // Two calls: the draft, and the critique the crash cost. The third scripted
+    // reply — the one that would have kept the loop going — is never asked for,
+    // because the loop read its verdict out of the journal.
+    expect(model).toHaveLength(2);
+  });
+});

package/dist/templates/research-workflow/agent.eval.test.ts

@@ -250,7 +250,7 @@ describeEval(
         // durable `sleep` rather than suspending, so what a case can honestly
         // claim is that the body asked — and that is the assertion that fails
         // if the suspension is ever deleted.
-        expect(run?.slept).toEqual([{ duration: REVIEW_DELAY_MS }]);
+        expect(run?.slept).toEqual([{ label: "reviewWindow", duration: REVIEW_DELAY_MS }]);
 
         // Six model calls, all through the step slot: the fan-out's width came
         // from a journaled stage rather than from anything the body recomputed.