@alexkroman1/aai-cli 7.0.0 → 8.0.0

package/dist/templates/podcast-digest/workflows/slack.ts

@@ -1,42 +1,35 @@
 // Copyright 2026 the AAI authors. MIT license.
 /**
- * Delivering the digest — the half of this template that is about a THIRD
- * PARTY's contract rather than about podcasts.
+ * Delivering the digest — which is now almost entirely a question of what the
+ * MESSAGE says, because where it goes is the SDK's job.
  *
- * Slack has two things people call "a webhook URL" and they take different
- * bodies. Getting this wrong is the single most common way a run of this
- * template ends in a red 400 with nobody able to say why, so the distinction is
- * modelled here rather than left to whoever pastes the URL:
+ * This module used to carry the whole third-party contract: Slack's two
+ * webhook shapes and the branch between them, Block Kit assembly, mrkdwn
+ * escaping, the 4xx/5xx split and the advice each refusal deserves. All of it
+ * is `@alexkroman1/aai/channels` now — `slackChannel()` names the destination,
+ * `sendToChannelClassified` posts and classifies — and what is left here is the
+ * part that is actually about podcasts: turning episode digests into a
+ * {@link ChannelMessage}.
  *
- * | URL | What it wants |
- * | --- | --- |
- * | `hooks.slack.com/services/…` | A classic incoming webhook: Block Kit |
- * | `hooks.slack.com/triggers/…` | A workflow trigger: FLAT string variables |
+ * That split is the point of the channel concept. Every one of those rules is
+ * about SLACK rather than about this template, and a template is the wrong
+ * place to learn them: the trigger-vs-incoming-webhook distinction alone is
+ * the most common way a run of this ends in a red 400 with nobody able to say
+ * why.
  *
- * Send Block Kit to a trigger URL and Slack rejects the whole payload, because
- * a trigger's body is a flat map of the variables its workflow declared — there
- * is no `blocks` variable and there never will be. {@link renderSlackPayload}
- * branches on the URL for exactly that reason, and it is a pure function so the
- * branch is testable without a network.
+ * ## What a step still owns
  *
- * ## Why the trigger case needs a parameter NAME from the user
- *
- * A trigger's variables are named by whoever built the Slack workflow. This
- * template cannot know that name, so it is an input field defaulting to `text`
- * — the name Slack's own example uses. That is an unusual thing to put in a
- * form and it is the honest option: the alternative is guessing, and a guess
- * fails with `invalid_arguments` and no indication of which name was wrong.
+ * The `"use step"` stays HERE, and it has to. The Workflow DevKit's builder
+ * rewrites step bodies it finds in this directory; one written inside the SDK
+ * would be transformed by nothing, so it would run inline with no journal and
+ * no retry while looking durable at the call site.
  */
 
-import { report, stepFetch } from "@alexkroman1/aai/step";
-import { toStepError } from "@alexkroman1/aai/step-errors";
-import { responseErrorMessage } from "@alexkroman1/aai/utils";
-import { FatalError } from "workflow";
+import { type ChannelMessage, slackChannel } from "@alexkroman1/aai/channels";
+import { report } from "@alexkroman1/aai/step";
+import { sendToChannelClassified } from "@alexkroman1/aai/step-errors";
 import type { EpisodeDigest } from "./digest.ts";
 
-/** Slack is not slow; a post that has not answered in 30s is not going to. */
-const POST_TIMEOUT_MS = 30_000;
-
 /** Everything the message needs, so rendering can stay a pure function. */
 export type SlackDigestInput = {
   slackWebhookUrl: string;
@@ -48,162 +41,48 @@ export type SlackDigestInput = {
 };
 
 /**
- * The step: post one digest, and classify the failure honestly.
+ * The step: post one digest.
  *
- * The 4xx/5xx split is the whole reason this is not a one-line `stepFetch`. A
- * revoked webhook, an unpublished workflow and a wrong variable name all answer
- * 4xx and will answer 4xx identically on every retry — retrying them burns the
- * DevKit's attempts and delays the real error by minutes. A 5xx is Slack having
- * a bad minute, which is precisely what retries are for, and `toStepError`
- * carries any `Retry-After` Slack named into the schedule.
+ * It is three lines because the interesting decisions moved. `slackChannel()` builds
+ * the descriptor, {@link renderDigestMessage} says what the message contains,
+ * and `sendToChannelClassified` does the render-post-classify round — throwing
+ * a `FatalError` on a 4xx (a revoked webhook and a wrong variable name answer
+ * identically on every retry, so retrying only delays the real error) and a
+ * `RetryableError` carrying Slack's own `Retry-After` on a 5xx.
  */
 export async function sendDigestToSlack(input: SlackDigestInput): Promise<string> {
   "use step";
 
   await report("Posting the digest to Slack.");
-  const response = await stepFetch(input.slackWebhookUrl, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(renderSlackPayload(input)),
-    signal: AbortSignal.timeout(POST_TIMEOUT_MS),
-  });
-  if (response.ok) return (await response.text()) || "ok";
-
-  // `responseErrorMessage` rather than `await response.text()` and a hand-rolled
-  // truncation: it prefers a JSON `error` field when the body has one — which
-  // Slack's does — and falls back to the status with a bounded preview.
-  const detail = await responseErrorMessage(response, "Slack webhook post");
-  if (response.status >= 400 && response.status < 500) {
-    throw new FatalError(`${slackAdvice(input.slackWebhookUrl, detail)} (HTTP ${response.status})`);
-  }
-  throw toStepError(response, `Slack webhook post failed: HTTP ${response.status}. ${detail}`);
-}
-
-/**
- * The sentence a person can act on, chosen from what the URL and body say.
- *
- * `workflow_not_published` is called out by name because it is the one 4xx with
- * a fix that is not "check your URL" — the URL is fine and the workflow behind
- * it was never published — and nothing in Slack's generic message says so.
- */
-export function slackAdvice(webhookUrl: string, detail: string): string {
-  if (isSlackWorkflowTriggerUrl(webhookUrl)) {
-    if (detail.includes("workflow_not_published")) {
-      return "That Slack workflow trigger exists but its workflow is not published. Publish it in Slack, then start a new run.";
-    }
-    return `Slack rejected the workflow trigger: ${detail}. Check that the text parameter matches a variable the workflow declares.`;
-  }
-  return `Slack rejected the incoming webhook: ${detail}. Check that the webhook is still active and has not been revoked.`;
-}
-
-/** A workflow trigger, which takes flat variables and not Block Kit. */
-export function isSlackWorkflowTriggerUrl(url: string): boolean {
-  try {
-    const parsed = new URL(url);
-    return (
-      parsed.hostname.toLowerCase() === "hooks.slack.com" &&
-      parsed.pathname.startsWith("/triggers/")
-    );
-  } catch {
-    return false;
-  }
+  return await sendToChannelClassified(
+    slackChannel({ webhookUrl: input.slackWebhookUrl, textParam: input.slackWorkflowTextParam }),
+    renderDigestMessage(input),
+  );
 }
 
 /**
- * Accepted at the form's edge: an incoming webhook or a workflow trigger, on
- * one of Slack's two webhook hosts.
+ * The digest as a channel message — PURE, so a spec asserts what a run would
+ * post without a network, and without knowing Slack's payload shape.
  *
- * A host check rather than "is it a URL", and this is a security boundary as
- * much as a usability one: this value becomes the target of a POST carrying
- * summarized content, so anything that is not Slack is an exfiltration target
- * somebody typed into a form. Refusing at `start()` is a 400 at the call site;
- * refusing later would be a failed run after transcription had already been paid for.
+ * `text` is the notification line and, on a Slack workflow trigger, the whole
+ * message: the SDK folds the rest into it when the destination has no rich
+ * format. So it says how many episodes rather than repeating the headline.
  */
-export function isSlackWebhookUrl(value: string): boolean {
-  try {
-    const parsed = new URL(value);
-    const host = parsed.hostname.toLowerCase();
-    return (
-      parsed.protocol === "https:" &&
-      (host === "hooks.slack.com" || host === "hooks.slack-gov.com") &&
-      parsed.pathname.length > 1
-    );
-  } catch {
-    return false;
-  }
-}
-
-/** Block Kit, or flat variables — see the module doc. */
-export function renderSlackPayload(input: SlackDigestInput): Record<string, unknown> {
-  if (isSlackWorkflowTriggerUrl(input.slackWebhookUrl)) {
-    return { [input.slackWorkflowTextParam || "text"]: renderPlainTextDigest(input) };
-  }
-  return renderSlackMessage(input);
-}
-
-/** The trigger body: one string, because that is all a variable can hold. */
-export function renderPlainTextDigest(input: SlackDigestInput): string {
-  return [
-    digestHeadline(input),
-    `Feeds: ${input.podcastChannels}`,
-    "",
-    ...input.episodes.flatMap((episode) => [
-      `${episode.title} — ${episode.podcastTitle}`,
-      episode.url,
-      episode.summary,
-      ...episode.keyPoints.map((point) => `- ${point}`),
-      "",
-    ]),
-  ]
-    .join("\n")
-    .trim();
-}
-
-/** The incoming-webhook body: Block Kit, with `text` as the notification line. */
-function renderSlackMessage(input: SlackDigestInput): Record<string, unknown> {
+export function renderDigestMessage(input: SlackDigestInput): ChannelMessage {
   return {
-    // Not decoration — this is what a push notification and a screen reader
-    // read. A Block Kit payload with no `text` notifies as "[no preview]".
     text: `${digestHeadline(input)}: ${input.episodes.length} episode summaries`,
-    blocks: [
-      { type: "header", text: { type: "plain_text", text: digestHeadline(input) } },
-      {
-        type: "section",
-        text: { type: "mrkdwn", text: `Feeds: ${escapeSlack(input.podcastChannels)}` },
-      },
-      { type: "divider" },
-      ...input.episodes.flatMap((episode) => [
-        {
-          type: "section",
-          text: { type: "mrkdwn", text: renderEpisodeBlock(episode) },
-        },
-        { type: "divider" },
-      ]),
-    ],
+    heading: digestHeadline(input),
+    subtitle: `Feeds: ${input.podcastChannels}`,
+    sections: input.episodes.map((episode) => ({
+      title: episode.title,
+      url: episode.url,
+      subtitle: episode.podcastTitle,
+      body: episode.summary,
+      bullets: episode.keyPoints,
+    })),
   };
 }
 
-function renderEpisodeBlock(episode: EpisodeDigest): string {
-  return [
-    `*<${escapeSlack(episode.url)}|${escapeSlack(episode.title)}>*`,
-    `_${escapeSlack(episode.podcastTitle)}_`,
-    escapeSlack(episode.summary),
-    ...episode.keyPoints.map((point) => `• ${escapeSlack(point)}`),
-  ].join("\n");
-}
-
 function digestHeadline(input: SlackDigestInput): string {
   return `Podcast digest ${input.digestNumber}/${input.totalDigests}`;
 }
-
-/**
- * The three characters Slack's mrkdwn reserves.
- *
- * Only three, and only these: Slack's own escaping rules say `&`, `<` and `>`
- * and nothing else, so escaping more would put backslashes in front of
- * apostrophes in every summary. `&` first, or the ampersands introduced by the
- * other two get double-escaped.
- */
-export function escapeSlack(text: string): string {
-  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
-}

package/dist/templates/recap-workflow/agent.eval.test.ts

@@ -0,0 +1,366 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does this desk actually behave? Run it with `aai eval`.
+//
+// `agent.test.ts` drives the five tools against a STUBBED `ctx.workflows`, the
+// steps against a stubbed provider, and the body's two helpers with `sleep` and
+// `createHook` mocked. What none of those tiers can say is whether the AGENT —
+// a model, reading this system prompt, holding these five tools — hands a
+// recording off at all, whether the run it starts is the same run a later turn
+// finds, and whether a caller who says "forget it" is told the truth about what
+// cancelling does. That is what this file is for, and it is only possible
+// because `describeEval` opens a real workflow engine per case and hands its
+// client to the session.
+//
+// Three boundaries, each of which a case here would be dishonest to imply:
+//
+//   * **The engine is not durable.** No journal, no replay, no retry, and a
+//     `sleep()` is RECORDED rather than taken — so the poll loop below runs at
+//     full speed and the only way to observe a run in flight is to hold the
+//     provider's answer, which is what `hold` does.
+//   * **`createHook()` throws untransformed**, so the RETENTION GATE — this
+//     template's headline port of Temporal's `expense` — cannot be reached at
+//     all here, and neither can `keep_transcript`'s signal: a run that gets past
+//     `summarize` fails on the hook. Nothing below asserts the gate, its three
+//     outcomes, or its safe default; `agent.test.ts` drives
+//     `askWhetherToKeep` directly for those, and it is the only tier that can.
+//   * **The provider is scripted, in BOTH modes**, through `stepFetch`'s
+//     published slot — so the transcription, the recap's model call and the
+//     compensating DELETE are all deterministic and free, while the SESSION's
+//     model stays live and is what a live run measures.
+//
+// And what no eval here can see: anything below the audio boundary.
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
+import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import { recap, SAMPLE_RECORDING } from "./shared.ts";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * Load-bearing, and not applied by `agent()` — the BUILD enumerates `tools/`,
+ * so an eval driving the raw default export would measure a desk with none of
+ * its five tools. This template declares its prompt inline in `agent.ts`, so
+ * there is no `system-prompt.md` for `withSystemPrompt` to layer on.
+ *
+ * The glob is written here rather than reached for from a shared helper because
+ * this file SHIPS — see `agent.test.ts`.
+ */
+const agentDef = withDiscoveredTools(
+  authoredAgent,
+  import.meta.glob("./tools/*.ts", { eager: true }),
+);
+
+/**
+ * The key every step reads with `requireStepEnv` — the one name `agent.ts`
+ * declares in `requiredEnv`.
+ *
+ * Passed as the agent env so the eval's workflow engine publishes it: the
+ * provider calls are answered by a fake, but each asks for the key BEFORE it
+ * makes its request, so a run with no key fails on the credential rather than
+ * reaching the script. The ENVIRONMENT and nothing else — a template may not
+ * read a developer's CLI config.
+ */
+const EVAL_ENV = { ASSEMBLYAI_API_KEY: process.env.ASSEMBLYAI_API_KEY ?? "eval-scripted-key" };
+
+/** The job id the scripted provider mints for every submission. */
+const TRANSCRIPT_ID = "t_eval_1";
+
+/** What the scripted transcript says, once it completes. */
+const TRANSCRIPT_TEXT =
+  "Smoke from the Canadian wildfires reached the eastern seaboard overnight, " +
+  "and air quality indexes went into the unhealthy range.";
+
+/** The recap the scripted model writes. `spoken` is the field the phone reads. */
+const RECAP_JSON = JSON.stringify({
+  headline: "Wildfire smoke reaches the east coast",
+  points: ["Smoke crossed overnight", "Air quality is unhealthy", "Advisories are in force"],
+  spoken: "Wildfire smoke drifted east overnight and pushed air quality into the unhealthy range.",
+});
+
+/** How the scripted provider ends a job. */
+type Ending = "completed" | "error";
+
+/** A scripted provider, and the gate that holds its first status answer. */
+type ScriptedProvider = {
+  /** Every step request, in order — what the run really asked the provider. */
+  readonly calls: readonly { url: string; method: string; body?: unknown }[];
+  /** Let a held first poll answer. Idempotent. */
+  readonly release: () => void;
+};
+
+/**
+ * Answer AssemblyAI's batch API and the LLM gateway in memory.
+ *
+ * Through `stepFetch`'s published slot rather than `vi.stubGlobal("fetch")`,
+ * which is the path a step really takes — and which leaves the SESSION's model
+ * live, so a live case still measures the agent. An unrecognised request THROWS
+ * rather than answering an empty 200: a step calling something nobody expected
+ * is a finding, where an empty body reads as a provider that said nothing.
+ *
+ * `hold` keeps the FIRST poll pending, and it is the only way to observe a run
+ * that is still going: a durable `sleep` is skipped here, so an unheld run
+ * burns its whole poll loop in milliseconds.
+ */
+function stubProvider(options: { hold?: boolean; ending?: Ending } = {}): ScriptedProvider {
+  const gate = Promise.withResolvers<void>();
+  let polls = 0;
+  const stub = installStubStepFetch(async (request) => {
+    if (request.url.includes("/chat/completions")) {
+      return { body: { choices: [{ message: { content: RECAP_JSON } }] } };
+    }
+    if (request.method === "POST") return { body: { id: TRANSCRIPT_ID, status: "queued" } };
+    // The compensation. A real DELETE removes the transcript from the account,
+    // which is what makes "a failed run leaves nothing behind" a claim rather
+    // than a comment — so the assertion that matters is that this was CALLED.
+    if (request.method === "DELETE") return { body: {} };
+    if (request.method === "GET") {
+      polls += 1;
+      if (options.hold === true && polls === 1) await gate.promise;
+      return options.ending === "error"
+        ? { body: { status: "error", error: "that recording could not be decoded" } }
+        : { body: { status: "completed", text: TRANSCRIPT_TEXT, audio_duration: 254 } };
+    }
+    throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
+  });
+  return { calls: stub.calls, release: () => gate.resolve() };
+}
+
+/** `request_recap`'s two answers — it started one, or it found the live one. */
+const RecapStart = z.union([
+  z.object({ started: z.literal(true), runId: z.string().min(1) }),
+  z.object({ started: z.literal(false), runId: z.string().min(1), note: z.string() }),
+]);
+
+/** `cancel_recap`'s answer. */
+const Cancelled = z.object({ cancelled: z.boolean(), note: z.string() });
+
+/**
+ * Every `request_recap` answer in a turn, parsed.
+ *
+ * Parsed rather than regexed: a tool result reaches the event stream as a
+ * serialized string, and a shape that stopped matching should fail HERE naming
+ * the field instead of handing the next assertion `undefined`.
+ */
+function recapStarts(calls: readonly EvalToolCall[]): z.infer<typeof RecapStart>[] {
+  return calls
+    .filter((one) => one.name === "request_recap" && one.result !== undefined)
+    .map((one) => RecapStart.parse(JSON.parse(String(one.result))));
+}
+
+/** The run id the first `request_recap` of this turn reported. */
+function startedRunId(calls: readonly EvalToolCall[]): string {
+  const [first] = recapStarts(calls);
+  if (first === undefined) {
+    throw new Error(
+      `the desk called no request_recap: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
+    );
+  }
+  return first.runId;
+}
+
+/** Every tool call in this turn that READS a run, whichever the model picked. */
+function readbacks(calls: readonly EvalToolCall[]): readonly EvalToolCall[] {
+  return calls.filter((one) => one.name === "recap_status" || one.name === "recap_progress");
+}
+
+/** Requests of one method the run has made so far. */
+function requests(provider: ScriptedProvider, method: string) {
+  return provider.calls.filter((one) => one.method === method);
+}
+
+/** `text` as a regex that matches only itself. */
+function literal(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** The one utterance every case that starts work opens with. */
+const ASK = "Can you write up that recording for me?";
+
+/** The scripted session turn that answers {@link ASK}. */
+const START_TURN = [
+  { tool: "request_recap", args: {} },
+  "I've started writing that up — I'll let you know when it lands.",
+] as const;
+
+/**
+ * Let the run finish before the case ends.
+ *
+ * Not tidiness: the scripted provider is unpublished when the test that
+ * installed it finishes, so a body still mid-flight would make its next request
+ * against whatever the next case publishes — or against the real provider, with
+ * a real key. Every run here ends FAILED, on `createHook()`; see the header.
+ */
+async function drain(workflows: EvalWorkflows | undefined, provider: ScriptedProvider) {
+  provider.release();
+  for (const run of await (workflows?.runs() ?? [])) await workflows?.settle(run.runId, recap);
+}
+
+describeEval(
+  agentDef,
+  (test) => {
+    test(
+      "starts one run for the caller and will not pay for a second",
+      async ({ session, workflows }) => {
+        // Held, so the first run is unambiguously still live when the caller
+        // asks again — which is the state the live-run check exists for.
+        const provider = stubProvider({ hold: true });
+
+        const first = await session.say(ASK);
+        const runId = startedRunId(first.toolCalls);
+        const again = await session.say("Actually, start it again from scratch, please.");
+
+        // Temporal's workflow-id reuse policy, as this desk spells it: a caller
+        // who asks twice is told about the run they already have. WHETHER the
+        // model calls the tool a second time is its business — it may simply
+        // remember — so each call it did make has to have been refused with the
+        // run it found.
+        for (const answer of recapStarts(again.toolCalls)) {
+          expect(answer.started).toBe(false);
+          expect(answer.runId).toBe(runId);
+        }
+
+        // The half that is not vacuous either way, and the one the caller pays
+        // for: ONE run, and ONE submission to the provider.
+        const runs = await (workflows?.runs() ?? []);
+        expect(runs.map((one) => one.runId)).toEqual([runId]);
+        expect(runs[0]?.workflow).toBe("recap");
+        const posts = requests(provider, "POST");
+        expect(posts).toHaveLength(1);
+        // And it submitted the recording the DESK supplies, because a phone
+        // caller cannot read a URL aloud.
+        expect(JSON.parse(String(posts[0]?.body))).toMatchObject({
+          audio_url: SAMPLE_RECORDING,
+          speaker_labels: true,
+        });
+
+        await drain(workflows, provider);
+      },
+      {
+        stubReply: [
+          ...START_TURN,
+          { tool: "request_recap", args: {} },
+          "There's already one running for you — I'll tell you as soon as it lands.",
+        ],
+      },
+    );
+
+    test(
+      "reads the live run back rather than guessing at it",
+      async ({ session, workflows }) => {
+        const provider = stubProvider({ hold: true });
+
+        const started = await session.say(ASK);
+        const runId = startedRunId(started.toolCalls);
+        const turn = await session.say("How's that going?");
+
+        const read = readbacks(turn.toolCalls);
+        expect(read.length).toBeGreaterThan(0);
+        const answered = read.map((one) => one.result ?? "").join("\n");
+
+        // The load-bearing half: that readback happened while the run really
+        // was in flight, waiting on the provider — which is the only state
+        // these two tools exist for.
+        const runs = await (workflows?.runs() ?? []);
+        const live = runs.find((one) => one.runId === runId);
+        expect(live?.status).toBe("running");
+        expect(requests(provider, "GET")).toHaveLength(1);
+
+        // WHICH readback tool the model picks is its business — the prompt
+        // offers both — so the claim is about what it was TOLD, and each has its
+        // own shape: `recap_progress` hands back the run's own latest line and
+        // `recap_status` the snapshot's status. Compared against what the RUN
+        // really wrote rather than against a literal, because which line that is
+        // depends on how far the body got: with the durable `sleep` skipped here
+        // the `PATIENCE` race resolves at once, so the note the caller would
+        // hear two minutes in is already written.
+        const lastLine = String(live?.reported.at(-1));
+        expect(live?.reported.length).toBeGreaterThan(0);
+        expect(answered).toMatch(new RegExp(`${literal(lastLine)}|Still working on that one`));
+        expect(answered).not.toMatch(/Nothing started yet/);
+
+        await drain(workflows, provider);
+      },
+      {
+        stubReply: [
+          ...START_TURN,
+          { tool: "recap_progress", args: {} },
+          "It's with the transcription service now — nothing back yet.",
+        ],
+      },
+    );
+
+    test(
+      "cancelling says plainly what it did NOT roll back, and really does not",
+      async ({ session, workflows }) => {
+        const provider = stubProvider({ hold: true });
+
+        const started = await session.say(ASK);
+        const runId = startedRunId(started.toolCalls);
+        const turn = await session.say("Forget it — cancel that, please.");
+
+        const cancel = turn.toolCalls.find((one) => one.name === "cancel_recap");
+        const answer = Cancelled.parse(JSON.parse(String(cancel?.result)));
+        expect(answer.cancelled).toBe(true);
+        // The sentence is a documented promise of this template, not a
+        // decoration: cancellation is NOT cooperative here, so the transcript
+        // the run had already created stays on the account and the caller is
+        // told so rather than left to assume a rollback.
+        expect(answer.note).toMatch(/left behind/);
+        expect(answer.note).toMatch(/does not roll back/);
+
+        // And it is TRUE, which is the part only an eval with a real run can
+        // check: the run is cancelled, and no compensating DELETE went out.
+        const runs = await (workflows?.runs() ?? []);
+        expect(runs.find((one) => one.runId === runId)?.status).toBe("cancelled");
+        expect(requests(provider, "DELETE")).toEqual([]);
+
+        // Released after the assertions on purpose: the body runs on regardless
+        // (there is no queue here to stop delivering to, and Temporal's
+        // deliver-cancellation-into-the-workflow is the one thing this template
+        // says does not port), so anything it does afterwards is not what the
+        // caller was told about.
+        await drain(workflows, provider);
+      },
+      {
+        stubReply: [
+          ...START_TURN,
+          { tool: "cancel_recap", args: {} },
+          "Stopped it. The partial transcript stays on file — cancelling doesn't undo that.",
+        ],
+      },
+    );
+
+    test("a run that fails after creating a transcript deletes it again", async ({ workflows }) => {
+      // Started from the CASE rather than through a tool, because the subject
+      // is the saga and the failure has to be injected: the provider refuses
+      // the job, which is the branch that unwinds the compensation stack.
+      // `request_recap` is what the other three cases drive.
+      const provider = stubProvider({ ending: "error" });
+
+      const run = await workflows?.run(recap, {
+        url: SAMPLE_RECORDING,
+        requestedBy: "eval-session",
+      });
+
+      expect(run?.status).toBe("failed");
+      expect(run?.error).toMatch(/could not transcribe/);
+      // The unwind, off the run's own narration — one compensation, named.
+      const narration = run?.reported.join("\n") ?? "";
+      expect(narration).toMatch(/undoing 1 step/);
+      expect(narration).toMatch(`Discarding transcript ${TRANSCRIPT_ID}.`);
+      // And it really happened: the transcript this run created was deleted
+      // from the account, which is the promise "a failed recap leaves nothing
+      // behind" rests on. An undo registered BEFORE its step, or a `catch`
+      // that stopped compensating, fails here.
+      expect(requests(provider, "DELETE").map((one) => one.url)).toEqual([
+        `https://api.assemblyai.com/v2/transcript/${TRANSCRIPT_ID}`,
+      ]);
+    });
+  },
+  { env: EVAL_ENV },
+);