@alexkroman1/aai-cli 7.0.0 → 8.0.0

package/dist/templates/call-audit/agent.eval.test.ts

@@ -0,0 +1,474 @@
+// Copyright 2026 the AAI authors. MIT license.
+// An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
+// `aai eval`.
+//
+// `agent.test.ts` covers everything this desk DECIDES — every ffmpeg argv, both
+// analysis parsers, and where `planSegments` cuts — as pure functions, because
+// `workflows/media.ts` exists so that it can. What it cannot do is spawn, so
+// the one thing it never sees is whether ffmpeg AGREES: whether the argv this
+// desk builds really produces the pauses it plans against, and whether a plan
+// derived from those pauses really addresses audio that exists.
+//
+// That is what this file is for. **Every case here runs a real ffmpeg** — five
+// invocations per run, over audio built sample by sample so the pauses in it
+// are known — and then checks where the desk cut. The twelve-byte planning bug
+// `durationSeconds` documents was found exactly this way, and it is the class of
+// bug that cannot be found any other way.
+//
+// It therefore needs `ffmpeg` and `ffprobe` on `PATH` (or `AAI_FFMPEG_PATH` /
+// `AAI_FFPROBE_PATH`), which is what the template's own doc says a developer
+// needs for anything that is not already a WAV. Measured against ffmpeg 7.1.5.
+// A missing binary fails these cases with the SDK's installable message, which
+// is the same thing `aai dev` does on a laptop without one.
+//
+// `describeWorkflowEval` picks the providers for you and says which it picked:
+//
+//   * with `ASSEMBLYAI_API_KEY` — a LIVE run. The `{ live: true }` case below
+//     downloads a real four-and-a-half-minute recording, levels it with a real
+//     ffmpeg, fans three real requests out at its real pauses, and asks a real
+//     model to audit what came back. That spends money and about a minute.
+//   * without one — a SCRIPTED run: the same ffmpeg, the same plan, the same
+//     seams, with the transcription endpoint and the model answered in memory.
+//
+// Three of the four cases are SCRIPTED IN BOTH MODES, deliberately. Their claims
+// are about WHERE THE DESK CUT and what it did at the seam, and a live provider
+// can neither confirm nor deny either — it can only transcribe whatever it is
+// sent. The live case is the one that answers "is the transcript really of the
+// recording, and is the audit really of the transcript".
+//
+// TWO THINGS ARE FAKED IN EVERY MODE, and both are harness limits rather than
+// choices:
+//
+//   * the UPLOAD STORE, because it is this app's own storage (a row and a blob)
+//     rather than a provider, and an eval has neither. `{ writable: true }` is
+//     not optional here: this desk WRITES two files.
+//   * the VOICE — see `speakATone` below, which carries both reasons.
+//
+// WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
+// in the path, a `"use workflow"` body is an ordinary async function — no
+// journal, no replay, and no per-step retry, so the resume-after-segment-27
+// property is NOT exercised and a rate-limited live run fails where a deployed
+// one would have ridden it out. `aai-cli`'s `dev-workflow.scenario.test.ts` is
+// the tier that really resumes a run.
+import { spawnSync } from "node:child_process";
+import { encodeWav } from "@alexkroman1/aai/step";
+import { installStubTranscribe, installStubUploads } from "@alexkroman1/aai/testing/vitest";
+import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { describe, expect, test } from "vitest";
+import agentDef, { audit } from "./agent.ts";
+import { ANALYSIS_FORMAT, BYTES_PER_SECOND, MAX_SEGMENT_SECONDS } from "./workflows/media.ts";
+
+/**
+ * Both binaries, or neither — every case here decodes, and three of the four
+ * read `ffprobe` back to check what the desk did.
+ */
+const HAVE_FFMPEG = ["ffmpeg", "ffprobe"].every(
+  (bin) => spawnSync(bin, ["-version"], { stdio: "ignore" }).status === 0,
+);
+
+const HOW_TO =
+  "Install ffmpeg (`apt-get install ffmpeg`, `brew install ffmpeg`) or point\n" +
+  "AAI_FFMPEG_PATH / AAI_FFPROBE_PATH at binaries. A deployed guest always has them.";
+
+// Biome's `noSkippedTests` flags the `describe.skip(…)` CALL form, so the gated
+// suite references it instead — exactly as `aai/host/ffmpeg.scenario.test.ts`
+// and `_pg-test-utils.ts` do.
+const skipSuite = describe.skip;
+
+/**
+ * `describeWorkflowEval`, gated on a real ffmpeg — and the skip ANNOUNCES itself.
+ *
+ * This file used to call `describeWorkflowEval` directly, on the reasoning (still
+ * in the header above) that "a missing binary fails these cases with the SDK's
+ * installable message, which is the same thing `aai dev` does on a laptop without
+ * one". That is right for a laptop and wrong for a RUNNER: `check.yml` installs
+ * ffmpeg on the Linux leg only — deliberately, because the scenario suite it was
+ * added for is OS-independent and brew costs minutes — so when `check:eval` joined
+ * that same job, the macOS leg went permanently red on a binary nobody had decided
+ * it should have.
+ *
+ * So this follows `describeWithFfmpeg` (`aai/host/ffmpeg.scenario.test.ts`), which
+ * follows `describeWithPg`: skip loudly, and let **`AAI_REQUIRE_FFMPEG`** — which
+ * the Linux leg's install step sets — turn the skip into a hard failure, so a
+ * broken install step cannot read as a green run. The coverage is unchanged on
+ * every machine that has the binary, which includes all of CI's Linux legs.
+ */
+const describeWorkflowEvalWithFfmpeg: typeof describeWorkflowEval = (agent, define, options) => {
+  if (HAVE_FFMPEG) {
+    describeWorkflowEval(agent, define, options);
+    return;
+  }
+  if ((process.env.AAI_REQUIRE_FFMPEG ?? "") !== "") {
+    throw new Error(`AAI_REQUIRE_FFMPEG is set but no ffmpeg was found.\n${HOW_TO}`);
+  }
+  console.warn(`\n[skipped: no ffmpeg] ${agent.name} eval not run.\n${HOW_TO}\n`);
+  skipSuite(agent.name, () => {
+    // Named `test`, not aliased: Biome's `noMisplacedAssertion` matches the
+    // CALLEE IDENTIFIER, so an `expect` inside a `vitestTest(…)` is an error.
+    // Same trap `describeEval`'s own signature documents.
+    test("needs a real ffmpeg", () => {
+      expect(HAVE_FFMPEG).toBe(true);
+    });
+  });
+};
+
+/** The id every case uploads the recording under. */
+const UPLOAD_ID = "upl_eval";
+
+/** The public sample recording — four and a half minutes of real speech. */
+const LIVE_RECORDING = "https://assembly.ai/wildfires.mp3";
+
+/** Long enough to force a cut: the desk's cap is 110 seconds. */
+const OVER_THE_CAP_SECONDS = 130;
+
+/** What the model is scripted to answer with. */
+const REPLY = {
+  headline: "Renewal call with Northwind",
+  risks: ["Nobody owns the migration date"],
+  actions: ["Ana to send revised pricing by Friday"],
+  spoken:
+    "The renewal is close to done, but the migration date still has nobody's name on it. " +
+    "Ana is sending revised pricing before the end of the week.",
+};
+
+/**
+ * One span of a synthetic recording: `loud` is speech, quiet is a pause.
+ *
+ * A 220 Hz sine at about 40% of full scale, which is ~27 dB above the desk's
+ * −35 dB silence floor, so `silencedetect` reads it as speech; a run of zero
+ * samples is what it reads as a pause. Built sample by sample rather than
+ * captured, because the POINT is that the pauses are at second 60 and second 62
+ * and nowhere else — which is the only way "did it cut in the pause" has a
+ * right answer.
+ */
+type Span = { seconds: number; loud: boolean };
+
+/** A 16 kHz mono 16-bit WAV of the spans, ready to hand ffmpeg. */
+function recording(spans: readonly Span[]): Uint8Array {
+  const samples = spans.reduce((total, span) => total + Math.round(span.seconds * 16_000), 0);
+  const pcm = new Uint8Array(samples * 2);
+  const view = new DataView(pcm.buffer);
+  let at = 0;
+  for (const span of spans) {
+    const count = Math.round(span.seconds * 16_000);
+    for (let i = 0; i < count; i += 1, at += 1) {
+      // `setInt16(…, true)` rather than an `Int16Array`: little-endian is what
+      // `s16le` means, and a typed array would be whatever the platform is.
+      const value = span.loud
+        ? Math.round(13_000 * Math.sin((2 * Math.PI * 220 * at) / 16_000))
+        : 0;
+      view.setInt16(at * 2, value, true);
+    }
+  }
+  return encodeWav(pcm, ANALYSIS_FORMAT);
+}
+
+/**
+ * The synthesizer every case here runs on, live included — and it answers with a
+ * TONE rather than silence.
+ *
+ * Two reasons, and the second is a measurement:
+ *
+ *   * `stepSpeak` reads a PUBLISHED synthesizer, the eval engine publishes none
+ *     by default, and `@alexkroman1/aai-runtime` exports no real one to pass —
+ *     so there is no arrangement in which the voice is real. The live case
+ *     therefore claims nothing about audible audio.
+ *   * it may not answer with SILENCE, which is what `installStubSpeech` does.
+ *     `libmp3lame` ABORTS on pure digital silence — `Assertion failed:
+ *     (el >= 0), psymodel.c:576`, exit 134 on ffmpeg 7.1.5 — so the mastering
+ *     pass this template exists to demonstrate would fail on the fixture rather
+ *     than on the code, and the failure would read as a broken template.
+ */
+const speakATone = (request: { text: string; sampleRate: number }): Promise<Uint8Array> => {
+  // A second of tone per twenty characters, so a longer script really is a
+  // longer file and `audioDurationMs` is a number that means something.
+  const samples = Math.max(1, Math.round((request.text.length / 20) * request.sampleRate));
+  const pcm = new Uint8Array(samples * 2);
+  const view = new DataView(pcm.buffer);
+  for (let at = 0; at < samples; at += 1) {
+    view.setInt16(
+      at * 2,
+      Math.round(9000 * Math.sin((2 * Math.PI * 180 * at) / request.sampleRate)),
+      true,
+    );
+  }
+  return Promise.resolve(pcm);
+};
+
+/** Publish this app's own store, writable because this desk writes two files. */
+function publish(bytes: Uint8Array, name: string, type: string) {
+  return installStubUploads({ [UPLOAD_ID]: { bytes, name, type } }, { writable: true });
+}
+
+/**
+ * Answer the sync transcription endpoint and the model in memory.
+ *
+ * ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
+ * AND calls a model cannot install two, which is what `otherwise` is for. The
+ * transcription half is the SDK's own fake rather than this file's hand-typed
+ * wire: it routes off the SDK's endpoint constants, so a case cannot pass
+ * because the fake and the step agree on a typo.
+ */
+function scriptProvider(text: readonly string[]) {
+  return installStubTranscribe({
+    text,
+    otherwise: (request) =>
+      request.url.includes("/chat/completions")
+        ? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
+        : undefined,
+  });
+}
+
+describeWorkflowEvalWithFfmpeg(
+  agentDef,
+  (test) => {
+    test("cuts where ffmpeg heard a PAUSE, and reads the seam as a paragraph", async ({ app }) => {
+      // The case this template exists for, and the one that cannot be written
+      // without a real decoder: a 2-second pause at second 60 of a 130-second
+      // recording is the only place a cut may land, and where it landed decides
+      // both the byte ranges and how the transcript reads at the seam.
+      const uploads = publish(
+        recording([
+          { seconds: 60, loud: true },
+          { seconds: 2, loud: false },
+          { seconds: 68, loud: true },
+        ]),
+        "call.wav",
+        "audio/wav",
+      );
+      const provider = scriptProvider([
+        "so that is the renewal settled",
+        "on the migration date nobody has committed yet",
+      ]);
+
+      const run = await app.run(audit, { recording: UPLOAD_ID });
+
+      // The error FIRST, so a failed run names its own reason instead of
+      // reporting "expected 'failed' to be 'completed'".
+      expect(run.error).toBeUndefined();
+      expect(run.status).toBe("completed");
+      const output = run.output;
+      if (output === undefined) expect.fail("a completed run must carry an output");
+
+      // ffmpeg really ran: this is ffprobe's reading of the file, and the
+      // duration is the normalized PCM's own byte count rather than anything a
+      // container claimed.
+      expect(output.codec).toBe("pcm_s16le");
+      expect(output.durationMs).toBe(OVER_THE_CAP_SECONDS * 1000);
+
+      // TWO segments, and NEITHER is a blind cut — which is the whole claim.
+      // The desk found the pause, cut in the middle of it, and therefore owes
+      // no overlap, no seam matching and no de-duplication.
+      expect(output.segments).toBe(2);
+      expect(output.blindCuts).toBe(0);
+
+      // And the cut is at the pause's MIDPOINT, second 61 — not at 110, which
+      // is where a desk with no decoder would have had to cut.
+      const sync = provider.calls.filter((call) => call.leg === "sync");
+      expect(sync).toHaveLength(2);
+      const first = sync[0]?.body?.length ?? 0;
+      const second = sync[1]?.body?.length ?? 0;
+      // 61 seconds of audio plus a WAV header inside a multipart envelope.
+      expect(first).toBeGreaterThan(61 * BYTES_PER_SECOND);
+      expect(first).toBeLessThan(61 * BYTES_PER_SECOND + 2000);
+      // 69 seconds — and together they are the whole recording, exactly once.
+      expect(second).toBeGreaterThan(69 * BYTES_PER_SECOND);
+      expect(second).toBeLessThan(69 * BYTES_PER_SECOND + 2000);
+
+      // A cut placed in a pause is a turn boundary, so the seam is a PARAGRAPH
+      // break. This is the one place `cutInSpeech` changes an output rather
+      // than a report, and the assertion that pairs with the next case.
+      expect(output.transcript).toBe(
+        "so that is the renewal settled\n\non the migration date nobody has committed yet",
+      );
+
+      // 2 seconds of pause in 130 is 98% speech, measured by ffmpeg rather
+      // than assumed, and the loudness is a real measurement of real audio.
+      expect(output.speechPercent).toBe(98);
+      expect(output.loudnessBefore).toBeLessThan(0);
+      expect(output.loudnessBefore).toBeGreaterThan(-70);
+
+      // The narration says what it found, which is what makes a long run
+      // legible — and names the ONE pause it cut in.
+      expect(run.reported[0]).toMatch(/^Reading call\.wav \(/);
+      expect(run.reported).toContain("Levelling 2:10 of pcm_s16le to 16 kHz mono.");
+      expect(run.reported).toContain(
+        `Levelled 2:10 from ${output.loudnessBefore} LUFS, 98% speech across 1 pause.`,
+      );
+      expect(run.reported).toContain("Transcribing 0:00–1:01.");
+      expect(run.reported).toContain("Transcribing 1:01–2:10.");
+
+      // Two files written: the levelled PCM everything downstream addresses,
+      // and the mastered audit. Nothing durable was asked for.
+      expect(uploads.writes.map((one) => one.name)).toEqual(["call.pcm", "audit.mp3"]);
+      expect(run.slept).toEqual([]);
+    });
+
+    test("an unbroken monologue gets a BLIND cut, and says so", async ({ app }) => {
+      // The case the desk cannot serve, driven rather than argued: 130 seconds
+      // of unbroken speech has no pause to cut in, so it gets the arithmetic
+      // cut `transcription-workflow` always makes — at exactly the cap. Hiding
+      // that would leave a reader looking at a mangled word with no
+      // explanation, which is why `cutInSpeech` is counted and rendered.
+      publish(
+        recording([{ seconds: OVER_THE_CAP_SECONDS, loud: true }]),
+        "monologue.wav",
+        "audio/wav",
+      );
+      const provider = scriptProvider(["and then the second thing we agreed", "was the pricing"]);
+
+      const run = await app.run(audit, { recording: UPLOAD_ID });
+
+      expect(run.error).toBeUndefined();
+      const output = run.output;
+      if (output === undefined) expect.fail("a completed run must carry an output");
+
+      expect(output.segments).toBe(2);
+      // ONE blind cut — the first segment's end. The recording's own end is
+      // not a cut this planner invented, so it is not counted.
+      expect(output.blindCuts).toBe(1);
+      // 100% speech, because ffmpeg found no pause at all.
+      expect(output.speechPercent).toBe(100);
+      expect(run.reported).toContain(
+        `Levelled 2:10 from ${output.loudnessBefore} LUFS, 100% speech across 0 pauses.`,
+      );
+
+      // The cut is at the cap exactly, which is what "no candidate in range"
+      // means — and the first request is therefore 110 seconds of audio.
+      expect(run.reported).toContain("Transcribing 0:00–1:50.");
+      const sync = provider.calls.filter((call) => call.leg === "sync");
+      const first = sync[0]?.body?.length ?? 0;
+      expect(first).toBeGreaterThan(MAX_SEGMENT_SECONDS * BYTES_PER_SECOND);
+      expect(first).toBeLessThan(MAX_SEGMENT_SECONDS * BYTES_PER_SECOND + 2000);
+
+      // A blind cut landed MID-SENTENCE, so the seam is a space rather than a
+      // paragraph break. The pair with the previous case is the point: the same
+      // recording length, the same two segments, a different reading.
+      expect(output.transcript).toBe("and then the second thing we agreed was the pricing");
+    });
+
+    test("the audit comes back as a real MP3 that the run's output NAMES", async ({ app }) => {
+      // The way OUT, which is the second thing having a decoder buys: a step
+      // speaks, ffmpeg masters what it said, and the run's output carries an
+      // upload ID rather than the bytes. A short recording, because what this
+      // case is about is the last invocation rather than the fan-out.
+      const uploads = publish(recording([{ seconds: 5, loud: true }]), "quick.wav", "audio/wav");
+      scriptProvider(["short and to the point"]);
+
+      const run = await app.run(audit, { recording: UPLOAD_ID });
+
+      expect(run.error).toBeUndefined();
+      const output = run.output;
+      if (output === undefined) expect.fail("a completed run must carry an output");
+      // Under the cap, so no cut at all.
+      expect(output.segments).toBe(1);
+      expect(output.blindCuts).toBe(0);
+
+      // TWO writes, in this order, and the output names the SECOND. A run that
+      // returned the levelled PCM's id would be handing a page a headerless
+      // file nothing can play.
+      expect(uploads.writes).toHaveLength(2);
+      const [pcm, mp3] = uploads.writes;
+      if (pcm === undefined || mp3 === undefined) expect.fail("both files must be written");
+      // Named after the recording, and typed HONESTLY: raw samples with no
+      // container, so not `audio/wav` and not `audio/L16` either.
+      expect(pcm.name).toBe("quick.pcm");
+      expect(pcm.type).toBe("application/octet-stream");
+      expect(pcm.bytes.byteLength).toBe(5 * BYTES_PER_SECOND);
+      expect(output.audio).toBe(mp3.id);
+      expect(mp3.name).toBe("audit.mp3");
+      // Typed for the browser: the byte route serves what it was given, and
+      // nothing plays inline a file handed over as octet-stream.
+      expect(mp3.type).toBe("audio/mpeg");
+
+      // And it is really an MP3, which is a claim about ffmpeg's output rather
+      // than about our naming: an MPEG audio frame begins with eleven set bits,
+      // or the file opens with an ID3 tag.
+      const head = mp3.bytes.subarray(0, 3);
+      const framed = head[0] === 0xff && ((head[1] ?? 0) & 0xe0) === 0xe0;
+      const tagged = String.fromCharCode(...head) === "ID3";
+      expect(framed || tagged).toBe(true);
+      expect(output.audioBytes).toBe(mp3.bytes.byteLength);
+      expect(output.audioBytes).toBeGreaterThan(0);
+      // The compression is the reason the pass exists: the WAV that went in is
+      // 2 bytes per sample at 24 kHz, and the MP3 is a fraction of it.
+      const wavBytes = 44 + (output.audioDurationMs / 1000) * 24_000 * 2;
+      expect(output.audioBytes).toBeLessThan(wavBytes / 4);
+      // The run says both numbers, which is what makes the trade visible.
+      expect(run.reported.at(-1)).toMatch(
+        /^Recorded a \d+s audit in \w+'s voice — .* of MP3, from .* of WAV\.$/,
+      );
+
+      // The model really read the transcript, and the audit really reached the
+      // output — including the empty-array case being an ANSWER.
+      expect(output.headline).toBe(REPLY.headline);
+      expect(output.risks).toEqual(REPLY.risks);
+      expect(output.spoken).toBe(REPLY.spoken);
+      // The RUN's own clock, from two journaled steps rather than a body-level
+      // `Date.now()` a replay would re-read.
+      expect(output.elapsedMs).toBeGreaterThanOrEqual(0);
+    });
+
+    test(
+      "really levels, cuts and audits a real call recording",
+      async ({ app }) => {
+        // LIVE ONLY, and the case that earns the template its name. Four things
+        // have to be real: ffmpeg has to decode an MP3 nobody here made, the
+        // pauses it finds have to cover the whole recording, every segment's
+        // request has to come back with the words that are in it, and the model
+        // has to audit what came back.
+        const response = await fetch(LIVE_RECORDING);
+        expect(response.ok).toBe(true);
+        const mp3 = new Uint8Array(await response.arrayBuffer());
+        const uploads = publish(mp3, "wildfires.mp3", "audio/mpeg");
+
+        const run = await app.run(audit, { recording: UPLOAD_ID });
+
+        expect(run.error).toBeUndefined();
+        expect(run.status).toBe("completed");
+        const output = run.output;
+        if (output === undefined) expect.fail("a completed run must carry an output");
+
+        // ffprobe read the container it really was, and the FILENAME a reader
+        // sees is the one they uploaded rather than the converted artifact's.
+        expect(output.codec).toBe("mp3");
+        expect(output.source).toBe("wildfires.mp3");
+        expect(output.durationMs).toBeGreaterThan(250_000);
+
+        // Four and a half minutes against a 110-second cap is at least three
+        // segments, and real speech has pauses — so the cuts should be in them.
+        expect(output.segments).toBeGreaterThanOrEqual(3);
+        expect(output.blindCuts).toBe(0);
+        // A real recording is mostly speech and not all of it.
+        expect(output.speechPercent).toBeGreaterThan(50);
+        expect(output.speechPercent).toBeLessThan(100);
+        expect(output.loudnessBefore).toBeLessThan(0);
+
+        // Every segment came back with words, which is how "the whole recording
+        // was covered" is checked rather than assumed: a plan that ran past the
+        // end would leave a silent tail segment and a short transcript.
+        expect(output.words).toBeGreaterThan(400);
+        expect(output.transcript).toMatch(/wildfire/i);
+        expect(output.transcript).toMatch(/canada/i);
+        expect(output.transcript).toMatch(/air quality/i);
+
+        // The audit is of THIS transcript, and the spoken half is a SCRIPT
+        // rather than a list — the template's central prompt decision,
+        // measured against a real model.
+        expect(output.headline.length).toBeGreaterThan(0);
+        expect(`${output.headline} ${output.spoken}`).toMatch(/smoke|wildfire|air|health/i);
+        expect(output.risks.length).toBeLessThanOrEqual(4);
+        expect(output.actions.length).toBeLessThanOrEqual(4);
+        expect(output.spoken).toMatch(/[.!?]/);
+        expect(output.spoken).not.toMatch(/^\s*[-*•]/m);
+
+        // And the mastering pass ran on what the voice produced.
+        expect(uploads.writes.map((one) => one.name)).toEqual(["wildfires.pcm", "audit.mp3"]);
+        expect(uploads.read(output.audio)?.type).toBe("audio/mpeg");
+        expect(output.audioBytes).toBeGreaterThan(0);
+      },
+      { live: true },
+    );
+  },
+  { speech: speakATone },
+);

package/dist/templates/code-interpreter/agent.eval.test.ts

@@ -0,0 +1,165 @@
+// An EVAL: does Coda actually answer by RUNNING CODE? Run it with `aai eval`.
+//
+// `agent.test.ts` asserts about the config; this drives the real thing — the
+// real session, the real tool executor, the real event stream, with only the
+// microphone and the speaker faked.
+//
+// Two things about this file are worth copying into any template eval.
+//
+// **The prompt is applied here, because `system-prompt.md` IS the prompt and
+// nothing in `agent.ts` imports it.** Discovery happens where the bundle is
+// assembled (`aai build` reads the file; a spec does the same lowering), so an
+// eval that drove `agent.ts` alone would measure an agent running on
+// `DEFAULT_SYSTEM_PROMPT` — i.e. not the agent anybody deploys, and every claim
+// this file makes about Coda's rules would be a claim about nothing.
+//
+// **`run_code` refuses unless the EVAL supplies an executor.** A deployed agent
+// runs it only inside the guest sandbox — the Modal container is the security
+// boundary — so off-platform the builtin declines rather than evaluating
+// model-written JavaScript in the host process. That is right, and it left this
+// template's whole subject assertable as a CALL and never as an answer: a
+// `toBeDefined()` on the result is satisfied by the refusal itself. So the suite
+// passes `runCode` (below) — a developer's own machine may, a deployment may
+// not — and every case here asserts BOTH halves: that Coda reached for code, and
+// what the code came back with.
+import { runInNewContext } from "node:vm";
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { errorMessage } from "@alexkroman1/aai/utils";
+import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import authored from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+const agentDef = withSystemPrompt(authored, systemPrompt);
+
+/** The code every `run_code` call in this turn carried, joined. */
+const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => String(c.args.code ?? ""))
+    .join("\n");
+
+/**
+ * A `run_code` executor, so these cases can assert the ANSWER.
+ *
+ * The builtin refuses without one — the Modal container is the security
+ * boundary, and off-platform there is none — so a case could assert the CALL and
+ * the code it carried, and never what the code came back with. A `node:vm`
+ * context with a capturing `console.log` is what a developer would reach for on
+ * their own machine, and it is enough here: what runs is arithmetic, not a
+ * program. It is NOT a sandbox and does not pretend to be one; a deployed agent
+ * still gets the refusal.
+ *
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
+ * node-reaching module graph into this project's TypeScript program, which is
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
+ * re-exported from `/eval` for exactly that reason.
+ */
+const runCode: RunCodeExecutor = async (code) => {
+  const lines: string[] = [];
+  const log = (...args: unknown[]): void => {
+    lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
+  };
+  try {
+    runInNewContext(code, { console: { log } }, { timeout: 1000 });
+  } catch (err) {
+    return { error: errorMessage(err) };
+  }
+  return lines.join("\n");
+};
+
+/** What every `run_code` call in this turn PRINTED, joined. */
+const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => c.result ?? "")
+    .join("\n");
+
+describeEval(
+  agentDef,
+  (test) => {
+    test(
+      "writes code for arithmetic instead of answering from its head",
+      async ({ session }) => {
+        const turn = await session.say("What is 127 times 849?");
+
+        // The template's CRITICAL RULE, and the whole reason it declares
+        // run_code: a model that answers this one directly has regressed, and it
+        // is the easiest question in the file to answer wrongly with confidence.
+        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        const code = codeIn(turn);
+        expect(code).toContain("127");
+        expect(code).toContain("849");
+
+        // And the code RAN, and got it right. That half needed an executor: with
+        // none, `run_code` answers "only available in the sandboxed runtime", so
+        // every claim above passes for an agent that then does the sum in its
+        // head — which is the exact regression the CRITICAL RULE exists to stop.
+        const output = outputIn(turn);
+        expect(output, `run_code printed: ${output}`).toContain("107823");
+
+        // "Report RESULTS, never intentions": the call goes out before Coda says
+        // anything, rather than after a sentence announcing it.
+        const firstTool = turn.events.findIndex((e) => e.type === "tool.called");
+        const firstSaid = turn.events.findIndex((e) => e.type === "agent-transcript.committed");
+        expect(firstSaid).toBeGreaterThan(-1);
+        expect(firstTool).toBeGreaterThan(-1);
+        expect(firstTool).toBeLessThan(firstSaid);
+      },
+      { live: true },
+    );
+
+    test(
+      "reaches for code for a calendar question too, not just sums",
+      async ({ session }) => {
+        const turn = await session.say("What day of the week was January 1st, 2000?");
+
+        // The prompt lists this exact question under "you MUST use code for".
+        // It is the case a narrower reading of the rule ("code is for maths")
+        // silently drops.
+        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        expect(codeIn(turn)).toMatch(/Date|2000/);
+        // The code RAN rather than being refused — but the ANSWER is
+        // deliberately not asserted here, and the reason is worth knowing before
+        // adding it back. Coda writes `new Date("2000-01-01").getDay()`, which
+        // parses as UTC midnight and is then read in LOCAL time: correct in the
+        // guest sandbox (UTC, Saturday) and one day out on any developer west of
+        // Greenwich (measured: this printed "Friday"). Asserting the weekday
+        // would measure the machine running the eval. The two arithmetic cases
+        // in this suite have no such dependency and do assert their answers.
+        const output = outputIn(turn);
+        expect(output, `run_code printed: ${output}`).not.toMatch(
+          /only available in the sandboxed runtime/,
+        );
+        expect(output).not.toBe("");
+      },
+      { live: true },
+    );
+
+    test(
+      "the run_code builtin is wired to the agent's tool executor",
+      async ({ session }) => {
+        const turn = await session.say("Add one and one for me.");
+
+        // The wiring claim, and it really discriminates: a tool the agent does
+        // NOT declare produces a `tool.called` with no result at all, so the
+        // paired result is what says `builtinTools: ["run_code"]` still resolves
+        // through to an executable tool.
+        //
+        // `toBeDefined()` used to be the whole assertion, and it was satisfied by
+        // the REFUSAL — "run_code is only available in the sandboxed runtime" is a
+        // defined result. Asserting the printed answer is what makes this a claim
+        // about the executor rather than about the string it declined with.
+        const [call] = turn.toolCalls;
+        expect(call?.name).toBe("run_code");
+        expect(call?.result).toBe("2");
+        expect(turn.completed).toBe(true);
+      },
+      { stubReply: [{ tool: "run_code", args: { code: "console.log(1 + 1)" } }, "That's two."] },
+    );
+  },
+  // `runCode` is what makes these cases about the ANSWER and not just the call.
+  { runCode },
+);