@alexkroman1/aai-cli 6.11.0 → 8.0.0

package/dist/templates/transcription-workflow/workflows/wav.ts

@@ -312,34 +312,3 @@ export function planSegments(format: WavFormat): Segment[] {
   }
   return segments;
 }
-
-/**
- * Put a canonical 44-byte WAV header in front of a slice of samples.
- *
- * A byte range of a WAV is raw PCM with no header, and the sync endpoint reads
- * the rate and channel count out of one. Sending `audio/pcm` with a `config`
- * part instead would work equally well; a header keeps the two parts of the
- * request from being able to disagree.
- */
-export function wavWithHeader(format: WavFormat, samples: Uint8Array): Uint8Array<ArrayBuffer> {
-  const out = new Uint8Array(44 + samples.length);
-  const view = new DataView(out.buffer);
-  const write = (at: number, text: string) => {
-    for (let i = 0; i < text.length; i++) view.setUint8(at + i, text.charCodeAt(i));
-  };
-
-  write(0, "RIFF");
-  view.setUint32(4, 36 + samples.length, true);
-  write(8, "WAVEfmt ");
-  view.setUint32(16, 16, true); // `fmt ` payload length
-  view.setUint16(20, 1, true); // linear PCM
-  view.setUint16(22, format.channels, true);
-  view.setUint32(24, format.sampleRate, true);
-  view.setUint32(28, bytesPerSecond(format), true);
-  view.setUint16(32, blockAlign(format), true);
-  view.setUint16(34, format.bitsPerSample, true);
-  write(36, "data");
-  view.setUint32(40, samples.length, true);
-  out.set(samples, 44);
-  return out;
-}

package/dist/templates/travel-concierge/agent.eval.test.ts

@@ -0,0 +1,323 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the confirmation gate actually gate? Run it with `aai eval`.
+//
+// `agent.test.ts` drives each tool directly and asserts about the state it
+// wrote. That is the right tier for "does `stageAction` refuse a second
+// staging" — and it cannot answer the question this template exists to
+// demonstrate, which is whether the gate still holds when a MODEL is the one
+// picking tools. So these cases drive a real session and read the mechanism off
+// the event stream: which tool ran, what the gated tool answered, and — the
+// claim that matters — whether the caller's booking moved before they said yes.
+//
+// Two modes, per `describeEval`: with a key, a live model chooses the tools;
+// without one, each case's `stubReply` scripts them. A scripted tool call really
+// executes, so the flow gate really runs either way — what a stub run cannot
+// tell you is whether the model would have chosen that tool.
+//
+// **`system-prompt.md` is applied HERE, not by `agent.ts`.** The build discovers
+// the file, so an eval driving the raw default export would run this agent with
+// the FRAMEWORK DEFAULT prompt — and the discipline that prompt imposes is the
+// entire subject of this file. Measured against the default: the model answers
+// product questions from its own knowledge and skips the tools the prompt exists
+// to route it through, so a case run that way measures nothing it claims to.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper for the
+ * reason `agent.test.ts` gives — this file ships, so it may not import anything
+ * outside its own template. An eval that forgot it would run an agent with NO
+ * tools and read as a model that refuses to act.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `tripView` may grow without
+ * touching this.
+ */
+const ProjectedTrip = z.object({
+  assistant: z.string(),
+  ticket: z.object({ flightId: z.string() }).nullable(),
+  bookings: z.array(z.unknown()),
+  pending: z.string().nullable(),
+});
+
+/**
+ * The `syncState` frames, in stream order, UP TO the first `confirm_action`.
+ *
+ * This is what makes "a staging tool mutates nothing" assertable against a live
+ * model: the caller's later yes is allowed to move the ticket, so a claim about
+ * the FINAL state is a claim about the conversation rather than about the gate.
+ * What the gate promises is that every frame before the confirmation shows the
+ * booking untouched — and `tripProjection` rides out on `state.updated` after
+ * every tool call, so the stream carries one per step.
+ */
+function framesBeforeConfirm(session: EvalSession): z.infer<typeof ProjectedTrip>[] {
+  const views: z.infer<typeof ProjectedTrip>[] = [];
+  for (const event of session.events()) {
+    if (event.type === "tool.called" && event.toolName === "confirm_action") break;
+    if (event.type === "state.updated") views.push(ProjectedTrip.parse(event.state));
+  }
+  return views;
+}
+
+/**
+ * The latest frame — what the browser would be rendering now. `lastStateIn` is
+ * the SDK's reader for exactly this; the schema is why it is worth passing one.
+ */
+const tripState = (session: EvalSession) => lastStateIn(session.events(), ProjectedTrip);
+
+/**
+ * Drive a whole call, one caller line at a time, and hand back every turn.
+ *
+ * The cases below assert about the turn a MECHANISM fired in rather than about
+ * turn one, because how many turns a desk takes to get there is the model's
+ * business and it moved when the desk gate landed: the flight desk's brief says
+ * to search before quoting anything, so measured live this concierge now spends
+ * its first turn on `to_flight_assistant` and `search_flights` and reads the
+ * fare back before it stages. A case pinned to turn one is a flake with a
+ * misleading name — the same argument `retail`'s eval carries.
+ */
+async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
+  const turns: EvalTurn[] = [];
+  for (const line of lines) turns.push(await session.say(line));
+  return turns;
+}
+
+/** Every tool call of the call so far, flattened, in order. */
+const callsIn = (turns: readonly EvalTurn[]) => turns.flatMap((turn) => turn.toolCalls);
+
+/** The turn a named tool STAGED something in — the call that answered with the
+ *  read-back rather than with a gate's refusal. */
+const stagingTurn = (turns: readonly EvalTurn[], tool: string) =>
+  turns.find((turn) =>
+    turn.toolCalls.some(
+      (call) => call.name === tool && /awaitingConfirmation/.test(call.result ?? ""),
+    ),
+  );
+
+describeEval(agentDef, (test) => {
+  test(
+    "a sensitive tool stages the change and moves nothing",
+    async ({ session }) => {
+      // Three lines, each of them the same REQUEST and none of them an answer to
+      // a read-back: what is asserted below is that the turn which staged did
+      // not also apply, so a line the model could read as consent ("correct",
+      // "that's right") would be measuring the caller instead of the desk.
+      const turns = await sayAll(session, [
+        "Move my ticket to flight LX52, the Wednesday one.",
+        "I want the Wednesday LX52 instead of the flight I'm on now.",
+        "Put me on LX52 on Wednesday, please.",
+      ]);
+
+      const staging = stagingTurn(turns, "update_ticket");
+      const attempts = callsIn(turns).filter((call) => call.name === "update_ticket");
+      // Named with the whole call, tools AND text: "expected undefined to be
+      // defined" says nothing about a desk that talked its way through three
+      // turns without staging, which is exactly the failure this case caught
+      // while the flight desk's brief had the read-back before the staging.
+      expect(
+        staging,
+        turns
+          .map(
+            (turn, i) =>
+              `turn ${i + 1}: [${turn.toolCalls.map((c) => c.name).join(", ")}] said: ${turn.text}`,
+          )
+          .join("\n"),
+      ).toBeDefined();
+      const staged = staging?.toolCalls.find((call) => call.name === "update_ticket");
+      // The tool answered with the read-back rather than with a receipt.
+      expect(staged?.result).toMatch(/awaitingConfirmation/);
+      // Any attempt that did NOT stage is the DESK GATE refusing:
+      // `update_ticket` belongs to the flight desk, so a model reaching for it
+      // before `to_flight_assistant` is told so and recovers inside the turn.
+      // That refusal is a legal step and it is asserted rather than tolerated —
+      // its own message is what pointed the model at the way in.
+      for (const attempt of attempts.filter((call) => call !== staged)) {
+        expect(attempt.result).toMatch(/to_flight_assistant|Not available yet/);
+      }
+      // The desk asks; it does not decide. A `confirm_action` AFTER the staging
+      // in the same turn is the agent confirming on its own initiative, which
+      // the system prompt forbids in as many words.
+      //
+      // After, not anywhere in the turn: measured live, a model that hears a
+      // second request as a yes reaches for `confirm_action` BEFORE it has
+      // staged anything, the gate refuses it (the case above is where that
+      // refusal is the subject), and it then stages properly. That is a wasted
+      // step rather than an unasked-for change, and folding the two together
+      // would fail this case for the behaviour the next one proves is safe.
+      // Narrowed first: `indexOf` takes a value, and `staged` is optional — the
+      // rewrite Biome offers for `findIndex` over an identity is UNSAFE for
+      // exactly that reason, and the assertion above is what makes an absent
+      // staging call a failure rather than a slice from 0.
+      const stagedAt = staged === undefined ? -1 : (staging?.toolCalls.indexOf(staged) ?? -1);
+      expect(staging?.toolCalls.slice(stagedAt + 1).map((call) => call.name) ?? []).not.toContain(
+        "confirm_action",
+      );
+
+      const views = framesBeforeConfirm(session);
+      const waiting = views.filter((view) => view.pending !== null);
+      expect(waiting.length, "no frame ever showed a staged change").toBeGreaterThan(0);
+      expect(waiting.at(-1)?.pending).toMatch(/LX52/);
+      // THE claim: through every frame up to the confirmation, the ticket is
+      // still the one the caller phoned in on. A staging tool that mutated
+      // would satisfy every assertion above this one.
+      for (const view of views) expect(view.ticket?.flightId).toBe("LX40");
+    },
+    {
+      stubReply: [
+        { tool: "to_flight_assistant", args: { request: "move my ticket to LX52" } },
+        { tool: "update_ticket", args: { flightId: "LX52" } },
+        "That would move you to LX52 on Wednesday, five ninety. Shall I go ahead?",
+      ],
+    },
+  );
+
+  test(
+    "confirm_action is refused while nothing is waiting",
+    async ({ session, mode }) => {
+      const turn = await session.say("Yes, I confirm — go ahead and do it.");
+
+      const attempts = turn.toolCalls.filter((call) => call.name === "confirm_action");
+      // In stub mode the script FORCES the call, so the gate is really
+      // exercised; a live model that declines to call it at all has honoured
+      // the same rule one level earlier, which is why the count is asserted
+      // only where it is determined.
+      if (mode === "stub") expect(attempts).toHaveLength(1);
+      for (const attempt of attempts) {
+        // The refusal names the position and quotes the state's instruction —
+        // that is what the model recovers from, and a gate that ran the body
+        // and then apologized would not carry it.
+        expect(attempt.result).toMatch(/Not available yet/);
+        expect(attempt.result).toMatch(/browsing/);
+      }
+      expect(tripState(session)?.pending ?? null).toBeNull();
+      expect(tripState(session)?.bookings ?? []).toHaveLength(0);
+    },
+    { stubReply: [{ tool: "confirm_action" }, "Nothing is waiting for your yes just now."] },
+  );
+
+  test(
+    "the caller's yes is the only thing that moves the ticket",
+    async ({ session }) => {
+      // Four lines, three of them a yes: which turn the desk stages in and which
+      // it applies in is its own business — the flight desk's brief has it
+      // search first — and saying yes repeatedly is what makes "once each"
+      // below a claim about the MECHANISM rather than about the model's pacing.
+      await sayAll(session, [
+        "Move my ticket to flight LX52, the Wednesday one.",
+        "Correct — LX52 on Wednesday. Please move my ticket to it.",
+        "Yes, that's right — go ahead and change it.",
+        "Yes. Confirm it, please.",
+      ]);
+
+      // The two gated tools' calls, in order.
+      const gated = session
+        .toolCalls()
+        .filter((call) => ["update_ticket", "confirm_action"].includes(call.name));
+      // "Did something" is read POSITIVELY — a staging answers
+      // `awaitingConfirmation`, an apply answers `applied` — so the complement
+      // is a claim rather than a definition.
+      const effective = gated.filter((call) =>
+        /awaitingConfirmation|"applied"/.test(call.result ?? ""),
+      );
+      // Staged first, applied second, once each. Reversed — or a confirm with no
+      // stage — is the regression this template's whole shape exists to prevent.
+      expect(effective.map((call) => call.name)).toEqual(["update_ticket", "confirm_action"]);
+      // Everything else has to be a GATE refusing, and nothing else: the desk
+      // gate turns away an `update_ticket` issued before
+      // `to_flight_assistant`, and the confirmation gate turns away a
+      // `confirm_action` issued before anything is staged (the case above is
+      // where that is the subject). Both are steps a live model really takes,
+      // neither moved the ticket, and neither is what this case is about.
+      for (const call of gated.filter((c) => !effective.includes(c))) {
+        expect(call.result, `${call.name} neither acted nor was refused`).toMatch(
+          /Not available yet|belongs to the/,
+        );
+      }
+
+      const applied = effective.at(-1);
+      expect(applied?.result).toMatch(/LX52/);
+
+      const view = tripState(session);
+      expect(view?.ticket?.flightId).toBe("LX52");
+      expect(view?.pending ?? null).toBeNull();
+    },
+    {
+      stubReply: [
+        // The delegation is in the script because the DESK GATE is real: the
+        // flight desk's tools refuse until `to_flight_assistant` has run, so a
+        // script that skipped it would exercise that refusal rather than this
+        // case's subject.
+        { tool: "to_flight_assistant", args: { request: "move my ticket to LX52" } },
+        { tool: "update_ticket", args: { flightId: "LX52" } },
+        "That's LX52 on Wednesday at five ninety. Confirm it?",
+        { tool: "confirm_action" },
+        "Done — you're on LX52 on Wednesday.",
+      ],
+    },
+  );
+
+  test(
+    "a hotel request goes to the hotel desk rather than being answered at the front",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Leave the flight for now — I need a hotel in Boston for three nights, near the water.",
+      );
+
+      // The INDEX as well as the call, because what came before it is asserted
+      // below and `indexOf` on a possibly-undefined find is worse than both.
+      const handoffAt = turn.toolCalls.findIndex((call) => call.name === "to_hotel_assistant");
+      const handoff = turn.toolCalls[handoffAt];
+      expect(
+        handoff,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // The brief IS the tool result, which is the whole port of their
+      // per-assistant prompt onto a session whose prompt is fixed at connect.
+      expect(handoff?.result).toMatch(/hotel desk/);
+      expect(handoff?.args.request).toBeTruthy();
+      // And the stack moved, which is what the caller's sidebar renders and what
+      // `complete_or_escalate` will pop.
+      expect(tripState(session)?.assistant).toBe("hotel");
+      // Anything the model reached for BEFORE delegating was refused, and so
+      // told nothing about Boston hotels: the desk gate is what makes the
+      // handoff above unavoidable rather than requested. This is the assertion
+      // that would have caught the old behaviour even if the model had happened
+      // to delegate afterwards.
+      const before = turn.toolCalls.slice(0, handoffAt);
+      for (const call of before.filter((c) => c.name.startsWith("search_"))) {
+        expect(call.result).toMatch(/belongs to the/);
+      }
+    },
+    // Live only: which desk the model hands the call to is exactly the judgement
+    // a script would be making on its behalf.
+    //
+    // This case is what turned the narrowing into a MECHANISM. It was written
+    // when the delegation was asked for in the prompt and nothing enforced it —
+    // one session has one tool list — and it measured the asking losing 0 of 5:
+    // the model called `search_hotels` from the concierge desk every run. Every
+    // desk tool now checks the stack (`requireDesk` in `shared.ts`), so what
+    // this measures is that the refusal really does route the model through the
+    // desk, inside one turn.
+    { live: true },
+  );
+});