@alexkroman1/aai-cli 13.1.0 → 13.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/dist/scaffold/package.json +4 -4
  2. package/dist/templates/call-audit/agent.eval.test.ts +14 -11
  3. package/dist/templates/call-audit/agent.test.ts +20 -4
  4. package/dist/templates/call-audit/workflows/ingest.ts +10 -1
  5. package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
  6. package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
  7. package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
  8. package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
  9. package/dist/templates/link-digest/agent.eval.test.ts +24 -15
  10. package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
  11. package/dist/templates/night-owl/agent.eval.test.ts +30 -15
  12. package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
  13. package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
  14. package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
  15. package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
  16. package/dist/templates/redline/agent.eval.test.ts +32 -24
  17. package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
  18. package/dist/templates/retail/agent.eval.test.ts +18 -34
  19. package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
  20. package/dist/templates/spoken-summary/agent.test.ts +9 -4
  21. package/dist/templates/support-line/agent.eval.test.ts +23 -26
  22. package/dist/templates/transcription-workflow/agent.test.ts +10 -0
  23. package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
  24. package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
  25. package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
  26. package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
  27. package/package.json +4 -4
@@ -48,6 +48,7 @@
48
48
  // exercised. `run.slept` below is the other half of that admission written as
49
49
  // an assertion. `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that
50
50
  // really suspends and resumes a run.
51
+ import { stubGatewayRoute } from "@alexkroman1/aai/testing";
51
52
  import {
52
53
  installStubSpeech,
53
54
  installStubTranscribe,
@@ -115,23 +116,30 @@ function publish(bytes: Uint8Array, name: string, type: string) {
115
116
  *
116
117
  * ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
117
118
  * AND calls a model cannot install two, which is exactly what `otherwise` is
118
- * for. The transcription half is the SDK's own fake rather than this file's
119
- * hand-typed wire: it routes off the SDK's endpoint constants, so a case cannot
120
- * pass because the fake and the step agree on a typo.
119
+ * for. BOTH halves are the SDK's own fakes rather than this file's hand-typed
120
+ * wire, and it is the same argument twice: each routes off the SDK's own
121
+ * endpoint constant, so a case cannot pass because the fake and the step agree
122
+ * on a typo. The predicate here used to be `url.includes("llm-gateway")` — a
123
+ * HOST, which the default gateway happens to carry and a `gatewayUrl` pointed
124
+ * anywhere else does not, so the fake would have gone on answering the
125
+ * transcription 404 to a model call it no longer recognised.
126
+ *
127
+ * The reader also hands back DECODED calls, which is why the case below asks
128
+ * `model.calls[0].prompt` what the model was SHOWN: off a raw request body that
129
+ * is the whole serialized request, `model` and `reasoning_effort` included.
121
130
  */
122
131
  function scriptProvider(options: { text?: string; pendingPolls?: number } = {}) {
123
- return installStubTranscribe({
132
+ const model = stubGatewayRoute(JSON.stringify(REPLY));
133
+ const provider = installStubTranscribe({
124
134
  text: options.text ?? TRANSCRIPT,
125
135
  durationSec: 42,
126
136
  // Passed straight through rather than conditionally spread: the option
127
137
  // already admits `undefined`, and `guard-invariants` rule 2 counts the
128
138
  // spread.
129
139
  pendingPolls: options.pendingPolls,
130
- otherwise: (request) =>
131
- request.url.includes("llm-gateway")
132
- ? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
133
- : undefined,
140
+ otherwise: (request) => model.route(request),
134
141
  });
142
+ return { provider, model };
135
143
  }
136
144
 
137
145
  describeWorkflowEval(
@@ -144,7 +152,7 @@ describeWorkflowEval(
144
152
  // returned bytes, or an id nothing wrote, or two ids because the synthesis
145
153
  // and the store became two steps.
146
154
  const uploads = publish(new Uint8Array(64), "standup.wav", "audio/wav");
147
- const provider = scriptProvider();
155
+ const { provider } = scriptProvider();
148
156
  const speech = installStubSpeech({ pcmBytes: 96_000 });
149
157
 
150
158
  const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
@@ -207,7 +215,7 @@ describeWorkflowEval(
207
215
  // central prompt decision regressing — synthesize the bullet list and you
208
216
  // get a voice reading "one. two. three." with no connective tissue.
209
217
  publish(new Uint8Array(64), "standup.wav", "audio/wav");
210
- const provider = scriptProvider();
218
+ const { model } = scriptProvider();
211
219
  const speech = installStubSpeech();
212
220
 
213
221
  const run = await app.run(spokenSummary, { recording: UPLOAD_ID, voice: "michael" });
@@ -226,9 +234,10 @@ describeWorkflowEval(
226
234
 
227
235
  // And the model was ASKED for both, over the transcript it was given. A
228
236
  // prompt that stopped asking for a script is how the field goes missing.
229
- const prompt = String(provider.calls.find((call) => call.leg === "other")?.body ?? "");
230
- expect(prompt).toContain("READ ALOUD");
231
- expect(prompt).toContain("The launch is on for Tuesday the fourth");
237
+ const asked = model.calls[0];
238
+ if (asked === undefined) expect.fail("the run must have asked the model for a summary");
239
+ expect(asked.prompt).toContain("READ ALOUD");
240
+ expect(asked.prompt).toContain("The launch is on for Tuesday the fourth");
232
241
  });
233
242
 
234
243
  test("a recording with no speech stops before the model and the voice", async ({ app }) => {
@@ -237,7 +246,7 @@ describeWorkflowEval(
237
246
  // the run would go on to summarize no words and store half a second of
238
247
  // audio — a green run with an empty product.
239
248
  const uploads = publish(new Uint8Array(64), "silence.wav", "audio/wav");
240
- const provider = scriptProvider({ text: " " });
249
+ const { model } = scriptProvider({ text: " " });
241
250
  const speech = installStubSpeech();
242
251
 
243
252
  const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
@@ -247,7 +256,7 @@ describeWorkflowEval(
247
256
  expect(run.output).toBeUndefined();
248
257
  // Nothing was summarized and nothing was spoken, which is the half that
249
258
  // makes this more than an error-message assertion.
250
- expect(provider.calls.filter((call) => call.leg === "other")).toEqual([]);
259
+ expect(model.calls).toEqual([]);
251
260
  expect(speech.calls).toEqual([]);
252
261
  expect(uploads.writes).toEqual([]);
253
262
  expect(run.reported).not.toContain("Summarizing the transcript.");
@@ -262,7 +271,7 @@ describeWorkflowEval(
262
271
  // them. A loop that re-submitted, or one that spun with no wait, both
263
272
  // produce a correct transcript and a wrong bill.
264
273
  publish(new Uint8Array(64), "standup.wav", "audio/wav");
265
- const provider = scriptProvider({ pendingPolls: 2 });
274
+ const { provider } = scriptProvider({ pendingPolls: 2 });
266
275
  installStubSpeech();
267
276
 
268
277
  const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
@@ -257,15 +257,20 @@ describe("the whole run", () => {
257
257
  * spec cannot pass because the fake and the step agree on a typo.
258
258
  */
259
259
  function stubProvider(reply: { headline: string; points: string[]; spoken: string }) {
260
+ // The model leg goes through `stubGatewayRoute` for the same reason, which
261
+ // this helper used to claim and not do: it hand-typed the completion
262
+ // envelope and recognised a model call by `llm-gateway`, the HOST. That is a
263
+ // property of one deployment rather than of the request — `stepGenerate`
264
+ // dials `${gatewayUrl ?? ASSEMBLYAI_LLM_GATEWAY_URL}/chat/completions`, so a
265
+ // caller pointing `gatewayUrl` at an OpenAI-compatible proxy of their own
266
+ // stops matching and the transcription fake answers a 404 to a model call.
267
+ const model = stubGatewayRoute(JSON.stringify(reply));
260
268
  return installStubTranscribe({
261
269
  audioUrl: "https://cdn/aai/1",
262
270
  jobIdPrefix: "t_",
263
271
  text: "we ship tuesday and two bugs are left",
264
272
  durationSec: 42,
265
- otherwise: (request) =>
266
- request.url.includes("llm-gateway")
267
- ? { body: { choices: [{ message: { content: JSON.stringify(reply) } }] } }
268
- : undefined,
273
+ otherwise: (request) => model.route(request),
269
274
  });
270
275
  }
271
276
 
@@ -27,7 +27,13 @@
27
27
 
28
28
  /** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
29
29
  import agentDef from "virtual:aai/agent";
30
- import { type EvalSession, toolResultIn } from "@alexkroman1/aai-runtime/eval";
30
+ import {
31
+ describeToolCalls,
32
+ describeTurn,
33
+ type EvalSession,
34
+ statesIn,
35
+ toolResultIn,
36
+ } from "@alexkroman1/aai-runtime/eval";
31
37
  import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
32
38
  import { expect } from "vitest";
33
39
  import { z } from "zod";
@@ -71,13 +77,14 @@ const Lookup = z.object({
71
77
  error: z.string().optional(),
72
78
  });
73
79
 
74
- /** Every `syncState` frame, in stream order. */
80
+ /**
81
+ * Every `syncState` frame, in stream order.
82
+ *
83
+ * `statesIn` reads an EVENT LIST rather than a session, which is what lets a
84
+ * case slice the stream first; the schema above is what it takes one for.
85
+ */
75
86
  function frames(session: EvalSession) {
76
- return session
77
- .events()
78
- .flatMap((event) =>
79
- event.type === "state.updated" ? [ProjectedSupport.parse(event.state)] : [],
80
- );
87
+ return statesIn(session.events(), ProjectedSupport);
81
88
  }
82
89
 
83
90
  /**
@@ -109,10 +116,10 @@ describeEval(agentDef, (test) => {
109
116
  // The reply rides in the message, because the failure that matters here is
110
117
  // a turn that SPOKE without looking anything up — "let me check that for
111
118
  // you" and then nothing, or worse, a notice period from memory.
112
- expect(
113
- asked.length,
114
- `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
115
- ).toBe(1);
119
+ // `describeTurn` is that sentence, and it says the two things a
120
+ // hand-built one left out: "called no tools" rather than an empty
121
+ // bracket, and whether the reply was cancelled.
122
+ expect(asked.length, describeTurn(turn)).toBe(1);
116
123
 
117
124
  for (const payload of lookups(session)) {
118
125
  // Three legal outcomes, and the invariant that spans them: an answer the
@@ -142,13 +149,9 @@ describeEval(agentDef, (test) => {
142
149
  await session.say("Yes please, log that one — my callback number is 07700 900123.");
143
150
 
144
151
  const logged = session.toolCalls().find((call) => call.name === "log_ticket");
145
- expect(
146
- logged,
147
- `tools called: ${session
148
- .toolCalls()
149
- .map((c) => c.name)
150
- .join(", ")}`,
151
- ).toBeDefined();
152
+ // The claim spans both turns, so the message does too: `describeToolCalls`
153
+ // over the session's own list, where `describeTurn` would describe one.
154
+ expect(logged, describeToolCalls(session.toolCalls())).toBeDefined();
152
155
  expect(logged?.result).toMatch(/TCK\d{4}/);
153
156
 
154
157
  const latest = frames(session).at(-1);
@@ -189,10 +192,7 @@ describeEval(agentDef, (test) => {
189
192
  const turn = await session.say("How much notice do I have to give to cancel my contract?");
190
193
 
191
194
  const [payload] = lookups(session);
192
- expect(
193
- payload,
194
- `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
195
- ).toBeDefined();
195
+ expect(payload, describeTurn(turn)).toBeDefined();
196
196
  // The whole verdict in the message: `grounded: undefined` on its own does
197
197
  // not say whether the lookup failed, or ran and refused.
198
198
  const verdict = JSON.stringify(payload);
@@ -238,10 +238,7 @@ describeEval(agentDef, (test) => {
238
238
  );
239
239
 
240
240
  const [payload] = lookups(session);
241
- expect(
242
- payload,
243
- `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
244
- ).toBeDefined();
241
+ expect(payload, describeTurn(turn)).toBeDefined();
245
242
  // Withheld, not softened: `answer: null` is the tool refusing to hand the
246
243
  // model something to read out, and the guidance is the exit the grading
247
244
  // apparatus needs — a support line that can only answer will answer wrong.
@@ -779,6 +779,16 @@ describe("transcribeSegment", () => {
779
779
  expect(decoded).toContain('name="audio"; filename="segment-0.wav"');
780
780
  // The WAV really rides in the part, header and all.
781
781
  expect(decoded).toContain("RIFF");
782
+ // And the header is CONTIGUOUS with its samples, which is what the two-chunk
783
+ // form (`[wavHeader(…), window]`) has to preserve and the only thing it could
784
+ // plausibly lose: the part's payload is exactly the 44 bytes plus the window,
785
+ // with nothing between them and nothing appended. A body that grew or shrank
786
+ // here is a file the endpoint decodes into confident nonsense rather than
787
+ // refusing.
788
+ const latin = new TextDecoder("latin1").decode(sent);
789
+ const from = latin.indexOf("RIFF");
790
+ const to = latin.lastIndexOf("\r\n--");
791
+ expect(to - from).toBe(44 + (SEGMENT.end - SEGMENT.start));
782
792
  });
783
793
 
784
794
  test("sends the DOWNSAMPLED window when the recording is heavier than 16 kHz mono", async () => {
@@ -159,7 +159,16 @@ export async function normalizeRecording(uploadId: string): Promise<NormalizedRe
159
159
  const source = join(dir, "source");
160
160
  const converted = join(dir, "converted.wav");
161
161
 
162
- await readUploadToFile(uploadId, source, { size: stored.size });
162
+ // NO `size`, though `stored.size` is right there — and that is the whole
163
+ // difference between this copy being one window at a time and being
164
+ // `STEP_FILE_READ_CONCURRENCY` of them. Passing `size` means "I am judging
165
+ // completeness myself", which is what a body polling a still-arriving
166
+ // upload needs and is the opposite of what happened above: this step has
167
+ // already called `requireCompleteUpload`, so the file IS whole and the
168
+ // windows may land in any order. Omitting it lets `readUploadToFile`
169
+ // establish that for itself and fan out. The cost is one metadata round
170
+ // trip, against the dozens of window reads it overlaps.
171
+ await readUploadToFile(uploadId, source);
163
172
 
164
173
  // What it WAS, for the progress line. Worth one ffprobe: "converted 41
165
174
  // minutes of aac" is a line that explains the run's shape, where
@@ -52,7 +52,10 @@ export function elapsed(ms: number): string {
52
52
  * `bytes` must be a whole file, header included — the endpoint decodes each
53
53
  * request independently, so a headerless tail is bytes it will refuse. Both
54
54
  * callers arrive at that differently: one re-attaches a header to a window it
55
- * read, the other is handed parts that already carry one.
55
+ * read, the other is handed parts that already carry one. A LIST is a whole
56
+ * file too: the segment caller passes `[wavHeader(...), window]` so the two are
57
+ * concatenated straight into the request body rather than into an intermediate
58
+ * buffer that doubles the segment's footprint.
56
59
  *
57
60
  * `stepTranscribeSyncClassified` — the SDK's own `stepTranscribeSync` plus
58
61
  * `throwStepError`, and nothing else — is the whole of what this adds to the SDK
@@ -68,7 +71,7 @@ export function elapsed(ms: number): string {
68
71
  * log has in front of them.
69
72
  */
70
73
  export async function transcribeWav(
71
- bytes: Uint8Array,
74
+ bytes: Uint8Array | readonly Uint8Array[],
72
75
  filename: string,
73
76
  label: string,
74
77
  ): Promise<string> {
@@ -66,12 +66,12 @@
66
66
  import type { WorkflowCtx } from "@alexkroman1/aai";
67
67
  import {
68
68
  emit,
69
- encodeWav,
70
69
  mapConcurrent,
71
70
  readUpload,
72
71
  report,
73
72
  requireCompleteUpload,
74
73
  uploadInfo,
74
+ wavHeader,
75
75
  } from "@alexkroman1/aai/step";
76
76
  import { throwFatalStepError } from "@alexkroman1/aai/step-errors";
77
77
  import { countWords, formatDuration, plural } from "@alexkroman1/aai/utils";
@@ -358,13 +358,20 @@ export async function transcribeSegment(
358
358
  // answers better — and getting it wrong is a whole transcript in the wrong
359
359
  // language. Add one back only for a desk that really knows.
360
360
  //
361
- // `encodeWav` is what makes a WINDOW decodable: the endpoint decodes each
361
+ // A HEADER is what makes a WINDOW decodable: the endpoint decodes each
362
362
  // request independently, so a slice of the middle of a recording is a headerless
363
363
  // tail until one is put back on it. The streaming flow needs no equivalent — its
364
364
  // parts were cut with a header each. The header is the SDK's rather than this
365
365
  // template's: a `WavFormat` is structurally a `PcmFormat`, and 22 lines of
366
366
  // `DataView` writes with a comment about which of the two declared lengths a
367
367
  // decoder trusts is not a thing worth a second copy of.
368
+ // It goes down as its own CHUNK rather than through `encodeWav`, which is a
369
+ // MEMORY decision and not a speed one: `encodeWav` allocates `44 + N` and
370
+ // copies the segment into it, and `multipartBody` then allocates the body and
371
+ // copies that again — so the audio was resident three times at the moment the
372
+ // request went out, on a fan-out whose width is set by exactly that peak (see
373
+ // `MAX_SEGMENT_CONCURRENCY`). Header and samples are contiguous on the wire
374
+ // either way; this holds `44 + N` once, ~3 MB per in-flight 16 kHz segment.
368
375
  // Down to 16 kHz mono BEFORE the header goes on, because the endpoint's budget
369
376
  // is 30 seconds of wall clock and that covers the upload. At 48 kHz stereo this
370
377
  // window is 17.66 MB and the same audio is 2.94 MB normalized — six times the
@@ -382,12 +389,12 @@ export async function transcribeSegment(
382
389
  // identical answer. BOTH flows can reach it, which is newer than it looks:
383
390
  // the check used to hang off the resampler, so a 12-bit recording already at
384
391
  // 16 kHz mono — light for both flows, and therefore converted by neither —
385
- // sailed past it into an unclassified `RangeError` from `encodeWav`.
392
+ // sailed past it into an unclassified `RangeError` from the header writer.
386
393
  const light = fatalOnUnsupported(() => downsampleSegment(audio.bytes, format));
387
394
 
388
395
  const { value: text, ms } = await timed(() =>
389
396
  transcribeWav(
390
- encodeWav(light.bytes, light.format),
397
+ [wavHeader(light.format, light.bytes.byteLength), light.bytes],
391
398
  `segment-${segment.index}.wav`,
392
399
  `Segment ${segment.index} (${formatDuration(segment.startMs)})`,
393
400
  ),
@@ -29,7 +29,15 @@
29
29
  * tools and read as a model that refuses to act.
30
30
  */
31
31
  import agentDef from "virtual:aai/agent";
32
- import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
32
+ import {
33
+ callsIn,
34
+ describeTurn,
35
+ type EvalSession,
36
+ type EvalToolCall,
37
+ lastStateIn,
38
+ toolNames,
39
+ turnCalling,
40
+ } from "@alexkroman1/aai-runtime/eval";
33
41
  import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
34
42
  import { expect } from "vitest";
35
43
  import { z } from "zod";
@@ -76,33 +84,17 @@ function framesBeforeConfirm(session: EvalSession): z.infer<typeof ProjectedTrip
76
84
  const tripState = (session: EvalSession) => lastStateIn(session.events(), ProjectedTrip);
77
85
 
78
86
  /**
79
- * Drive a whole call, one caller line at a time, and hand back every turn.
87
+ * A call that really STAGED it answered with the read-back rather than with a
88
+ * gate's refusal.
80
89
  *
81
- * The cases below assert about the turn a MECHANISM fired in rather than about
82
- * turn one, because how many turns a desk takes to get there is the model's
83
- * business and it moved when the desk gate landed: the flight desk's brief says
84
- * to search before quoting anything, so measured live this concierge now spends
85
- * its first turn on `to_flight_assistant` and `search_flights` and reads the
86
- * fare back before it stages. A case pinned to turn one is a flake with a
87
- * misleading name — the same argument `retail`'s eval carries.
90
+ * Passed to `turnCalling` as its `where`, so the turn a case reads is the one
91
+ * the MECHANISM fired in rather than turn one: how many turns a desk spends
92
+ * getting there is the model's business and it moved when the desk gate landed
93
+ * — the flight desk's brief says to search before quoting anything, so measured
94
+ * live this concierge spends its first turn on `to_flight_assistant` and
95
+ * `search_flights` and reads the fare back before it stages.
88
96
  */
89
- async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
90
- const turns: EvalTurn[] = [];
91
- for (const line of lines) turns.push(await session.say(line));
92
- return turns;
93
- }
94
-
95
- /** Every tool call of the call so far, flattened, in order. */
96
- const callsIn = (turns: readonly EvalTurn[]) => turns.flatMap((turn) => turn.toolCalls);
97
-
98
- /** The turn a named tool STAGED something in — the call that answered with the
99
- * read-back rather than with a gate's refusal. */
100
- const stagingTurn = (turns: readonly EvalTurn[], tool: string) =>
101
- turns.find((turn) =>
102
- turn.toolCalls.some(
103
- (call) => call.name === tool && /awaitingConfirmation/.test(call.result ?? ""),
104
- ),
105
- );
97
+ const stagedSomething = (call: EvalToolCall) => /awaitingConfirmation/.test(call.result ?? "");
106
98
 
107
99
  describeEval(agentDef, (test) => {
108
100
  test(
@@ -112,28 +104,25 @@ describeEval(agentDef, (test) => {
112
104
  // a read-back: what is asserted below is that the turn which staged did
113
105
  // not also apply, so a line the model could read as consent ("correct",
114
106
  // "that's right") would be measuring the caller instead of the desk.
115
- const turns = await sayAll(session, [
107
+ const turns = await session.sayAll([
116
108
  "Move my ticket to flight LX52, the Wednesday one.",
117
109
  "I want the Wednesday LX52 instead of the flight I'm on now.",
118
110
  "Put me on LX52 on Wednesday, please.",
119
111
  ]);
120
112
 
121
- const staging = stagingTurn(turns, "update_ticket");
113
+ // A desk that talks its way through three turns without staging fails
114
+ // HERE — the failure this case caught while the flight desk's brief had
115
+ // the read-back before the staging — and `turnCalling`'s throw names
116
+ // every turn's tool list AND tells the two findings apart: no
117
+ // `update_ticket` at all, or calls that were all refused by the desk gate.
118
+ const staging = turnCalling(turns, "update_ticket", stagedSomething);
122
119
  const attempts = callsIn(turns).filter((call) => call.name === "update_ticket");
123
- // Named with the whole call, tools AND text: "expected undefined to be
124
- // defined" says nothing about a desk that talked its way through three
125
- // turns without staging, which is exactly the failure this case caught
126
- // while the flight desk's brief had the read-back before the staging.
127
- expect(
128
- staging,
129
- turns
130
- .map(
131
- (turn, i) =>
132
- `turn ${i + 1}: [${turn.toolCalls.map((c) => c.name).join(", ")}] said: ${turn.text}`,
133
- )
134
- .join("\n"),
135
- ).toBeDefined();
136
- const staged = staging?.toolCalls.find((call) => call.name === "update_ticket");
120
+ // The staging call itself, by INDEX, because what follows it in the same
121
+ // turn is the subject of the assertion below.
122
+ const stagedAt = staging.toolCalls.findIndex(
123
+ (call) => call.name === "update_ticket" && stagedSomething(call),
124
+ );
125
+ const staged = staging.toolCalls[stagedAt];
137
126
  // The tool answered with the read-back rather than with a receipt.
138
127
  expect(staged?.result).toMatch(/awaitingConfirmation/);
139
128
  // Any attempt that did NOT stage is the DESK GATE refusing:
@@ -154,14 +143,7 @@ describeEval(agentDef, (test) => {
154
143
  // refusal is the subject), and it then stages properly. That is a wasted
155
144
  // step rather than an unasked-for change, and folding the two together
156
145
  // would fail this case for the behaviour the next one proves is safe.
157
- // Narrowed first: `indexOf` takes a value, and `staged` is optional — the
158
- // rewrite Biome offers for `findIndex` over an identity is UNSAFE for
159
- // exactly that reason, and the assertion above is what makes an absent
160
- // staging call a failure rather than a slice from 0.
161
- const stagedAt = staged === undefined ? -1 : (staging?.toolCalls.indexOf(staged) ?? -1);
162
- expect(staging?.toolCalls.slice(stagedAt + 1).map((call) => call.name) ?? []).not.toContain(
163
- "confirm_action",
164
- );
146
+ expect(toolNames(staging.toolCalls.slice(stagedAt + 1))).not.toContain("confirm_action");
165
147
 
166
148
  const views = framesBeforeConfirm(session);
167
149
  const waiting = views.filter((view) => view.pending !== null);
@@ -212,7 +194,7 @@ describeEval(agentDef, (test) => {
212
194
  // it applies in is its own business — the flight desk's brief has it
213
195
  // search first — and saying yes repeatedly is what makes "once each"
214
196
  // below a claim about the MECHANISM rather than about the model's pacing.
215
- await sayAll(session, [
197
+ await session.sayAll([
216
198
  "Move my ticket to flight LX52, the Wednesday one.",
217
199
  "Correct — LX52 on Wednesday. Please move my ticket to it.",
218
200
  "Yes, that's right — go ahead and change it.",
@@ -231,7 +213,7 @@ describeEval(agentDef, (test) => {
231
213
  );
232
214
  // Staged first, applied second, once each. Reversed — or a confirm with no
233
215
  // stage — is the regression this template's whole shape exists to prevent.
234
- expect(effective.map((call) => call.name)).toEqual(["update_ticket", "confirm_action"]);
216
+ expect(toolNames(effective)).toEqual(["update_ticket", "confirm_action"]);
235
217
  // Everything else has to be a GATE refusing, and nothing else: the desk
236
218
  // gate turns away an `update_ticket` issued before
237
219
  // `to_flight_assistant`, and the confirmation gate turns away a
@@ -277,10 +259,9 @@ describeEval(agentDef, (test) => {
277
259
  // below and `indexOf` on a possibly-undefined find is worse than both.
278
260
  const handoffAt = turn.toolCalls.findIndex((call) => call.name === "to_hotel_assistant");
279
261
  const handoff = turn.toolCalls[handoffAt];
280
- expect(
281
- handoff,
282
- `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
283
- ).toBeDefined();
262
+ // `describeTurn` is the message a bare `toBeDefined()` failure leaves
263
+ // out: what this turn reached for, and what it said instead.
264
+ expect(handoff, describeTurn(turn)).toBeDefined();
284
265
  // The brief IS the tool result, which is the whole port of their
285
266
  // per-assistant prompt onto a session whose prompt is fixed at connect.
286
267
  expect(handoff?.result).toMatch(/hotel desk/);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alexkroman1/aai-cli",
3
- "version": "13.1.0",
3
+ "version": "13.2.0",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -52,9 +52,9 @@
52
52
  "p-timeout": "^7.0.1",
53
53
  "vite": "^8.2.2",
54
54
  "zod": "^4.5.4",
55
- "@alexkroman1/aai": "13.1.0",
56
- "@alexkroman1/aai-runtime": "13.1.0",
57
- "@alexkroman1/aai-ui": "13.1.0"
55
+ "@alexkroman1/aai": "13.2.0",
56
+ "@alexkroman1/aai-runtime": "13.2.0",
57
+ "@alexkroman1/aai-ui": "13.2.0"
58
58
  },
59
59
  "devDependencies": {
60
60
  "playwright": "^1.62.1",