@alexkroman1/aai-cli 13.0.0 → 13.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scaffold/package.json +4 -4
- package/dist/templates/call-audit/agent.eval.test.ts +14 -11
- package/dist/templates/call-audit/agent.test.ts +20 -4
- package/dist/templates/call-audit/workflows/ingest.ts +10 -1
- package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
- package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
- package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
- package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
- package/dist/templates/link-digest/agent.eval.test.ts +24 -15
- package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
- package/dist/templates/night-owl/agent.eval.test.ts +30 -15
- package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
- package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
- package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
- package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
- package/dist/templates/redline/agent.eval.test.ts +32 -24
- package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
- package/dist/templates/retail/agent.eval.test.ts +18 -34
- package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
- package/dist/templates/spoken-summary/agent.test.ts +9 -4
- package/dist/templates/support-line/agent.eval.test.ts +23 -26
- package/dist/templates/transcription-workflow/agent.test.ts +10 -0
- package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
- package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
- package/package.json +4 -4
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
// exercised. `run.slept` below is the other half of that admission written as
|
|
49
49
|
// an assertion. `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that
|
|
50
50
|
// really suspends and resumes a run.
|
|
51
|
+
import { stubGatewayRoute } from "@alexkroman1/aai/testing";
|
|
51
52
|
import {
|
|
52
53
|
installStubSpeech,
|
|
53
54
|
installStubTranscribe,
|
|
@@ -115,23 +116,30 @@ function publish(bytes: Uint8Array, name: string, type: string) {
|
|
|
115
116
|
*
|
|
116
117
|
* ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
|
|
117
118
|
* AND calls a model cannot install two, which is exactly what `otherwise` is
|
|
118
|
-
* for.
|
|
119
|
-
*
|
|
120
|
-
* pass because the fake and the step agree
|
|
119
|
+
* for. BOTH halves are the SDK's own fakes rather than this file's hand-typed
|
|
120
|
+
* wire, and it is the same argument twice: each routes off the SDK's own
|
|
121
|
+
* endpoint constant, so a case cannot pass because the fake and the step agree
|
|
122
|
+
* on a typo. The predicate here used to be `url.includes("llm-gateway")` — a
|
|
123
|
+
* HOST, which the default gateway happens to carry and a `gatewayUrl` pointed
|
|
124
|
+
* anywhere else does not, so the fake would have gone on answering the
|
|
125
|
+
* transcription 404 to a model call it no longer recognised.
|
|
126
|
+
*
|
|
127
|
+
* The reader also hands back DECODED calls, which is why the case below asks
|
|
128
|
+
* `model.calls[0].prompt` what the model was SHOWN: off a raw request body that
|
|
129
|
+
* is the whole serialized request, `model` and `reasoning_effort` included.
|
|
121
130
|
*/
|
|
122
131
|
function scriptProvider(options: { text?: string; pendingPolls?: number } = {}) {
|
|
123
|
-
|
|
132
|
+
const model = stubGatewayRoute(JSON.stringify(REPLY));
|
|
133
|
+
const provider = installStubTranscribe({
|
|
124
134
|
text: options.text ?? TRANSCRIPT,
|
|
125
135
|
durationSec: 42,
|
|
126
136
|
// Passed straight through rather than conditionally spread: the option
|
|
127
137
|
// already admits `undefined`, and `guard-invariants` rule 2 counts the
|
|
128
138
|
// spread.
|
|
129
139
|
pendingPolls: options.pendingPolls,
|
|
130
|
-
otherwise: (request) =>
|
|
131
|
-
request.url.includes("llm-gateway")
|
|
132
|
-
? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
|
|
133
|
-
: undefined,
|
|
140
|
+
otherwise: (request) => model.route(request),
|
|
134
141
|
});
|
|
142
|
+
return { provider, model };
|
|
135
143
|
}
|
|
136
144
|
|
|
137
145
|
describeWorkflowEval(
|
|
@@ -144,7 +152,7 @@ describeWorkflowEval(
|
|
|
144
152
|
// returned bytes, or an id nothing wrote, or two ids because the synthesis
|
|
145
153
|
// and the store became two steps.
|
|
146
154
|
const uploads = publish(new Uint8Array(64), "standup.wav", "audio/wav");
|
|
147
|
-
const provider = scriptProvider();
|
|
155
|
+
const { provider } = scriptProvider();
|
|
148
156
|
const speech = installStubSpeech({ pcmBytes: 96_000 });
|
|
149
157
|
|
|
150
158
|
const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
|
|
@@ -207,7 +215,7 @@ describeWorkflowEval(
|
|
|
207
215
|
// central prompt decision regressing — synthesize the bullet list and you
|
|
208
216
|
// get a voice reading "one. two. three." with no connective tissue.
|
|
209
217
|
publish(new Uint8Array(64), "standup.wav", "audio/wav");
|
|
210
|
-
const
|
|
218
|
+
const { model } = scriptProvider();
|
|
211
219
|
const speech = installStubSpeech();
|
|
212
220
|
|
|
213
221
|
const run = await app.run(spokenSummary, { recording: UPLOAD_ID, voice: "michael" });
|
|
@@ -226,9 +234,10 @@ describeWorkflowEval(
|
|
|
226
234
|
|
|
227
235
|
// And the model was ASKED for both, over the transcript it was given. A
|
|
228
236
|
// prompt that stopped asking for a script is how the field goes missing.
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
expect(prompt).toContain("
|
|
237
|
+
const asked = model.calls[0];
|
|
238
|
+
if (asked === undefined) expect.fail("the run must have asked the model for a summary");
|
|
239
|
+
expect(asked.prompt).toContain("READ ALOUD");
|
|
240
|
+
expect(asked.prompt).toContain("The launch is on for Tuesday the fourth");
|
|
232
241
|
});
|
|
233
242
|
|
|
234
243
|
test("a recording with no speech stops before the model and the voice", async ({ app }) => {
|
|
@@ -237,7 +246,7 @@ describeWorkflowEval(
|
|
|
237
246
|
// the run would go on to summarize no words and store half a second of
|
|
238
247
|
// audio — a green run with an empty product.
|
|
239
248
|
const uploads = publish(new Uint8Array(64), "silence.wav", "audio/wav");
|
|
240
|
-
const
|
|
249
|
+
const { model } = scriptProvider({ text: " " });
|
|
241
250
|
const speech = installStubSpeech();
|
|
242
251
|
|
|
243
252
|
const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
|
|
@@ -247,7 +256,7 @@ describeWorkflowEval(
|
|
|
247
256
|
expect(run.output).toBeUndefined();
|
|
248
257
|
// Nothing was summarized and nothing was spoken, which is the half that
|
|
249
258
|
// makes this more than an error-message assertion.
|
|
250
|
-
expect(
|
|
259
|
+
expect(model.calls).toEqual([]);
|
|
251
260
|
expect(speech.calls).toEqual([]);
|
|
252
261
|
expect(uploads.writes).toEqual([]);
|
|
253
262
|
expect(run.reported).not.toContain("Summarizing the transcript.");
|
|
@@ -262,7 +271,7 @@ describeWorkflowEval(
|
|
|
262
271
|
// them. A loop that re-submitted, or one that spun with no wait, both
|
|
263
272
|
// produce a correct transcript and a wrong bill.
|
|
264
273
|
publish(new Uint8Array(64), "standup.wav", "audio/wav");
|
|
265
|
-
const provider = scriptProvider({ pendingPolls: 2 });
|
|
274
|
+
const { provider } = scriptProvider({ pendingPolls: 2 });
|
|
266
275
|
installStubSpeech();
|
|
267
276
|
|
|
268
277
|
const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
|
|
@@ -257,15 +257,20 @@ describe("the whole run", () => {
|
|
|
257
257
|
* spec cannot pass because the fake and the step agree on a typo.
|
|
258
258
|
*/
|
|
259
259
|
function stubProvider(reply: { headline: string; points: string[]; spoken: string }) {
|
|
260
|
+
// The model leg goes through `stubGatewayRoute` for the same reason, which
|
|
261
|
+
// this helper used to claim and not do: it hand-typed the completion
|
|
262
|
+
// envelope and recognised a model call by `llm-gateway`, the HOST. That is a
|
|
263
|
+
// property of one deployment rather than of the request — `stepGenerate`
|
|
264
|
+
// dials `${gatewayUrl ?? ASSEMBLYAI_LLM_GATEWAY_URL}/chat/completions`, so a
|
|
265
|
+
// caller pointing `gatewayUrl` at an OpenAI-compatible proxy of their own
|
|
266
|
+
// stops matching and the transcription fake answers a 404 to a model call.
|
|
267
|
+
const model = stubGatewayRoute(JSON.stringify(reply));
|
|
260
268
|
return installStubTranscribe({
|
|
261
269
|
audioUrl: "https://cdn/aai/1",
|
|
262
270
|
jobIdPrefix: "t_",
|
|
263
271
|
text: "we ship tuesday and two bugs are left",
|
|
264
272
|
durationSec: 42,
|
|
265
|
-
otherwise: (request) =>
|
|
266
|
-
request.url.includes("llm-gateway")
|
|
267
|
-
? { body: { choices: [{ message: { content: JSON.stringify(reply) } }] } }
|
|
268
|
-
: undefined,
|
|
273
|
+
otherwise: (request) => model.route(request),
|
|
269
274
|
});
|
|
270
275
|
}
|
|
271
276
|
|
|
@@ -27,7 +27,13 @@
|
|
|
27
27
|
|
|
28
28
|
/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
|
|
29
29
|
import agentDef from "virtual:aai/agent";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
describeToolCalls,
|
|
32
|
+
describeTurn,
|
|
33
|
+
type EvalSession,
|
|
34
|
+
statesIn,
|
|
35
|
+
toolResultIn,
|
|
36
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
31
37
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
32
38
|
import { expect } from "vitest";
|
|
33
39
|
import { z } from "zod";
|
|
@@ -71,13 +77,14 @@ const Lookup = z.object({
|
|
|
71
77
|
error: z.string().optional(),
|
|
72
78
|
});
|
|
73
79
|
|
|
74
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Every `syncState` frame, in stream order.
|
|
82
|
+
*
|
|
83
|
+
* `statesIn` reads an EVENT LIST rather than a session, which is what lets a
|
|
84
|
+
* case slice the stream first; the schema above is what it takes one for.
|
|
85
|
+
*/
|
|
75
86
|
function frames(session: EvalSession) {
|
|
76
|
-
return session
|
|
77
|
-
.events()
|
|
78
|
-
.flatMap((event) =>
|
|
79
|
-
event.type === "state.updated" ? [ProjectedSupport.parse(event.state)] : [],
|
|
80
|
-
);
|
|
87
|
+
return statesIn(session.events(), ProjectedSupport);
|
|
81
88
|
}
|
|
82
89
|
|
|
83
90
|
/**
|
|
@@ -109,10 +116,10 @@ describeEval(agentDef, (test) => {
|
|
|
109
116
|
// The reply rides in the message, because the failure that matters here is
|
|
110
117
|
// a turn that SPOKE without looking anything up — "let me check that for
|
|
111
118
|
// you" and then nothing, or worse, a notice period from memory.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
).toBe(1);
|
|
119
|
+
// `describeTurn` is that sentence, and it says the two things a
|
|
120
|
+
// hand-built one left out: "called no tools" rather than an empty
|
|
121
|
+
// bracket, and whether the reply was cancelled.
|
|
122
|
+
expect(asked.length, describeTurn(turn)).toBe(1);
|
|
116
123
|
|
|
117
124
|
for (const payload of lookups(session)) {
|
|
118
125
|
// Three legal outcomes, and the invariant that spans them: an answer the
|
|
@@ -142,13 +149,9 @@ describeEval(agentDef, (test) => {
|
|
|
142
149
|
await session.say("Yes please, log that one — my callback number is 07700 900123.");
|
|
143
150
|
|
|
144
151
|
const logged = session.toolCalls().find((call) => call.name === "log_ticket");
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
.toolCalls()
|
|
149
|
-
.map((c) => c.name)
|
|
150
|
-
.join(", ")}`,
|
|
151
|
-
).toBeDefined();
|
|
152
|
+
// The claim spans both turns, so the message does too: `describeToolCalls`
|
|
153
|
+
// over the session's own list, where `describeTurn` would describe one.
|
|
154
|
+
expect(logged, describeToolCalls(session.toolCalls())).toBeDefined();
|
|
152
155
|
expect(logged?.result).toMatch(/TCK\d{4}/);
|
|
153
156
|
|
|
154
157
|
const latest = frames(session).at(-1);
|
|
@@ -189,10 +192,7 @@ describeEval(agentDef, (test) => {
|
|
|
189
192
|
const turn = await session.say("How much notice do I have to give to cancel my contract?");
|
|
190
193
|
|
|
191
194
|
const [payload] = lookups(session);
|
|
192
|
-
expect(
|
|
193
|
-
payload,
|
|
194
|
-
`tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
|
|
195
|
-
).toBeDefined();
|
|
195
|
+
expect(payload, describeTurn(turn)).toBeDefined();
|
|
196
196
|
// The whole verdict in the message: `grounded: undefined` on its own does
|
|
197
197
|
// not say whether the lookup failed, or ran and refused.
|
|
198
198
|
const verdict = JSON.stringify(payload);
|
|
@@ -238,10 +238,7 @@ describeEval(agentDef, (test) => {
|
|
|
238
238
|
);
|
|
239
239
|
|
|
240
240
|
const [payload] = lookups(session);
|
|
241
|
-
expect(
|
|
242
|
-
payload,
|
|
243
|
-
`tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
|
|
244
|
-
).toBeDefined();
|
|
241
|
+
expect(payload, describeTurn(turn)).toBeDefined();
|
|
245
242
|
// Withheld, not softened: `answer: null` is the tool refusing to hand the
|
|
246
243
|
// model something to read out, and the guidance is the exit the grading
|
|
247
244
|
// apparatus needs — a support line that can only answer will answer wrong.
|
|
@@ -779,6 +779,16 @@ describe("transcribeSegment", () => {
|
|
|
779
779
|
expect(decoded).toContain('name="audio"; filename="segment-0.wav"');
|
|
780
780
|
// The WAV really rides in the part, header and all.
|
|
781
781
|
expect(decoded).toContain("RIFF");
|
|
782
|
+
// And the header is CONTIGUOUS with its samples, which is what the two-chunk
|
|
783
|
+
// form (`[wavHeader(…), window]`) has to preserve and the only thing it could
|
|
784
|
+
// plausibly lose: the part's payload is exactly the 44 bytes plus the window,
|
|
785
|
+
// with nothing between them and nothing appended. A body that grew or shrank
|
|
786
|
+
// here is a file the endpoint decodes into confident nonsense rather than
|
|
787
|
+
// refusing.
|
|
788
|
+
const latin = new TextDecoder("latin1").decode(sent);
|
|
789
|
+
const from = latin.indexOf("RIFF");
|
|
790
|
+
const to = latin.lastIndexOf("\r\n--");
|
|
791
|
+
expect(to - from).toBe(44 + (SEGMENT.end - SEGMENT.start));
|
|
782
792
|
});
|
|
783
793
|
|
|
784
794
|
test("sends the DOWNSAMPLED window when the recording is heavier than 16 kHz mono", async () => {
|
|
@@ -159,7 +159,16 @@ export async function normalizeRecording(uploadId: string): Promise<NormalizedRe
|
|
|
159
159
|
const source = join(dir, "source");
|
|
160
160
|
const converted = join(dir, "converted.wav");
|
|
161
161
|
|
|
162
|
-
|
|
162
|
+
// NO `size`, though `stored.size` is right there — and that is the whole
|
|
163
|
+
// difference between this copy being one window at a time and being
|
|
164
|
+
// `STEP_FILE_READ_CONCURRENCY` of them. Passing `size` means "I am judging
|
|
165
|
+
// completeness myself", which is what a body polling a still-arriving
|
|
166
|
+
// upload needs and is the opposite of what happened above: this step has
|
|
167
|
+
// already called `requireCompleteUpload`, so the file IS whole and the
|
|
168
|
+
// windows may land in any order. Omitting it lets `readUploadToFile`
|
|
169
|
+
// establish that for itself and fan out. The cost is one metadata round
|
|
170
|
+
// trip, against the dozens of window reads it overlaps.
|
|
171
|
+
await readUploadToFile(uploadId, source);
|
|
163
172
|
|
|
164
173
|
// What it WAS, for the progress line. Worth one ffprobe: "converted 41
|
|
165
174
|
// minutes of aac" is a line that explains the run's shape, where
|
|
@@ -52,7 +52,10 @@ export function elapsed(ms: number): string {
|
|
|
52
52
|
* `bytes` must be a whole file, header included — the endpoint decodes each
|
|
53
53
|
* request independently, so a headerless tail is bytes it will refuse. Both
|
|
54
54
|
* callers arrive at that differently: one re-attaches a header to a window it
|
|
55
|
-
* read, the other is handed parts that already carry one.
|
|
55
|
+
* read, the other is handed parts that already carry one. A LIST is a whole
|
|
56
|
+
* file too: the segment caller passes `[wavHeader(...), window]` so the two are
|
|
57
|
+
* concatenated straight into the request body rather than into an intermediate
|
|
58
|
+
* buffer that doubles the segment's footprint.
|
|
56
59
|
*
|
|
57
60
|
* `stepTranscribeSyncClassified` — the SDK's own `stepTranscribeSync` plus
|
|
58
61
|
* `throwStepError`, and nothing else — is the whole of what this adds to the SDK
|
|
@@ -68,7 +71,7 @@ export function elapsed(ms: number): string {
|
|
|
68
71
|
* log has in front of them.
|
|
69
72
|
*/
|
|
70
73
|
export async function transcribeWav(
|
|
71
|
-
bytes: Uint8Array,
|
|
74
|
+
bytes: Uint8Array | readonly Uint8Array[],
|
|
72
75
|
filename: string,
|
|
73
76
|
label: string,
|
|
74
77
|
): Promise<string> {
|
|
@@ -66,12 +66,12 @@
|
|
|
66
66
|
import type { WorkflowCtx } from "@alexkroman1/aai";
|
|
67
67
|
import {
|
|
68
68
|
emit,
|
|
69
|
-
encodeWav,
|
|
70
69
|
mapConcurrent,
|
|
71
70
|
readUpload,
|
|
72
71
|
report,
|
|
73
72
|
requireCompleteUpload,
|
|
74
73
|
uploadInfo,
|
|
74
|
+
wavHeader,
|
|
75
75
|
} from "@alexkroman1/aai/step";
|
|
76
76
|
import { throwFatalStepError } from "@alexkroman1/aai/step-errors";
|
|
77
77
|
import { countWords, formatDuration, plural } from "@alexkroman1/aai/utils";
|
|
@@ -358,13 +358,20 @@ export async function transcribeSegment(
|
|
|
358
358
|
// answers better — and getting it wrong is a whole transcript in the wrong
|
|
359
359
|
// language. Add one back only for a desk that really knows.
|
|
360
360
|
//
|
|
361
|
-
//
|
|
361
|
+
// A HEADER is what makes a WINDOW decodable: the endpoint decodes each
|
|
362
362
|
// request independently, so a slice of the middle of a recording is a headerless
|
|
363
363
|
// tail until one is put back on it. The streaming flow needs no equivalent — its
|
|
364
364
|
// parts were cut with a header each. The header is the SDK's rather than this
|
|
365
365
|
// template's: a `WavFormat` is structurally a `PcmFormat`, and 22 lines of
|
|
366
366
|
// `DataView` writes with a comment about which of the two declared lengths a
|
|
367
367
|
// decoder trusts is not a thing worth a second copy of.
|
|
368
|
+
// It goes down as its own CHUNK rather than through `encodeWav`, which is a
|
|
369
|
+
// MEMORY decision and not a speed one: `encodeWav` allocates `44 + N` and
|
|
370
|
+
// copies the segment into it, and `multipartBody` then allocates the body and
|
|
371
|
+
// copies that again — so the audio was resident three times at the moment the
|
|
372
|
+
// request went out, on a fan-out whose width is set by exactly that peak (see
|
|
373
|
+
// `MAX_SEGMENT_CONCURRENCY`). Header and samples are contiguous on the wire
|
|
374
|
+
// either way; this holds `44 + N` once, ~3 MB per in-flight 16 kHz segment.
|
|
368
375
|
// Down to 16 kHz mono BEFORE the header goes on, because the endpoint's budget
|
|
369
376
|
// is 30 seconds of wall clock and that covers the upload. At 48 kHz stereo this
|
|
370
377
|
// window is 17.66 MB and the same audio is 2.94 MB normalized — six times the
|
|
@@ -382,12 +389,12 @@ export async function transcribeSegment(
|
|
|
382
389
|
// identical answer. BOTH flows can reach it, which is newer than it looks:
|
|
383
390
|
// the check used to hang off the resampler, so a 12-bit recording already at
|
|
384
391
|
// 16 kHz mono — light for both flows, and therefore converted by neither —
|
|
385
|
-
// sailed past it into an unclassified `RangeError` from
|
|
392
|
+
// sailed past it into an unclassified `RangeError` from the header writer.
|
|
386
393
|
const light = fatalOnUnsupported(() => downsampleSegment(audio.bytes, format));
|
|
387
394
|
|
|
388
395
|
const { value: text, ms } = await timed(() =>
|
|
389
396
|
transcribeWav(
|
|
390
|
-
|
|
397
|
+
[wavHeader(light.format, light.bytes.byteLength), light.bytes],
|
|
391
398
|
`segment-${segment.index}.wav`,
|
|
392
399
|
`Segment ${segment.index} (${formatDuration(segment.startMs)})`,
|
|
393
400
|
),
|
|
@@ -29,7 +29,15 @@
|
|
|
29
29
|
* tools and read as a model that refuses to act.
|
|
30
30
|
*/
|
|
31
31
|
import agentDef from "virtual:aai/agent";
|
|
32
|
-
import {
|
|
32
|
+
import {
|
|
33
|
+
callsIn,
|
|
34
|
+
describeTurn,
|
|
35
|
+
type EvalSession,
|
|
36
|
+
type EvalToolCall,
|
|
37
|
+
lastStateIn,
|
|
38
|
+
toolNames,
|
|
39
|
+
turnCalling,
|
|
40
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
33
41
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
34
42
|
import { expect } from "vitest";
|
|
35
43
|
import { z } from "zod";
|
|
@@ -76,33 +84,17 @@ function framesBeforeConfirm(session: EvalSession): z.infer<typeof ProjectedTrip
|
|
|
76
84
|
const tripState = (session: EvalSession) => lastStateIn(session.events(), ProjectedTrip);
|
|
77
85
|
|
|
78
86
|
/**
|
|
79
|
-
*
|
|
87
|
+
* A call that really STAGED — it answered with the read-back rather than with a
|
|
88
|
+
* gate's refusal.
|
|
80
89
|
*
|
|
81
|
-
*
|
|
82
|
-
* turn one
|
|
83
|
-
* business and it moved when the desk gate landed
|
|
84
|
-
* to search before quoting anything, so measured
|
|
85
|
-
* its first turn on `to_flight_assistant` and
|
|
86
|
-
* fare back before it stages.
|
|
87
|
-
* misleading name — the same argument `retail`'s eval carries.
|
|
90
|
+
* Passed to `turnCalling` as its `where`, so the turn a case reads is the one
|
|
91
|
+
* the MECHANISM fired in rather than turn one: how many turns a desk spends
|
|
92
|
+
* getting there is the model's business and it moved when the desk gate landed
|
|
93
|
+
* — the flight desk's brief says to search before quoting anything, so measured
|
|
94
|
+
* live this concierge spends its first turn on `to_flight_assistant` and
|
|
95
|
+
* `search_flights` and reads the fare back before it stages.
|
|
88
96
|
*/
|
|
89
|
-
|
|
90
|
-
const turns: EvalTurn[] = [];
|
|
91
|
-
for (const line of lines) turns.push(await session.say(line));
|
|
92
|
-
return turns;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/** Every tool call of the call so far, flattened, in order. */
|
|
96
|
-
const callsIn = (turns: readonly EvalTurn[]) => turns.flatMap((turn) => turn.toolCalls);
|
|
97
|
-
|
|
98
|
-
/** The turn a named tool STAGED something in — the call that answered with the
|
|
99
|
-
* read-back rather than with a gate's refusal. */
|
|
100
|
-
const stagingTurn = (turns: readonly EvalTurn[], tool: string) =>
|
|
101
|
-
turns.find((turn) =>
|
|
102
|
-
turn.toolCalls.some(
|
|
103
|
-
(call) => call.name === tool && /awaitingConfirmation/.test(call.result ?? ""),
|
|
104
|
-
),
|
|
105
|
-
);
|
|
97
|
+
const stagedSomething = (call: EvalToolCall) => /awaitingConfirmation/.test(call.result ?? "");
|
|
106
98
|
|
|
107
99
|
describeEval(agentDef, (test) => {
|
|
108
100
|
test(
|
|
@@ -112,28 +104,25 @@ describeEval(agentDef, (test) => {
|
|
|
112
104
|
// a read-back: what is asserted below is that the turn which staged did
|
|
113
105
|
// not also apply, so a line the model could read as consent ("correct",
|
|
114
106
|
// "that's right") would be measuring the caller instead of the desk.
|
|
115
|
-
const turns = await sayAll(
|
|
107
|
+
const turns = await session.sayAll([
|
|
116
108
|
"Move my ticket to flight LX52, the Wednesday one.",
|
|
117
109
|
"I want the Wednesday LX52 instead of the flight I'm on now.",
|
|
118
110
|
"Put me on LX52 on Wednesday, please.",
|
|
119
111
|
]);
|
|
120
112
|
|
|
121
|
-
|
|
113
|
+
// A desk that talks its way through three turns without staging fails
|
|
114
|
+
// HERE — the failure this case caught while the flight desk's brief had
|
|
115
|
+
// the read-back before the staging — and `turnCalling`'s throw names
|
|
116
|
+
// every turn's tool list AND tells the two findings apart: no
|
|
117
|
+
// `update_ticket` at all, or calls that were all refused by the desk gate.
|
|
118
|
+
const staging = turnCalling(turns, "update_ticket", stagedSomething);
|
|
122
119
|
const attempts = callsIn(turns).filter((call) => call.name === "update_ticket");
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
turns
|
|
130
|
-
.map(
|
|
131
|
-
(turn, i) =>
|
|
132
|
-
`turn ${i + 1}: [${turn.toolCalls.map((c) => c.name).join(", ")}] said: ${turn.text}`,
|
|
133
|
-
)
|
|
134
|
-
.join("\n"),
|
|
135
|
-
).toBeDefined();
|
|
136
|
-
const staged = staging?.toolCalls.find((call) => call.name === "update_ticket");
|
|
120
|
+
// The staging call itself, by INDEX, because what follows it in the same
|
|
121
|
+
// turn is the subject of the assertion below.
|
|
122
|
+
const stagedAt = staging.toolCalls.findIndex(
|
|
123
|
+
(call) => call.name === "update_ticket" && stagedSomething(call),
|
|
124
|
+
);
|
|
125
|
+
const staged = staging.toolCalls[stagedAt];
|
|
137
126
|
// The tool answered with the read-back rather than with a receipt.
|
|
138
127
|
expect(staged?.result).toMatch(/awaitingConfirmation/);
|
|
139
128
|
// Any attempt that did NOT stage is the DESK GATE refusing:
|
|
@@ -154,14 +143,7 @@ describeEval(agentDef, (test) => {
|
|
|
154
143
|
// refusal is the subject), and it then stages properly. That is a wasted
|
|
155
144
|
// step rather than an unasked-for change, and folding the two together
|
|
156
145
|
// would fail this case for the behaviour the next one proves is safe.
|
|
157
|
-
|
|
158
|
-
// rewrite Biome offers for `findIndex` over an identity is UNSAFE for
|
|
159
|
-
// exactly that reason, and the assertion above is what makes an absent
|
|
160
|
-
// staging call a failure rather than a slice from 0.
|
|
161
|
-
const stagedAt = staged === undefined ? -1 : (staging?.toolCalls.indexOf(staged) ?? -1);
|
|
162
|
-
expect(staging?.toolCalls.slice(stagedAt + 1).map((call) => call.name) ?? []).not.toContain(
|
|
163
|
-
"confirm_action",
|
|
164
|
-
);
|
|
146
|
+
expect(toolNames(staging.toolCalls.slice(stagedAt + 1))).not.toContain("confirm_action");
|
|
165
147
|
|
|
166
148
|
const views = framesBeforeConfirm(session);
|
|
167
149
|
const waiting = views.filter((view) => view.pending !== null);
|
|
@@ -212,7 +194,7 @@ describeEval(agentDef, (test) => {
|
|
|
212
194
|
// it applies in is its own business — the flight desk's brief has it
|
|
213
195
|
// search first — and saying yes repeatedly is what makes "once each"
|
|
214
196
|
// below a claim about the MECHANISM rather than about the model's pacing.
|
|
215
|
-
await sayAll(
|
|
197
|
+
await session.sayAll([
|
|
216
198
|
"Move my ticket to flight LX52, the Wednesday one.",
|
|
217
199
|
"Correct — LX52 on Wednesday. Please move my ticket to it.",
|
|
218
200
|
"Yes, that's right — go ahead and change it.",
|
|
@@ -231,7 +213,7 @@ describeEval(agentDef, (test) => {
|
|
|
231
213
|
);
|
|
232
214
|
// Staged first, applied second, once each. Reversed — or a confirm with no
|
|
233
215
|
// stage — is the regression this template's whole shape exists to prevent.
|
|
234
|
-
expect(effective
|
|
216
|
+
expect(toolNames(effective)).toEqual(["update_ticket", "confirm_action"]);
|
|
235
217
|
// Everything else has to be a GATE refusing, and nothing else: the desk
|
|
236
218
|
// gate turns away an `update_ticket` issued before
|
|
237
219
|
// `to_flight_assistant`, and the confirmation gate turns away a
|
|
@@ -277,10 +259,9 @@ describeEval(agentDef, (test) => {
|
|
|
277
259
|
// below and `indexOf` on a possibly-undefined find is worse than both.
|
|
278
260
|
const handoffAt = turn.toolCalls.findIndex((call) => call.name === "to_hotel_assistant");
|
|
279
261
|
const handoff = turn.toolCalls[handoffAt];
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
).toBeDefined();
|
|
262
|
+
// `describeTurn` is the message a bare `toBeDefined()` failure leaves
|
|
263
|
+
// out: what this turn reached for, and what it said instead.
|
|
264
|
+
expect(handoff, describeTurn(turn)).toBeDefined();
|
|
284
265
|
// The brief IS the tool result, which is the whole port of their
|
|
285
266
|
// per-assistant prompt onto a session whose prompt is fixed at connect.
|
|
286
267
|
expect(handoff?.result).toMatch(/hotel desk/);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alexkroman1/aai-cli",
|
|
3
|
-
"version": "13.
|
|
3
|
+
"version": "13.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -52,9 +52,9 @@
|
|
|
52
52
|
"p-timeout": "^7.0.1",
|
|
53
53
|
"vite": "^8.2.2",
|
|
54
54
|
"zod": "^4.5.4",
|
|
55
|
-
"@alexkroman1/aai": "13.
|
|
56
|
-
"@alexkroman1/aai-runtime": "13.
|
|
57
|
-
"@alexkroman1/aai-ui": "13.
|
|
55
|
+
"@alexkroman1/aai": "13.2.0",
|
|
56
|
+
"@alexkroman1/aai-runtime": "13.2.0",
|
|
57
|
+
"@alexkroman1/aai-ui": "13.2.0"
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"playwright": "^1.62.1",
|