@alexkroman1/aai-cli 13.1.0 → 13.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +1 -1
- package/dist/{_agent-BTwZJu4P.mjs → _agent-Clbq3a4h.mjs} +2 -2
- package/dist/{_bundler-C31rlcYz.mjs → _bundler-DEkjO1d8.mjs} +3 -3
- package/dist/{_config-CmJOFsAP.mjs → _config-ZY2euhdh.mjs} +7 -4
- package/dist/_config.d.ts +25 -1
- package/dist/{_dev-server-DWvkNoIS.mjs → _dev-server-C13xuGB1.mjs} +13 -13
- package/dist/_e2e-test-utils.d.ts +15 -7
- package/dist/{_init-BURbdHNM.mjs → _init-Ci4QGjp8.mjs} +58 -10
- package/dist/{_output-CKkmWs7i.mjs → _output-DbJt9gsl.mjs} +34 -3
- package/dist/_output.d.ts +17 -1
- package/dist/{_preflight-Co1G9Jww.mjs → _preflight-B2laWPdU.mjs} +1 -1
- package/dist/{_server-common-6e0QI4mq.mjs → _server-common-BV4_CrP3.mjs} +3 -3
- package/dist/{_slug-api-CfGQq1Tt.mjs → _slug-api-DlKpj91X.mjs} +3 -3
- package/dist/{_studio-B1waXMuR.mjs → _studio-DTi3EdXQ.mjs} +3 -3
- package/dist/{_templates-0Zr2z6yA.mjs → _templates-V4l5sZJy.mjs} +2 -2
- package/dist/{_typecheck-gate-BXvUNnfs.mjs → _typecheck-gate-sV4N5F93.mjs} +3 -3
- package/dist/{_ui-u7T4YooX.mjs → _ui-DCt4qZrk.mjs} +1 -1
- package/dist/{_utils-B8QmtFhK.mjs → _utils-Dhd058D9.mjs} +1 -1
- package/dist/{_vite-env-D9Qj6KPX.mjs → _vite-env-kejF0p2L.mjs} +1 -1
- package/dist/{_workflow-determinism-f9J-5Zlj.mjs → _workflow-determinism-C9l8uC2R.mjs} +1 -1
- package/dist/{build-BLlLOrng.mjs → build-CWyFGe-W.mjs} +11 -8
- package/dist/cli.d.ts +2 -1
- package/dist/cli.mjs +120 -46
- package/dist/{client-bundler-j5c46x0G.mjs → client-bundler-CT-_26c5.mjs} +4 -4
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-X_CRgCf6.mjs → delete-RqlQZ6Y_.mjs} +6 -6
- package/dist/{deploy-BzZ-r4iM.mjs → deploy-B7f3CK7v.mjs} +13 -13
- package/dist/{dev-BDykFvcc.mjs → dev-DamXieW_.mjs} +5 -5
- package/dist/{eval-0qEOdm0l.mjs → eval-h3DTVfRw.mjs} +27 -11
- package/dist/eval.d.ts +2 -0
- package/dist/{init-BtJI306C.mjs → init-D-vFrV8h.mjs} +7 -7
- package/dist/{login-DeUETobb.mjs → login-CDiemNmC.mjs} +5 -5
- package/dist/{logs-hWnJ66Bl.mjs → logs-DGwPNCzB.mjs} +4 -4
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/.env.example +17 -1
- package/dist/scaffold/package.json +6 -5
- package/dist/secret-V1w7y9-j.mjs +162 -0
- package/dist/secret.d.ts +41 -8
- package/dist/{studio-BxAS_FQQ.mjs → studio-BpMCfUNx.mjs} +9 -9
- package/dist/templates/briefing-desk/agent.test.ts +7 -3
- package/dist/templates/call-audit/agent.eval.test.ts +14 -11
- package/dist/templates/call-audit/agent.test.ts +34 -7
- package/dist/templates/call-audit/client.tsx +14 -17
- package/dist/templates/call-audit/workflows/ingest.ts +10 -1
- package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
- package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
- package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
- package/dist/templates/embedded-assets/agent.test.ts +7 -1
- package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
- package/dist/templates/health-assistant/agent.test.ts +8 -4
- package/dist/templates/link-digest/agent.eval.test.ts +24 -15
- package/dist/templates/link-digest/agent.test.ts +7 -1
- package/dist/templates/link-digest/client.tsx +17 -21
- package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
- package/dist/templates/night-owl/agent.eval.test.ts +30 -15
- package/dist/templates/night-owl/agent.test.ts +6 -1
- package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
- package/dist/templates/pipeline-simple/agent.test.ts +31 -14
- package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
- package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
- package/dist/templates/podcast-digest/agent.test.ts +13 -3
- package/dist/templates/podcast-digest/client.tsx +12 -15
- package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
- package/dist/templates/recap-workflow/agent.test.ts +22 -9
- package/dist/templates/redline/agent.eval.test.ts +32 -24
- package/dist/templates/redline/agent.test.ts +7 -1
- package/dist/templates/redline/client.tsx +12 -15
- package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
- package/dist/templates/research-workflow/agent.test.ts +7 -1
- package/dist/templates/retail/agent.eval.test.ts +18 -34
- package/dist/templates/retail/registry.test.ts +99 -58
- package/dist/templates/simple/agent.test.ts +39 -6
- package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
- package/dist/templates/spoken-summary/agent.test.ts +21 -7
- package/dist/templates/spoken-summary/client.tsx +14 -18
- package/dist/templates/support-line/agent.eval.test.ts +23 -26
- package/dist/templates/transcription-workflow/agent.test.ts +26 -8
- package/dist/templates/transcription-workflow/client.tsx +19 -26
- package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
- package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
- package/dist/test-guenJUxi.mjs +255 -0
- package/dist/test.d.ts +97 -31
- package/dist/typecheck.mjs +2 -2
- package/dist/worker-bundler.mjs +3 -3
- package/dist/{workflow-BBu5oEDw.mjs → workflow-BZyKLkhV.mjs} +5 -5
- package/package.json +10 -10
- package/dist/bin.d.mts +0 -2
- package/dist/secret-B-3T5tvr.mjs +0 -61
- package/dist/test-CiLab-AA.mjs +0 -182
- package/dist/tsdown.config.d.ts +0 -2
|
@@ -55,13 +55,22 @@ describe("the declaration", () => {
|
|
|
55
55
|
test("is a workflow app with the one workflow the page starts by name", () => {
|
|
56
56
|
// The page calls `api.start("spokenSummary", …)`, so a rename here is a
|
|
57
57
|
// runtime 400 rather than a compile error. This is what pins it.
|
|
58
|
-
|
|
58
|
+
// `toContain` rather than an exact key list: adding a second workflow is an
|
|
59
|
+
// invited edit and must not redden a test the author did not write. The
|
|
60
|
+
// NAME is still pinned, deliberately — the page starts a run by this
|
|
61
|
+
// string, so renaming the key is a runtime 400 rather than a compile
|
|
62
|
+
// error, and this pin is the only thing that says so. Rename it here and
|
|
63
|
+
// in `client.tsx` together.
|
|
64
|
+
expect(Object.keys(agentDef.workflows ?? {})).toContain("spokenSummary");
|
|
65
|
+
expect(agentDef.workflows?.spokenSummary).toBe(spokenSummary);
|
|
59
66
|
});
|
|
60
67
|
|
|
61
|
-
test("declares
|
|
68
|
+
test("declares the credential its steps read, so a deploy checks for it", () => {
|
|
62
69
|
// A workflow app has no session, so nothing else in its config could name
|
|
63
70
|
// one — and one AssemblyAI key covers transcription, the model and the voice.
|
|
64
|
-
|
|
71
|
+
// `toContain`, so adding a step that reads a second credential and
|
|
72
|
+
// declaring it here does not fail this test.
|
|
73
|
+
expect(agentDef.requiredEnv).toContain("ASSEMBLYAI_API_KEY");
|
|
65
74
|
});
|
|
66
75
|
|
|
67
76
|
test("takes the recording as an UPLOAD, which is what makes the form a file picker", () => {
|
|
@@ -257,15 +266,20 @@ describe("the whole run", () => {
|
|
|
257
266
|
* spec cannot pass because the fake and the step agree on a typo.
|
|
258
267
|
*/
|
|
259
268
|
function stubProvider(reply: { headline: string; points: string[]; spoken: string }) {
|
|
269
|
+
// The model leg goes through `stubGatewayRoute` for the same reason, which
|
|
270
|
+
// this helper used to claim and not do: it hand-typed the completion
|
|
271
|
+
// envelope and recognised a model call by `llm-gateway`, the HOST. That is a
|
|
272
|
+
// property of one deployment rather than of the request — `stepGenerate`
|
|
273
|
+
// dials `${gatewayUrl ?? ASSEMBLYAI_LLM_GATEWAY_URL}/chat/completions`, so a
|
|
274
|
+
// caller pointing `gatewayUrl` at an OpenAI-compatible proxy of their own
|
|
275
|
+
// stops matching and the transcription fake answers a 404 to a model call.
|
|
276
|
+
const model = stubGatewayRoute(JSON.stringify(reply));
|
|
260
277
|
return installStubTranscribe({
|
|
261
278
|
audioUrl: "https://cdn/aai/1",
|
|
262
279
|
jobIdPrefix: "t_",
|
|
263
280
|
text: "we ship tuesday and two bugs are left",
|
|
264
281
|
durationSec: 42,
|
|
265
|
-
otherwise: (request) =>
|
|
266
|
-
request.url.includes("llm-gateway")
|
|
267
|
-
? { body: { choices: [{ message: { content: JSON.stringify(reply) } }] } }
|
|
268
|
-
: undefined,
|
|
282
|
+
otherwise: (request) => model.route(request),
|
|
269
283
|
});
|
|
270
284
|
}
|
|
271
285
|
|
|
@@ -37,14 +37,15 @@
|
|
|
37
37
|
* in React state — so a refresh lost it while the recording carried on being
|
|
38
38
|
* transcribed, summarized and spoken. The bytes were already stored, so the
|
|
39
39
|
* expensive half had happened; what an empty form invites is somebody uploading
|
|
40
|
-
* the recording again and paying for all four legs twice.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* that did
|
|
40
|
+
* the recording again and paying for all four legs twice. Both halves are the
|
|
41
|
+
* SDK's now and this page writes neither: `useWorkflowSubmit` records the run
|
|
42
|
+
* under a correlation KEY it keeps for this page and asks for it again on the
|
|
43
|
+
* next load, and picking the same file again sends only the windows that did
|
|
44
|
+
* not land.
|
|
44
45
|
*
|
|
45
|
-
* **
|
|
46
|
-
* same lifetime as
|
|
47
|
-
* promise. What
|
|
46
|
+
* **That key is `useRunKey()`'s** — opaque, and in `sessionStorage`, which is
|
|
47
|
+
* the same lifetime as the upload recall, so both halves of a reload make the
|
|
48
|
+
* same promise. What it rules out is a `?key=` parameter, and it is worth being
|
|
48
49
|
* plain about the trade because this template exists to produce something
|
|
49
50
|
* sendable: a summary you can listen to is exactly the sort of thing somebody
|
|
50
51
|
* forwards, and a URL is how people forward things. But there is no per-user
|
|
@@ -57,9 +58,9 @@
|
|
|
57
58
|
* the recording would be worse again: two people summarizing one file would
|
|
58
59
|
* recover each other's runs.
|
|
59
60
|
*
|
|
60
|
-
* A real app with accounts passes the account's own id instead, and
|
|
61
|
-
* summary follows the person to another device — a promise only a login
|
|
62
|
-
* keep.
|
|
61
|
+
* A real app with accounts passes the account's own id as `key` instead, and
|
|
62
|
+
* then a summary follows the person to another device — a promise only a login
|
|
63
|
+
* can keep.
|
|
63
64
|
*/
|
|
64
65
|
|
|
65
66
|
import "@alexkroman1/aai-ui/styles.css";
|
|
@@ -74,7 +75,6 @@ import {
|
|
|
74
75
|
SubmitButton,
|
|
75
76
|
UploadProgressBar,
|
|
76
77
|
useDownloadUrl,
|
|
77
|
-
useRunKey,
|
|
78
78
|
useWorkflowSubmit,
|
|
79
79
|
WorkflowFields,
|
|
80
80
|
WorkflowProgress,
|
|
@@ -128,20 +128,16 @@ function captionsUrl(text: string, durationMs: number): string {
|
|
|
128
128
|
}
|
|
129
129
|
|
|
130
130
|
export function App() {
|
|
131
|
-
// This tab's handle on its own summaries, in the store the upload recall uses
|
|
132
|
-
// — see the module doc for why a `?key=` is the wrong trade here.
|
|
133
|
-
const key = useRunKey();
|
|
134
131
|
// Did THIS load start the run? A reload cannot have, and that is the only way
|
|
135
132
|
// the page can tell "working on what you just sent" from "picking up where
|
|
136
133
|
// you left off" — the hook reports the run, not who asked for it.
|
|
137
134
|
const [startedHere, setStartedHere] = useState(false);
|
|
138
135
|
// The generic is what makes `run.status === "completed"` narrow to a TYPED
|
|
139
|
-
// `run.output` instead of `unknown`.
|
|
140
|
-
//
|
|
141
|
-
// `recover` the key is only ever written.
|
|
136
|
+
// `run.output` instead of `unknown`. The reload is the hook's own doing — see
|
|
137
|
+
// the module doc for why the key it mints is the right one for this page.
|
|
142
138
|
const { submitForm, run, pending, upload, pauseUpload, resumeUpload, error } = useWorkflowSubmit<
|
|
143
139
|
typeof spokenSummary
|
|
144
|
-
>(WORKFLOW, { api
|
|
140
|
+
>(WORKFLOW, { api });
|
|
145
141
|
const output = run?.status === "completed" ? run.output : undefined;
|
|
146
142
|
// `useDownloadUrl` is the SDK's: the byte route takes the agent's bearer, so the
|
|
147
143
|
// bytes have to be FETCHED and handed to the element as an object URL — and the
|
|
@@ -27,7 +27,13 @@
|
|
|
27
27
|
|
|
28
28
|
/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
|
|
29
29
|
import agentDef from "virtual:aai/agent";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
describeToolCalls,
|
|
32
|
+
describeTurn,
|
|
33
|
+
type EvalSession,
|
|
34
|
+
statesIn,
|
|
35
|
+
toolResultIn,
|
|
36
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
31
37
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
32
38
|
import { expect } from "vitest";
|
|
33
39
|
import { z } from "zod";
|
|
@@ -71,13 +77,14 @@ const Lookup = z.object({
|
|
|
71
77
|
error: z.string().optional(),
|
|
72
78
|
});
|
|
73
79
|
|
|
74
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Every `syncState` frame, in stream order.
|
|
82
|
+
*
|
|
83
|
+
* `statesIn` reads an EVENT LIST rather than a session, which is what lets a
|
|
84
|
+
* case slice the stream first; the schema above is what it takes one for.
|
|
85
|
+
*/
|
|
75
86
|
function frames(session: EvalSession) {
|
|
76
|
-
return session
|
|
77
|
-
.events()
|
|
78
|
-
.flatMap((event) =>
|
|
79
|
-
event.type === "state.updated" ? [ProjectedSupport.parse(event.state)] : [],
|
|
80
|
-
);
|
|
87
|
+
return statesIn(session.events(), ProjectedSupport);
|
|
81
88
|
}
|
|
82
89
|
|
|
83
90
|
/**
|
|
@@ -109,10 +116,10 @@ describeEval(agentDef, (test) => {
|
|
|
109
116
|
// The reply rides in the message, because the failure that matters here is
|
|
110
117
|
// a turn that SPOKE without looking anything up — "let me check that for
|
|
111
118
|
// you" and then nothing, or worse, a notice period from memory.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
).toBe(1);
|
|
119
|
+
// `describeTurn` is that sentence, and it says the two things a
|
|
120
|
+
// hand-built one left out: "called no tools" rather than an empty
|
|
121
|
+
// bracket, and whether the reply was cancelled.
|
|
122
|
+
expect(asked.length, describeTurn(turn)).toBe(1);
|
|
116
123
|
|
|
117
124
|
for (const payload of lookups(session)) {
|
|
118
125
|
// Three legal outcomes, and the invariant that spans them: an answer the
|
|
@@ -142,13 +149,9 @@ describeEval(agentDef, (test) => {
|
|
|
142
149
|
await session.say("Yes please, log that one — my callback number is 07700 900123.");
|
|
143
150
|
|
|
144
151
|
const logged = session.toolCalls().find((call) => call.name === "log_ticket");
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
.toolCalls()
|
|
149
|
-
.map((c) => c.name)
|
|
150
|
-
.join(", ")}`,
|
|
151
|
-
).toBeDefined();
|
|
152
|
+
// The claim spans both turns, so the message does too: `describeToolCalls`
|
|
153
|
+
// over the session's own list, where `describeTurn` would describe one.
|
|
154
|
+
expect(logged, describeToolCalls(session.toolCalls())).toBeDefined();
|
|
152
155
|
expect(logged?.result).toMatch(/TCK\d{4}/);
|
|
153
156
|
|
|
154
157
|
const latest = frames(session).at(-1);
|
|
@@ -189,10 +192,7 @@ describeEval(agentDef, (test) => {
|
|
|
189
192
|
const turn = await session.say("How much notice do I have to give to cancel my contract?");
|
|
190
193
|
|
|
191
194
|
const [payload] = lookups(session);
|
|
192
|
-
expect(
|
|
193
|
-
payload,
|
|
194
|
-
`tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
|
|
195
|
-
).toBeDefined();
|
|
195
|
+
expect(payload, describeTurn(turn)).toBeDefined();
|
|
196
196
|
// The whole verdict in the message: `grounded: undefined` on its own does
|
|
197
197
|
// not say whether the lookup failed, or ran and refused.
|
|
198
198
|
const verdict = JSON.stringify(payload);
|
|
@@ -238,10 +238,7 @@ describeEval(agentDef, (test) => {
|
|
|
238
238
|
);
|
|
239
239
|
|
|
240
240
|
const [payload] = lookups(session);
|
|
241
|
-
expect(
|
|
242
|
-
payload,
|
|
243
|
-
`tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
|
|
244
|
-
).toBeDefined();
|
|
241
|
+
expect(payload, describeTurn(turn)).toBeDefined();
|
|
245
242
|
// Withheld, not softened: `answer: null` is the tool refusing to hand the
|
|
246
243
|
// model something to read out, and the guidance is the exit the grading
|
|
247
244
|
// apparatus needs — a support line that can only answer will answer wrong.
|
|
@@ -143,15 +143,19 @@ function wavFile(
|
|
|
143
143
|
return head;
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
describe("the agent declares its three workflows
|
|
146
|
+
describe("the agent declares its three workflows", () => {
|
|
147
147
|
test("under the names the REST route resolves them by", () => {
|
|
148
148
|
// The page starts a run by these strings, so a rename is a runtime 400 rather
|
|
149
149
|
// than a compile error — which is what makes pinning them worth a test.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
150
|
+
// `arrayContaining` rather than an exact key list: a fourth desk of your own
|
|
151
|
+
// is an invited edit and must not redden a test you did not write. These
|
|
152
|
+
// three NAMES stay pinned, deliberately — the page starts a run by each
|
|
153
|
+
// string, so renaming one is a runtime 400 rather than a compile error, and
|
|
154
|
+
// this is the only thing that says so. Rename here and in `client.tsx`
|
|
155
|
+
// together.
|
|
156
|
+
expect(Object.keys(agentDef.workflows ?? {})).toEqual(
|
|
157
|
+
expect.arrayContaining(["transcribe", "transcribeStream", "transcribeBatch"]),
|
|
158
|
+
);
|
|
155
159
|
expect(agentDef.workflows?.transcribe).toBe(transcribe);
|
|
156
160
|
expect(agentDef.workflows?.transcribeStream).toBe(transcribeStream);
|
|
157
161
|
expect(agentDef.workflows?.transcribeBatch).toBe(transcribeBatch);
|
|
@@ -168,8 +172,12 @@ describe("the agent declares its three workflows and nothing else", () => {
|
|
|
168
172
|
});
|
|
169
173
|
|
|
170
174
|
test("with no tools, because the interface is the page and the API", () => {
|
|
171
|
-
//
|
|
172
|
-
//
|
|
175
|
+
// A PIN, deliberately, and the one assertion in this file you are expected
|
|
176
|
+
// to edit rather than satisfy. `workflowApp()` declares no session, so a
|
|
177
|
+
// tool has no conversation to be called from — a name appearing here means
|
|
178
|
+
// the voice path crept back in and the tool will never run. If you want a
|
|
179
|
+
// voice agent that STARTS a workflow, `research-workflow` is that template,
|
|
180
|
+
// and this line is what tells you you are on the wrong one.
|
|
173
181
|
expect(Object.keys(agentDef.tools ?? {})).toEqual([]);
|
|
174
182
|
});
|
|
175
183
|
|
|
@@ -779,6 +787,16 @@ describe("transcribeSegment", () => {
|
|
|
779
787
|
expect(decoded).toContain('name="audio"; filename="segment-0.wav"');
|
|
780
788
|
// The WAV really rides in the part, header and all.
|
|
781
789
|
expect(decoded).toContain("RIFF");
|
|
790
|
+
// And the header is CONTIGUOUS with its samples, which is what the two-chunk
|
|
791
|
+
// form (`[wavHeader(…), window]`) has to preserve and the only thing it could
|
|
792
|
+
// plausibly lose: the part's payload is exactly the 44 bytes plus the window,
|
|
793
|
+
// with nothing between them and nothing appended. A body that grew or shrank
|
|
794
|
+
// here is a file the endpoint decodes into confident nonsense rather than
|
|
795
|
+
// refusing.
|
|
796
|
+
const latin = new TextDecoder("latin1").decode(sent);
|
|
797
|
+
const from = latin.indexOf("RIFF");
|
|
798
|
+
const to = latin.lastIndexOf("\r\n--");
|
|
799
|
+
expect(to - from).toBe(44 + (SEGMENT.end - SEGMENT.start));
|
|
782
800
|
});
|
|
783
801
|
|
|
784
802
|
test("sends the DOWNSAMPLED window when the recording is heavier than 16 kHz mono", async () => {
|
|
@@ -153,15 +153,17 @@
|
|
|
153
153
|
* ## A reload keeps two of the three runs, and the third CANNOT be kept
|
|
154
154
|
*
|
|
155
155
|
* The run id lives in React state, so a refresh loses it while the fan-out
|
|
156
|
-
* carries on.
|
|
157
|
-
*
|
|
156
|
+
* carries on. A correlation KEY is the handle that survives that, and
|
|
157
|
+
* `useWorkflowSubmit` keeps one and asks for its newest run on mount — which
|
|
158
|
+
* here lands differently PER MODE:
|
|
158
159
|
*
|
|
159
160
|
* - **"After it uploads"** and **"Let the provider do it"** recover. Their input
|
|
160
161
|
* names a recording that is already stored, so a later load adopting the run
|
|
161
162
|
* is adopting something complete: the transcript arrives, the progress log
|
|
162
163
|
* replays, and nobody is asked to send a 600 MB file a second time.
|
|
163
164
|
* - **"While it uploads" does not, and the hook REFUSES the option rather than
|
|
164
|
-
* ignoring it
|
|
165
|
+
* ignoring it** — `useWorkflowStream` omits `recover` from its options type
|
|
166
|
+
* and mints no key. That run's input names an upload id this page load minted and
|
|
165
167
|
* is still filling, so a later load could only adopt a run waiting for bytes
|
|
166
168
|
* nobody is sending — and it is worse than useless: `workflows/stream.ts`
|
|
167
169
|
* fails a run whose upload stops growing (`MAX_IDLE_POLLS`), so the reload
|
|
@@ -171,10 +173,11 @@
|
|
|
171
173
|
* The MODE is remembered too, and that is not decoration: without it a reload
|
|
172
174
|
* opens on the default flow while the recovered run sits behind a radio nobody
|
|
173
175
|
* pressed, so the reader sees an empty form and starts a second run — the exact
|
|
174
|
-
* thing the key exists to prevent. The KEY is
|
|
175
|
-
* minting, the storage and the argument for
|
|
176
|
-
* `?key=` parameter
|
|
177
|
-
*
|
|
176
|
+
* thing the key exists to prevent. The KEY is the hook's own — `use-run-key.ts`
|
|
177
|
+
* owns the minting, the storage and the argument for it being opaque rather
|
|
178
|
+
* than a `?key=` parameter, and the two submit hooks here share one because the
|
|
179
|
+
* slot is keyed by the PAGE; `recover.ts` owns the mode, which is this page's
|
|
180
|
+
* own concept, and the validation on the way back out of storage that turning a
|
|
178
181
|
* stored string into a workflow name obliges.
|
|
179
182
|
*
|
|
180
183
|
* Two smaller consequences worth knowing. Both recovering hooks look up on
|
|
@@ -193,7 +196,6 @@ import {
|
|
|
193
196
|
page,
|
|
194
197
|
SubmitButton,
|
|
195
198
|
UploadProgressBar,
|
|
196
|
-
useRunKey,
|
|
197
199
|
useWorkflowRuns,
|
|
198
200
|
useWorkflowStream,
|
|
199
201
|
useWorkflowSubmit,
|
|
@@ -268,9 +270,6 @@ function TranscriptionDesk() {
|
|
|
268
270
|
// piece of state for all three hooks, because it describes the UPLOAD and every
|
|
269
271
|
// mode has one — see the module doc.
|
|
270
272
|
const [parallel, setParallel] = useState(true);
|
|
271
|
-
// This tab's handle on its own runs — minted once and remembered, which is
|
|
272
|
-
// what a later load produces to find the run again.
|
|
273
|
-
const key = useRunKey();
|
|
274
273
|
// Did THIS load press Transcribe? A reload cannot have, and it is the only way
|
|
275
274
|
// the page can tell "working on what you just sent" from "picking up where you
|
|
276
275
|
// left off" — the hooks report the run, not who asked for it.
|
|
@@ -279,22 +278,16 @@ function TranscriptionDesk() {
|
|
|
279
278
|
// and that costs nothing here: none of them does anything until its `submit` is
|
|
280
279
|
// called, and `useWorkflowRun` underneath them holds no id until then either.
|
|
281
280
|
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
281
|
+
// The two submit hooks recover unconditionally rather than on `mode === …`:
|
|
282
|
+
// the lookup is a MOUNT-time act, so arming it when a mode is picked would
|
|
283
|
+
// re-adopt a run the reader had just cleared. They share ONE key without
|
|
284
|
+
// saying so — the hook's slot is keyed by the page — and `find` is scoped by
|
|
285
|
+
// workflow as well as by key, so the two modes recover separate runs. The
|
|
286
|
+
// streaming hook has neither half: it refuses `recover` by type, and mints no
|
|
287
|
+
// key it would never read back.
|
|
287
288
|
const streamed = useWorkflowStream<typeof transcribe>(WORKFLOWS.streaming, { parallel });
|
|
288
|
-
const stored = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.classic, {
|
|
289
|
-
|
|
290
|
-
key,
|
|
291
|
-
recover: true,
|
|
292
|
-
});
|
|
293
|
-
const batched = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.batch, {
|
|
294
|
-
parallel,
|
|
295
|
-
key,
|
|
296
|
-
recover: true,
|
|
297
|
-
});
|
|
289
|
+
const stored = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.classic, { parallel });
|
|
290
|
+
const batched = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.batch, { parallel });
|
|
298
291
|
// The batch flow uploads the same way the classic one does — the id comes from the
|
|
299
292
|
// store — so it is the SAME hook against a different workflow. Only the streaming
|
|
300
293
|
// mode needs the other one, because only it needs the id before the bytes.
|
|
@@ -159,7 +159,16 @@ export async function normalizeRecording(uploadId: string): Promise<NormalizedRe
|
|
|
159
159
|
const source = join(dir, "source");
|
|
160
160
|
const converted = join(dir, "converted.wav");
|
|
161
161
|
|
|
162
|
-
|
|
162
|
+
// NO `size`, though `stored.size` is right there — and that is the whole
|
|
163
|
+
// difference between this copy being one window at a time and being
|
|
164
|
+
// `STEP_FILE_READ_CONCURRENCY` of them. Passing `size` means "I am judging
|
|
165
|
+
// completeness myself", which is what a body polling a still-arriving
|
|
166
|
+
// upload needs and is the opposite of what happened above: this step has
|
|
167
|
+
// already called `requireCompleteUpload`, so the file IS whole and the
|
|
168
|
+
// windows may land in any order. Omitting it lets `readUploadToFile`
|
|
169
|
+
// establish that for itself and fan out. The cost is one metadata round
|
|
170
|
+
// trip, against the dozens of window reads it overlaps.
|
|
171
|
+
await readUploadToFile(uploadId, source);
|
|
163
172
|
|
|
164
173
|
// What it WAS, for the progress line. Worth one ffprobe: "converted 41
|
|
165
174
|
// minutes of aac" is a line that explains the run's shape, where
|
|
@@ -52,7 +52,10 @@ export function elapsed(ms: number): string {
|
|
|
52
52
|
* `bytes` must be a whole file, header included — the endpoint decodes each
|
|
53
53
|
* request independently, so a headerless tail is bytes it will refuse. Both
|
|
54
54
|
* callers arrive at that differently: one re-attaches a header to a window it
|
|
55
|
-
* read, the other is handed parts that already carry one.
|
|
55
|
+
* read, the other is handed parts that already carry one. A LIST is a whole
|
|
56
|
+
* file too: the segment caller passes `[wavHeader(...), window]` so the two are
|
|
57
|
+
* concatenated straight into the request body rather than into an intermediate
|
|
58
|
+
* buffer that doubles the segment's footprint.
|
|
56
59
|
*
|
|
57
60
|
* `stepTranscribeSyncClassified` — the SDK's own `stepTranscribeSync` plus
|
|
58
61
|
* `throwStepError`, and nothing else — is the whole of what this adds to the SDK
|
|
@@ -68,7 +71,7 @@ export function elapsed(ms: number): string {
|
|
|
68
71
|
* log has in front of them.
|
|
69
72
|
*/
|
|
70
73
|
export async function transcribeWav(
|
|
71
|
-
bytes: Uint8Array,
|
|
74
|
+
bytes: Uint8Array | readonly Uint8Array[],
|
|
72
75
|
filename: string,
|
|
73
76
|
label: string,
|
|
74
77
|
): Promise<string> {
|
|
@@ -66,12 +66,12 @@
|
|
|
66
66
|
import type { WorkflowCtx } from "@alexkroman1/aai";
|
|
67
67
|
import {
|
|
68
68
|
emit,
|
|
69
|
-
encodeWav,
|
|
70
69
|
mapConcurrent,
|
|
71
70
|
readUpload,
|
|
72
71
|
report,
|
|
73
72
|
requireCompleteUpload,
|
|
74
73
|
uploadInfo,
|
|
74
|
+
wavHeader,
|
|
75
75
|
} from "@alexkroman1/aai/step";
|
|
76
76
|
import { throwFatalStepError } from "@alexkroman1/aai/step-errors";
|
|
77
77
|
import { countWords, formatDuration, plural } from "@alexkroman1/aai/utils";
|
|
@@ -358,13 +358,20 @@ export async function transcribeSegment(
|
|
|
358
358
|
// answers better — and getting it wrong is a whole transcript in the wrong
|
|
359
359
|
// language. Add one back only for a desk that really knows.
|
|
360
360
|
//
|
|
361
|
-
//
|
|
361
|
+
// A HEADER is what makes a WINDOW decodable: the endpoint decodes each
|
|
362
362
|
// request independently, so a slice of the middle of a recording is a headerless
|
|
363
363
|
// tail until one is put back on it. The streaming flow needs no equivalent — its
|
|
364
364
|
// parts were cut with a header each. The header is the SDK's rather than this
|
|
365
365
|
// template's: a `WavFormat` is structurally a `PcmFormat`, and 22 lines of
|
|
366
366
|
// `DataView` writes with a comment about which of the two declared lengths a
|
|
367
367
|
// decoder trusts is not a thing worth a second copy of.
|
|
368
|
+
// It goes down as its own CHUNK rather than through `encodeWav`, which is a
|
|
369
|
+
// MEMORY decision and not a speed one: `encodeWav` allocates `44 + N` and
|
|
370
|
+
// copies the segment into it, and `multipartBody` then allocates the body and
|
|
371
|
+
// copies that again — so the audio was resident three times at the moment the
|
|
372
|
+
// request went out, on a fan-out whose width is set by exactly that peak (see
|
|
373
|
+
// `MAX_SEGMENT_CONCURRENCY`). Header and samples are contiguous on the wire
|
|
374
|
+
// either way; this holds `44 + N` once, ~3 MB per in-flight 16 kHz segment.
|
|
368
375
|
// Down to 16 kHz mono BEFORE the header goes on, because the endpoint's budget
|
|
369
376
|
// is 30 seconds of wall clock and that covers the upload. At 48 kHz stereo this
|
|
370
377
|
// window is 17.66 MB and the same audio is 2.94 MB normalized — six times the
|
|
@@ -382,12 +389,12 @@ export async function transcribeSegment(
|
|
|
382
389
|
// identical answer. BOTH flows can reach it, which is newer than it looks:
|
|
383
390
|
// the check used to hang off the resampler, so a 12-bit recording already at
|
|
384
391
|
// 16 kHz mono — light for both flows, and therefore converted by neither —
|
|
385
|
-
// sailed past it into an unclassified `RangeError` from
|
|
392
|
+
// sailed past it into an unclassified `RangeError` from the header writer.
|
|
386
393
|
const light = fatalOnUnsupported(() => downsampleSegment(audio.bytes, format));
|
|
387
394
|
|
|
388
395
|
const { value: text, ms } = await timed(() =>
|
|
389
396
|
transcribeWav(
|
|
390
|
-
|
|
397
|
+
[wavHeader(light.format, light.bytes.byteLength), light.bytes],
|
|
391
398
|
`segment-${segment.index}.wav`,
|
|
392
399
|
`Segment ${segment.index} (${formatDuration(segment.startMs)})`,
|
|
393
400
|
),
|
|
@@ -29,7 +29,15 @@
|
|
|
29
29
|
* tools and read as a model that refuses to act.
|
|
30
30
|
*/
|
|
31
31
|
import agentDef from "virtual:aai/agent";
|
|
32
|
-
import {
|
|
32
|
+
import {
|
|
33
|
+
callsIn,
|
|
34
|
+
describeTurn,
|
|
35
|
+
type EvalSession,
|
|
36
|
+
type EvalToolCall,
|
|
37
|
+
lastStateIn,
|
|
38
|
+
toolNames,
|
|
39
|
+
turnCalling,
|
|
40
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
33
41
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
34
42
|
import { expect } from "vitest";
|
|
35
43
|
import { z } from "zod";
|
|
@@ -76,33 +84,17 @@ function framesBeforeConfirm(session: EvalSession): z.infer<typeof ProjectedTrip
|
|
|
76
84
|
const tripState = (session: EvalSession) => lastStateIn(session.events(), ProjectedTrip);
|
|
77
85
|
|
|
78
86
|
/**
|
|
79
|
-
*
|
|
87
|
+
* A call that really STAGED — it answered with the read-back rather than with a
|
|
88
|
+
* gate's refusal.
|
|
80
89
|
*
|
|
81
|
-
*
|
|
82
|
-
* turn one
|
|
83
|
-
* business and it moved when the desk gate landed
|
|
84
|
-
* to search before quoting anything, so measured
|
|
85
|
-
* its first turn on `to_flight_assistant` and
|
|
86
|
-
* fare back before it stages.
|
|
87
|
-
* misleading name — the same argument `retail`'s eval carries.
|
|
90
|
+
* Passed to `turnCalling` as its `where`, so the turn a case reads is the one
|
|
91
|
+
* the MECHANISM fired in rather than turn one: how many turns a desk spends
|
|
92
|
+
* getting there is the model's business and it moved when the desk gate landed
|
|
93
|
+
* — the flight desk's brief says to search before quoting anything, so measured
|
|
94
|
+
* live this concierge spends its first turn on `to_flight_assistant` and
|
|
95
|
+
* `search_flights` and reads the fare back before it stages.
|
|
88
96
|
*/
|
|
89
|
-
|
|
90
|
-
const turns: EvalTurn[] = [];
|
|
91
|
-
for (const line of lines) turns.push(await session.say(line));
|
|
92
|
-
return turns;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/** Every tool call of the call so far, flattened, in order. */
|
|
96
|
-
const callsIn = (turns: readonly EvalTurn[]) => turns.flatMap((turn) => turn.toolCalls);
|
|
97
|
-
|
|
98
|
-
/** The turn a named tool STAGED something in — the call that answered with the
|
|
99
|
-
* read-back rather than with a gate's refusal. */
|
|
100
|
-
const stagingTurn = (turns: readonly EvalTurn[], tool: string) =>
|
|
101
|
-
turns.find((turn) =>
|
|
102
|
-
turn.toolCalls.some(
|
|
103
|
-
(call) => call.name === tool && /awaitingConfirmation/.test(call.result ?? ""),
|
|
104
|
-
),
|
|
105
|
-
);
|
|
97
|
+
const stagedSomething = (call: EvalToolCall) => /awaitingConfirmation/.test(call.result ?? "");
|
|
106
98
|
|
|
107
99
|
describeEval(agentDef, (test) => {
|
|
108
100
|
test(
|
|
@@ -112,28 +104,25 @@ describeEval(agentDef, (test) => {
|
|
|
112
104
|
// a read-back: what is asserted below is that the turn which staged did
|
|
113
105
|
// not also apply, so a line the model could read as consent ("correct",
|
|
114
106
|
// "that's right") would be measuring the caller instead of the desk.
|
|
115
|
-
const turns = await sayAll(
|
|
107
|
+
const turns = await session.sayAll([
|
|
116
108
|
"Move my ticket to flight LX52, the Wednesday one.",
|
|
117
109
|
"I want the Wednesday LX52 instead of the flight I'm on now.",
|
|
118
110
|
"Put me on LX52 on Wednesday, please.",
|
|
119
111
|
]);
|
|
120
112
|
|
|
121
|
-
|
|
113
|
+
// A desk that talks its way through three turns without staging fails
|
|
114
|
+
// HERE — the failure this case caught while the flight desk's brief had
|
|
115
|
+
// the read-back before the staging — and `turnCalling`'s throw names
|
|
116
|
+
// every turn's tool list AND tells the two findings apart: no
|
|
117
|
+
// `update_ticket` at all, or calls that were all refused by the desk gate.
|
|
118
|
+
const staging = turnCalling(turns, "update_ticket", stagedSomething);
|
|
122
119
|
const attempts = callsIn(turns).filter((call) => call.name === "update_ticket");
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
turns
|
|
130
|
-
.map(
|
|
131
|
-
(turn, i) =>
|
|
132
|
-
`turn ${i + 1}: [${turn.toolCalls.map((c) => c.name).join(", ")}] said: ${turn.text}`,
|
|
133
|
-
)
|
|
134
|
-
.join("\n"),
|
|
135
|
-
).toBeDefined();
|
|
136
|
-
const staged = staging?.toolCalls.find((call) => call.name === "update_ticket");
|
|
120
|
+
// The staging call itself, by INDEX, because what follows it in the same
|
|
121
|
+
// turn is the subject of the assertion below.
|
|
122
|
+
const stagedAt = staging.toolCalls.findIndex(
|
|
123
|
+
(call) => call.name === "update_ticket" && stagedSomething(call),
|
|
124
|
+
);
|
|
125
|
+
const staged = staging.toolCalls[stagedAt];
|
|
137
126
|
// The tool answered with the read-back rather than with a receipt.
|
|
138
127
|
expect(staged?.result).toMatch(/awaitingConfirmation/);
|
|
139
128
|
// Any attempt that did NOT stage is the DESK GATE refusing:
|
|
@@ -154,14 +143,7 @@ describeEval(agentDef, (test) => {
|
|
|
154
143
|
// refusal is the subject), and it then stages properly. That is a wasted
|
|
155
144
|
// step rather than an unasked-for change, and folding the two together
|
|
156
145
|
// would fail this case for the behaviour the next one proves is safe.
|
|
157
|
-
|
|
158
|
-
// rewrite Biome offers for `findIndex` over an identity is UNSAFE for
|
|
159
|
-
// exactly that reason, and the assertion above is what makes an absent
|
|
160
|
-
// staging call a failure rather than a slice from 0.
|
|
161
|
-
const stagedAt = staged === undefined ? -1 : (staging?.toolCalls.indexOf(staged) ?? -1);
|
|
162
|
-
expect(staging?.toolCalls.slice(stagedAt + 1).map((call) => call.name) ?? []).not.toContain(
|
|
163
|
-
"confirm_action",
|
|
164
|
-
);
|
|
146
|
+
expect(toolNames(staging.toolCalls.slice(stagedAt + 1))).not.toContain("confirm_action");
|
|
165
147
|
|
|
166
148
|
const views = framesBeforeConfirm(session);
|
|
167
149
|
const waiting = views.filter((view) => view.pending !== null);
|
|
@@ -212,7 +194,7 @@ describeEval(agentDef, (test) => {
|
|
|
212
194
|
// it applies in is its own business — the flight desk's brief has it
|
|
213
195
|
// search first — and saying yes repeatedly is what makes "once each"
|
|
214
196
|
// below a claim about the MECHANISM rather than about the model's pacing.
|
|
215
|
-
await sayAll(
|
|
197
|
+
await session.sayAll([
|
|
216
198
|
"Move my ticket to flight LX52, the Wednesday one.",
|
|
217
199
|
"Correct — LX52 on Wednesday. Please move my ticket to it.",
|
|
218
200
|
"Yes, that's right — go ahead and change it.",
|
|
@@ -231,7 +213,7 @@ describeEval(agentDef, (test) => {
|
|
|
231
213
|
);
|
|
232
214
|
// Staged first, applied second, once each. Reversed — or a confirm with no
|
|
233
215
|
// stage — is the regression this template's whole shape exists to prevent.
|
|
234
|
-
expect(effective
|
|
216
|
+
expect(toolNames(effective)).toEqual(["update_ticket", "confirm_action"]);
|
|
235
217
|
// Everything else has to be a GATE refusing, and nothing else: the desk
|
|
236
218
|
// gate turns away an `update_ticket` issued before
|
|
237
219
|
// `to_flight_assistant`, and the confirmation gate turns away a
|
|
@@ -277,10 +259,9 @@ describeEval(agentDef, (test) => {
|
|
|
277
259
|
// below and `indexOf` on a possibly-undefined find is worse than both.
|
|
278
260
|
const handoffAt = turn.toolCalls.findIndex((call) => call.name === "to_hotel_assistant");
|
|
279
261
|
const handoff = turn.toolCalls[handoffAt];
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
).toBeDefined();
|
|
262
|
+
// `describeTurn` is the message a bare `toBeDefined()` failure leaves
|
|
263
|
+
// out: what this turn reached for, and what it said instead.
|
|
264
|
+
expect(handoff, describeTurn(turn)).toBeDefined();
|
|
284
265
|
// The brief IS the tool result, which is the whole port of their
|
|
285
266
|
// per-assistant prompt onto a session whose prompt is fixed at connect.
|
|
286
267
|
expect(handoff?.result).toMatch(/hotel desk/);
|