@alexkroman1/aai-cli 10.0.0 → 11.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_bundler-B31MqYaD.mjs → _bundler-C31rlcYz.mjs} +1 -1
- package/dist/{_dev-server-Bo5dSBzQ.mjs → _dev-server-DWvkNoIS.mjs} +3 -3
- package/dist/{_vite-env-BNveawd1.mjs → _vite-env-D9Qj6KPX.mjs} +4 -2
- package/dist/_workflow-determinism-f9J-5Zlj.mjs +206 -0
- package/dist/_workflow-determinism.d.ts +118 -0
- package/dist/{build-DKC2e9o_.mjs → build-BLlLOrng.mjs} +4 -2
- package/dist/cli.mjs +372 -353
- package/dist/{client-bundler-CWnG42cU.mjs → client-bundler-j5c46x0G.mjs} +3 -2
- package/dist/client-bundler.mjs +1 -1
- package/dist/{deploy-CchW5pmw.mjs → deploy-BzZ-r4iM.mjs} +6 -4
- package/dist/{dev-BfVl1uBA.mjs → dev-BDykFvcc.mjs} +1 -1
- package/dist/{eval-DwNwdqmS.mjs → eval-0qEOdm0l.mjs} +1 -1
- package/dist/{init-BRfoc2EK.mjs → init-BtJI306C.mjs} +1 -1
- package/dist/{login-BBuM1sxH.mjs → login-DeUETobb.mjs} +4 -1
- package/dist/scaffold/CLAUDE.md +190 -11
- package/dist/scaffold/package.json +4 -4
- package/dist/{studio-wVWNLREn.mjs → studio-BxAS_FQQ.mjs} +4 -1
- package/dist/templates/call-audit/agent.test.ts +105 -26
- package/dist/templates/call-audit/workflows/audit.ts +9 -26
- package/dist/templates/dispatch-center/client.tsx +36 -2
- package/dist/templates/infocom-adventure/client.tsx +37 -9
- package/dist/templates/link-digest/agent.eval.test.ts +1 -1
- package/dist/templates/link-digest/agent.test.ts +164 -8
- package/dist/templates/link-digest/workflows/digest.ts +63 -20
- package/dist/templates/podcast-digest/agent.eval.test.ts +7 -4
- package/dist/templates/podcast-digest/agent.test.ts +265 -24
- package/dist/templates/podcast-digest/workflows/digest.ts +2 -2
- package/dist/templates/podcast-digest/workflows/feeds.ts +78 -73
- package/dist/templates/recap-workflow/agent.test.ts +206 -20
- package/dist/templates/recap-workflow/workflows/recap.ts +17 -14
- package/dist/templates/redline/agent.test.ts +137 -11
- package/dist/templates/research-workflow/agent.eval.test.ts +1 -1
- package/dist/templates/research-workflow/agent.test.ts +131 -9
- package/dist/templates/research-workflow/workflows/research.ts +1 -1
- package/dist/templates/retail/client.tsx +30 -2
- package/dist/templates/spoken-summary/agent.eval.test.ts +4 -1
- package/dist/templates/spoken-summary/agent.test.ts +117 -8
- package/dist/templates/spoken-summary/workflows/summarize.ts +1 -1
- package/dist/templates/transcription-workflow/agent.test.ts +127 -13
- package/dist/templates/transcription-workflow/workflows/batch.ts +6 -6
- package/dist/templates/transcription-workflow/workflows/stream.ts +4 -3
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +4 -23
- package/dist/{test-CvwgeVSQ.mjs → test-CiLab-AA.mjs} +2 -1
- package/dist/worker-bundler.mjs +1 -1
- package/package.json +25 -24
|
@@ -4,20 +4,26 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Specs for the research desk's four tools.
|
|
6
6
|
*
|
|
7
|
-
* All are exercised against a STUBBED `ctx.workflows`, which is the
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* All are exercised against a STUBBED `ctx.workflows`, which is the honest way
|
|
8
|
+
* to unit-test a TOOL: what a tool owns is the call it makes, not what the run
|
|
9
|
+
* does afterwards.
|
|
10
10
|
* What these assert is the agent's half of the contract — that the handoff tool
|
|
11
11
|
* passes the correlation key, that the status tool narrows a snapshot correctly
|
|
12
12
|
* before reading it aloud, and that the two tools reaching PAST a status (the
|
|
13
13
|
* progress stream, the early wake) ask for what a voice reply can use.
|
|
14
14
|
*
|
|
15
|
-
* The STEPS are exercised separately, and directly:
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
15
|
+
* The STEPS are exercised separately, and directly: a step is an ordinary
|
|
16
|
+
* exported async function, so its prompt handling, its parsing and its
|
|
17
|
+
* `FatalError` guards are all testable without an engine.
|
|
18
|
+
*
|
|
19
|
+
* The BODY is driven here only through `createWorkflowCtx`, which records what
|
|
20
|
+
* it asked for and replays nothing. That is a choice rather than a limit now:
|
|
21
|
+
* `runWorkflow` from `@alexkroman1/aai-runtime/testing` will run this body on
|
|
22
|
+
* the real engine, and `link-digest` is the template that shows it — three
|
|
23
|
+
* steps and one suspension, where this desk's body is six model steps deep and a
|
|
24
|
+
* durable spec of it would be mostly stubs. `aai-cli`'s
|
|
25
|
+
* `dev-workflow.scenario.test.ts` is the tier above both, with a built project
|
|
26
|
+
* and a real queue.
|
|
21
27
|
*/
|
|
22
28
|
|
|
23
29
|
import type { WorkflowClient } from "@alexkroman1/aai";
|
|
@@ -34,6 +40,7 @@ import {
|
|
|
34
40
|
import { mockWorkflows, installStubGateway as stubGateway } from "@alexkroman1/aai/testing/vitest";
|
|
35
41
|
import { visitWebpage, webSearch } from "@alexkroman1/aai/tools";
|
|
36
42
|
import type { WorkflowRunSnapshot } from "@alexkroman1/aai/workflow-api";
|
|
43
|
+
import { runWorkflow } from "@alexkroman1/aai-runtime/testing";
|
|
37
44
|
import { beforeEach, describe, expect, test, vi } from "vitest";
|
|
38
45
|
import { research } from "./shared.ts";
|
|
39
46
|
import {
|
|
@@ -42,6 +49,7 @@ import {
|
|
|
42
49
|
findGaps,
|
|
43
50
|
investigate,
|
|
44
51
|
planAngles,
|
|
52
|
+
REVIEW_DELAY_MS,
|
|
45
53
|
researchFlow,
|
|
46
54
|
writeBrief,
|
|
47
55
|
writeReport,
|
|
@@ -538,3 +546,117 @@ describe("the steps that research", () => {
|
|
|
538
546
|
).rejects.toThrow(/empty completion/);
|
|
539
547
|
});
|
|
540
548
|
});
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* `researchFlow` itself, on the real replay engine.
|
|
552
|
+
*
|
|
553
|
+
* The block above drives this body through `createWorkflowCtx`, which records
|
|
554
|
+
* what it ASKED for and replays nothing — right for the retry policy and the
|
|
555
|
+
* step order, and silent about the desk's actual promise: **answer the caller
|
|
556
|
+
* now, finish the work later**. `runWorkflow`
|
|
557
|
+
* (`@alexkroman1/aai-runtime/testing`) is the engine `aai dev` runs, over an
|
|
558
|
+
* in-memory journal, so the review wait really suspends and the resume really
|
|
559
|
+
* comes off the journal.
|
|
560
|
+
*
|
|
561
|
+
* The model is scripted POSITIONALLY, which is only safe because the run is made
|
|
562
|
+
* sequential: one angle, and a researcher that stops on its first turn, so the
|
|
563
|
+
* `mapConcurrent` fan-out has a single item and nothing races. A case that wants
|
|
564
|
+
* two angles at once wants a router keyed on the system prompt instead — the
|
|
565
|
+
* shape `link-digest` uses — because the order two concurrent step bodies reach
|
|
566
|
+
* the gateway in is the scheduler's business.
|
|
567
|
+
*
|
|
568
|
+
* Five calls make a whole run: the brief, the angles, the researcher's first
|
|
569
|
+
* action, the gap pass, and the report — plus its summary, which is a second
|
|
570
|
+
* call on the same step.
|
|
571
|
+
*/
|
|
572
|
+
describe("the run is DURABLE", () => {
|
|
573
|
+
const SCRIPT = [
|
|
574
|
+
// writeBrief
|
|
575
|
+
JSON.stringify({ brief: "How otters use tools", criteria: ["Which species"] }),
|
|
576
|
+
// planAngles — ONE, so the fan-out is sequential and the script positional.
|
|
577
|
+
JSON.stringify({ angles: ["Tool use"] }),
|
|
578
|
+
// investigate#0's first action: stop, which also skips `compress` (nothing
|
|
579
|
+
// was seen, so there is nothing to compress).
|
|
580
|
+
JSON.stringify({ action: "stop", why: "nothing to add" }),
|
|
581
|
+
// findGaps — none, so there is no second wave.
|
|
582
|
+
JSON.stringify({ angles: [] }),
|
|
583
|
+
// writeReport, then its summary.
|
|
584
|
+
"The report about otters.",
|
|
585
|
+
"Otters use tools.",
|
|
586
|
+
];
|
|
587
|
+
const INPUT = { topic: "otters", requestedBy: "sess_1" };
|
|
588
|
+
|
|
589
|
+
beforeEach(() => {
|
|
590
|
+
vi.stubEnv("ASSEMBLYAI_API_KEY", "sk-test");
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
test("suspends on the review wait with the whole report already journaled", async () => {
|
|
594
|
+
const started = Date.now();
|
|
595
|
+
const model = stubGateway(SCRIPT);
|
|
596
|
+
const run = await runWorkflow(research, INPUT, { name: "research" });
|
|
597
|
+
|
|
598
|
+
// Not blocked — suspended. The sandbox is free here, which is the whole
|
|
599
|
+
// reason a caller can hang up.
|
|
600
|
+
expect(run.status).toBe("running");
|
|
601
|
+
expect(run.wakeAt).toBeGreaterThanOrEqual(started + REVIEW_DELAY_MS);
|
|
602
|
+
// Everything except the filing is durable already, and `file` has not run.
|
|
603
|
+
expect(run.steps.map((step) => step.key)).toEqual([
|
|
604
|
+
"findGaps#0",
|
|
605
|
+
"investigate#0",
|
|
606
|
+
"planAngles#0",
|
|
607
|
+
"writeBrief#0",
|
|
608
|
+
"writeReport#0",
|
|
609
|
+
]);
|
|
610
|
+
expect(model).toHaveLength(6);
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
test("resumes past the review wait and files, without researching again", async () => {
|
|
614
|
+
const model = stubGateway(SCRIPT);
|
|
615
|
+
const run = await runWorkflow(research, INPUT, { name: "research" });
|
|
616
|
+
// `advanceSleep` is `ctx.workflows.wakeUp`'s own mechanism, which is what
|
|
617
|
+
// the `file_it_now` tool calls to cut the review short — so this is that
|
|
618
|
+
// tool's effect, asserted on the run rather than on the tool.
|
|
619
|
+
await run.advanceSleep();
|
|
620
|
+
|
|
621
|
+
expect(run.status).toBe("completed");
|
|
622
|
+
expect(run.output).toMatchObject({
|
|
623
|
+
topic: "otters",
|
|
624
|
+
summary: "Otters use tools.",
|
|
625
|
+
report: "The report about otters.",
|
|
626
|
+
angles: ["Tool use"],
|
|
627
|
+
});
|
|
628
|
+
expect(run.output?.filedAt).toBeTruthy();
|
|
629
|
+
expect(run.deliveries).toBe(2);
|
|
630
|
+
// The second walk re-entered the body from the top and paid the model
|
|
631
|
+
// NOTHING: every step above the wait came back out of the journal.
|
|
632
|
+
expect(model).toHaveLength(6);
|
|
633
|
+
});
|
|
634
|
+
|
|
635
|
+
test("a worker that dies at the report replays the research rather than repeating it", async () => {
|
|
636
|
+
// The expensive claim. A deep-research pass is five to twelve model calls
|
|
637
|
+
// and as many searches; a resume that redid them would cost the run twice.
|
|
638
|
+
const model = stubGateway(SCRIPT);
|
|
639
|
+
const run = await runWorkflow(research, INPUT, {
|
|
640
|
+
name: "research",
|
|
641
|
+
crashAt: "writeReport",
|
|
642
|
+
});
|
|
643
|
+
|
|
644
|
+
expect(run.crashed).toBe(true);
|
|
645
|
+
expect(run.steps.map((step) => step.key)).toEqual([
|
|
646
|
+
"findGaps#0",
|
|
647
|
+
"investigate#0",
|
|
648
|
+
"planAngles#0",
|
|
649
|
+
"writeBrief#0",
|
|
650
|
+
]);
|
|
651
|
+
const spentBeforeTheCrash = model.length;
|
|
652
|
+
expect(spentBeforeTheCrash).toBe(4);
|
|
653
|
+
|
|
654
|
+
await run.restart();
|
|
655
|
+
await run.advanceSleep();
|
|
656
|
+
expect(run.status).toBe("completed");
|
|
657
|
+
// Six in total: the four the crash already paid for came back out of the
|
|
658
|
+
// journal, and only the report and its summary were re-issued.
|
|
659
|
+
expect(model).toHaveLength(6);
|
|
660
|
+
expect(run.output?.report).toBe("The report about otters.");
|
|
661
|
+
});
|
|
662
|
+
});
|
|
@@ -246,7 +246,7 @@ export async function researchFlow(
|
|
|
246
246
|
// Suspended, not blocked. On resume the body re-runs from the top and every
|
|
247
247
|
// step above returns its journaled result rather than researching again —
|
|
248
248
|
// which is also what `file_it_now` ends early, through `ctx.workflows.wakeUp`.
|
|
249
|
-
await ctx.sleep(REVIEW_DELAY_MS);
|
|
249
|
+
await ctx.sleep("reviewWindow", REVIEW_DELAY_MS);
|
|
250
250
|
|
|
251
251
|
// Whatever this returns is what `ctx.workflows.get(runId)` reports as `output`
|
|
252
252
|
// on a completed run — so it is what the agent reads back, and what the
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import "@alexkroman1/aai-ui/styles.css";
|
|
2
|
-
import type { AgentState, ConversationItem } from "@alexkroman1/aai-ui";
|
|
2
|
+
import type { AgentState, ConversationItem, Session } from "@alexkroman1/aai-ui";
|
|
3
3
|
import {
|
|
4
4
|
AutoScroll,
|
|
5
5
|
client,
|
|
@@ -417,8 +417,21 @@ function ErrorBanner() {
|
|
|
417
417
|
);
|
|
418
418
|
}
|
|
419
419
|
|
|
420
|
+
/**
|
|
421
|
+
* Start a fresh conversation without leaving the console.
|
|
422
|
+
*
|
|
423
|
+
* Written out here rather than reached for on the session, because there is no
|
|
424
|
+
* one method that does it: `reset()` clears the CONVERSATION and keeps the
|
|
425
|
+
* session, which is wrong for any agent that also keeps session-scoped state —
|
|
426
|
+
* this one's store would come back with the next tool call.
|
|
427
|
+
*/
|
|
428
|
+
function newConversation(session: Session): void {
|
|
429
|
+
session.end();
|
|
430
|
+
session.start();
|
|
431
|
+
}
|
|
432
|
+
|
|
420
433
|
/** The call controls: the one place a whole-session read is what is wanted —
|
|
421
|
-
* `started`, `running` and
|
|
434
|
+
* `started`, `running` and four methods, in four buttons. */
|
|
422
435
|
function CallControls({ productCount }: { productCount: number }) {
|
|
423
436
|
const session = useSession();
|
|
424
437
|
return (
|
|
@@ -448,6 +461,21 @@ function CallControls({ productCount }: { productCount: number }) {
|
|
|
448
461
|
>
|
|
449
462
|
{session.running ? "Hold" : "Resume"}
|
|
450
463
|
</button>
|
|
464
|
+
{/* The one-click new conversation the default shell's
|
|
465
|
+
`<Controls>` gives every other template — a custom
|
|
466
|
+
`component:` renders no `<Controls>`, so a console like
|
|
467
|
+
this one has to say it itself. end() then start(), so the
|
|
468
|
+
redial is a brand-new session (fresh store, greeting
|
|
469
|
+
included) and the console stays on the call rather than
|
|
470
|
+
dropping back to "Start call". */}
|
|
471
|
+
<button
|
|
472
|
+
type="button"
|
|
473
|
+
className="px-4 py-2 rounded-md text-xs font-semibold cursor-pointer"
|
|
474
|
+
style={{ background: "#ffffff", color: "#18181b", border: "1px solid #e4e4e7" }}
|
|
475
|
+
onClick={() => newConversation(session)}
|
|
476
|
+
>
|
|
477
|
+
New Conversation
|
|
478
|
+
</button>
|
|
451
479
|
{/* end() hangs up and flips `started` back, so the UI
|
|
452
480
|
returns to "Start call" and the next start is a brand-new
|
|
453
481
|
session (fresh store, greeting included). reset() would
|
|
@@ -271,7 +271,10 @@ describeWorkflowEval(
|
|
|
271
271
|
expect(run.output?.transcript).toBe(TRANSCRIPT);
|
|
272
272
|
// Two waits for three polls: asked for and — this being an eval rather
|
|
273
273
|
// than a deployment — recorded rather than taken.
|
|
274
|
-
expect(run.slept).toEqual([
|
|
274
|
+
expect(run.slept).toEqual([
|
|
275
|
+
{ label: "poll", duration: POLL_INTERVAL_MS },
|
|
276
|
+
{ label: "poll", duration: POLL_INTERVAL_MS },
|
|
277
|
+
]);
|
|
275
278
|
expect(provider.calls.filter((call) => call.leg === "poll")).toHaveLength(3);
|
|
276
279
|
// The expensive half happened ONCE, which is the whole reason the upload
|
|
277
280
|
// and the submit are separate steps.
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Specs for the spoken-summary app's declaration and its four legs.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* the
|
|
11
|
-
* `dev-workflow.scenario.test.ts
|
|
5
|
+
* The four legs are ordinary exported async functions, so their HTTP handling,
|
|
6
|
+
* their fatal/retryable classification and what they return are all testable
|
|
7
|
+
* directly. **And the body IS driven**, on the real replay engine — the last
|
|
8
|
+
* block — which this file used to say was another tier's job. It is still true
|
|
9
|
+
* that a body test dressed up as a durability test would be the worse failure;
|
|
10
|
+
* what changed is that the durable one is available, so the claim can be made
|
|
11
|
+
* honestly rather than deferred. `aai-cli`'s `dev-workflow.scenario.test.ts`
|
|
12
|
+
* remains the tier above, with a built project and a real queue.
|
|
12
13
|
*
|
|
13
14
|
* The two legs worth their own sections are the ones the SDK grew for this
|
|
14
15
|
* template. `speak` is where a step SPEAKS and STORES, and the assertion that
|
|
@@ -18,7 +19,7 @@
|
|
|
18
19
|
|
|
19
20
|
import { readUpload, uploadInfo } from "@alexkroman1/aai/step";
|
|
20
21
|
import { FatalError, RetryableError } from "@alexkroman1/aai/step-errors";
|
|
21
|
-
import { createWorkflowCtx } from "@alexkroman1/aai/testing";
|
|
22
|
+
import { createWorkflowCtx, stubGatewayRoute } from "@alexkroman1/aai/testing";
|
|
22
23
|
import {
|
|
23
24
|
installStubGateway,
|
|
24
25
|
installStubReporter,
|
|
@@ -26,6 +27,7 @@ import {
|
|
|
26
27
|
installStubTranscribe,
|
|
27
28
|
installStubUploads,
|
|
28
29
|
} from "@alexkroman1/aai/testing/vitest";
|
|
30
|
+
import { runWorkflow } from "@alexkroman1/aai-runtime/testing";
|
|
29
31
|
import { beforeEach, describe, expect, test, vi } from "vitest";
|
|
30
32
|
import agentDef, { spokenSummary } from "./agent.ts";
|
|
31
33
|
import { speak, spokenSummaryFlow, summarize } from "./workflows/summarize.ts";
|
|
@@ -313,3 +315,110 @@ describe("the whole run", () => {
|
|
|
313
315
|
);
|
|
314
316
|
});
|
|
315
317
|
});
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* The whole run, on the real replay engine.
|
|
321
|
+
*
|
|
322
|
+
* `runWorkflow` (`@alexkroman1/aai-runtime/testing`) starts the declared
|
|
323
|
+
* workflow on the engine `aai dev` uses, over an in-memory journal, and records
|
|
324
|
+
* a suspension rather than waiting it out. That is what makes the poll cadence
|
|
325
|
+
* — fifteen seconds a turn in a deployment — free here, and it is the only tier
|
|
326
|
+
* at which this app's central claim is checkable: **the recording is uploaded
|
|
327
|
+
* and transcribed ONCE**, however many times the body is walked.
|
|
328
|
+
*
|
|
329
|
+
* Three published slots have to agree for a whole run, and one of them is a
|
|
330
|
+
* shared seam: `stubTranscribe` publishes `stepFetch`, and publishing REPLACES —
|
|
331
|
+
* so the model call cannot have a stub of its own and is routed through
|
|
332
|
+
* `otherwise` instead. `installStubSpeech` and `installStubUploads` are
|
|
333
|
+
* different slots and compose freely.
|
|
334
|
+
*/
|
|
335
|
+
describe("the run is DURABLE", () => {
|
|
336
|
+
const SUMMARY = JSON.stringify({
|
|
337
|
+
headline: "Launch is on",
|
|
338
|
+
points: ["Ship Tuesday"],
|
|
339
|
+
spoken: "The launch is on for Tuesday.",
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
/** The provider, the model and the voice — one world for a whole run. */
|
|
343
|
+
function stubWorld({ pendingPolls = 0 } = {}) {
|
|
344
|
+
const model = stubGatewayRoute(SUMMARY);
|
|
345
|
+
const provider = installStubTranscribe({
|
|
346
|
+
pendingPolls,
|
|
347
|
+
text: "We shipped it on Tuesday.",
|
|
348
|
+
durationSec: 30,
|
|
349
|
+
// Anything that is not a transcription leg — which here is the model call
|
|
350
|
+
// — because this fake owns the one published `stepFetch`.
|
|
351
|
+
otherwise: (request) => model.route(request),
|
|
352
|
+
});
|
|
353
|
+
const speech = installStubSpeech({ pcmBytes: 48_000 });
|
|
354
|
+
installStubReporter();
|
|
355
|
+
return { model, provider, speech };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const INPUT = { recording: UPLOAD_ID };
|
|
359
|
+
|
|
360
|
+
test("uploads, submits, then parks on the poll cadence", async () => {
|
|
361
|
+
const world = stubWorld({ pendingPolls: 1 });
|
|
362
|
+
const started = Date.now();
|
|
363
|
+
const run = await runWorkflow(spokenSummary, INPUT, { name: "spokenSummary" });
|
|
364
|
+
|
|
365
|
+
expect(run.status).toBe("running");
|
|
366
|
+
expect(run.wakeAt).toBeGreaterThan(started);
|
|
367
|
+
expect(run.steps.map((step) => step.key)).toEqual([
|
|
368
|
+
"createJob#0",
|
|
369
|
+
"pollTranscript#0",
|
|
370
|
+
"uploadToProvider#0",
|
|
371
|
+
]);
|
|
372
|
+
// The recording has crossed the wire once. It is the expensive step — it
|
|
373
|
+
// streams the whole file, which is why it is the one with extra patience.
|
|
374
|
+
expect(world.provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
test("resumes past the poll and finishes, without re-uploading the recording", async () => {
|
|
378
|
+
const world = stubWorld({ pendingPolls: 1 });
|
|
379
|
+
const run = await runWorkflow(spokenSummary, INPUT, { name: "spokenSummary" });
|
|
380
|
+
await run.advanceSleep();
|
|
381
|
+
|
|
382
|
+
expect(run.status).toBe("completed");
|
|
383
|
+
expect(run.output).toMatchObject({
|
|
384
|
+
headline: "Launch is on",
|
|
385
|
+
points: ["Ship Tuesday"],
|
|
386
|
+
transcript: "We shipped it on Tuesday.",
|
|
387
|
+
// An ID, never the bytes: audio in a journaled step result is megabytes
|
|
388
|
+
// replayed on every resume.
|
|
389
|
+
audio: "upl_stub_1",
|
|
390
|
+
});
|
|
391
|
+
expect(run.deliveries).toBe(2);
|
|
392
|
+
// ONE upload and ONE submit across two walks — both came back out of the
|
|
393
|
+
// journal on the second.
|
|
394
|
+
expect(world.provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
395
|
+
expect(world.provider.calls.filter((call) => call.leg === "submit")).toHaveLength(1);
|
|
396
|
+
// Two polls, which is the loop doing its job: `pollTranscript#0` was
|
|
397
|
+
// journaled and `#1` is the one that found it done.
|
|
398
|
+
expect(run.steps.filter((step) => step.name === "pollTranscript")).toHaveLength(2);
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
test("a worker that dies at the voice keeps the transcript and the summary", async () => {
|
|
402
|
+
// The whole point of three steps rather than one: reading a summary aloud
|
|
403
|
+
// is cheap, transcribing a recording is not, and a crash at the cheap end
|
|
404
|
+
// must not buy the expensive one again.
|
|
405
|
+
const world = stubWorld();
|
|
406
|
+
const run = await runWorkflow(spokenSummary, INPUT, {
|
|
407
|
+
name: "spokenSummary",
|
|
408
|
+
crashAt: "speak",
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
expect(run.crashed).toBe(true);
|
|
412
|
+
expect(run.steps.map((step) => step.name)).toContain("summarize");
|
|
413
|
+
expect(world.speech.calls).toHaveLength(0);
|
|
414
|
+
|
|
415
|
+
await run.restart();
|
|
416
|
+
expect(run.status).toBe("completed");
|
|
417
|
+
expect(run.output?.audio).toBe("upl_stub_1");
|
|
418
|
+
// The voice ran once, and neither the provider nor the model was asked
|
|
419
|
+
// again.
|
|
420
|
+
expect(world.speech.calls).toHaveLength(1);
|
|
421
|
+
expect(world.provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
422
|
+
expect(world.model.calls).toHaveLength(1);
|
|
423
|
+
});
|
|
424
|
+
});
|
|
@@ -172,7 +172,7 @@ async function transcribe(recording: string, ctx: WorkflowCtx): Promise<Transcri
|
|
|
172
172
|
// rather than starting the wait over.
|
|
173
173
|
const progress = await ctx.step("pollTranscript", () => pollTranscript(recording, job.id));
|
|
174
174
|
if (progress.done) return progress.transcript;
|
|
175
|
-
await ctx.sleep(POLL_INTERVAL_MS);
|
|
175
|
+
await ctx.sleep("poll", POLL_INTERVAL_MS);
|
|
176
176
|
}
|
|
177
177
|
// A plain throw: this is the BODY, where the fatal/retryable distinction has
|
|
178
178
|
// nothing to apply to. The transcript is not lost, so the message says where
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
* Specs for the transcription desk's declaration, its WAV arithmetic, and its
|
|
4
4
|
* steps.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* `dev-workflow.scenario.test.ts
|
|
6
|
+
* A step is an ordinary exported async function, so its retries, its
|
|
7
|
+
* `FatalError` guards, its HTTP handling and its merge are all testable
|
|
8
|
+
* directly. **And one of the three bodies IS driven**, on the real replay
|
|
9
|
+
* engine — `transcribeBatch`, in the last block, which is the flow that reaches
|
|
10
|
+
* no ffmpeg. The other two normalize the recording first and this repo's test
|
|
11
|
+
* environment has no ffmpeg, so their durability stays `aai-cli`'s
|
|
12
|
+
* `dev-workflow.scenario.test.ts`'s. Saying which of the three is covered is
|
|
13
|
+
* the point: a body test dressed up as a durability test would be the worse
|
|
14
|
+
* failure, and so would a durable one that implied it covered all three.
|
|
13
15
|
*
|
|
14
16
|
* The WAV half is worth its own section because it is where a silent bug lives:
|
|
15
17
|
* a cut that lands mid-frame, or an off-by-one in the chunk walk, produces audio
|
|
@@ -27,6 +29,7 @@ import {
|
|
|
27
29
|
installStubTranscribe,
|
|
28
30
|
installStubUploads,
|
|
29
31
|
} from "@alexkroman1/aai/testing/vitest";
|
|
32
|
+
import { runWorkflow } from "@alexkroman1/aai-runtime/testing";
|
|
30
33
|
import { beforeEach, describe, expect, test, vi } from "vitest";
|
|
31
34
|
import { z } from "zod";
|
|
32
35
|
import agentDef, { transcribe, transcribeBatch, transcribeStream } from "./agent.ts";
|
|
@@ -77,10 +80,10 @@ const UPLOAD_ID = "upl_test";
|
|
|
77
80
|
/**
|
|
78
81
|
* A fixed run-start epoch, so `elapsedMs` is assertable at all.
|
|
79
82
|
*
|
|
80
|
-
* `
|
|
81
|
-
* the point of threading it as an argument rather
|
|
82
|
-
* merge — the duration is then a function of
|
|
83
|
-
* test took.
|
|
83
|
+
* The body reads it with `ctx.now()`, which the engine journals; a spec supplies
|
|
84
|
+
* the value directly, which is the point of threading it as an argument rather
|
|
85
|
+
* than reading a clock inside the merge — the duration is then a function of
|
|
86
|
+
* journaled values and not of how long the test took.
|
|
84
87
|
*/
|
|
85
88
|
const STARTED_AT = 1_000_000;
|
|
86
89
|
|
|
@@ -946,7 +949,6 @@ describe("transcribeSegment", () => {
|
|
|
946
949
|
const ctx = createWorkflowCtx({
|
|
947
950
|
runSteps: false,
|
|
948
951
|
results: {
|
|
949
|
-
startClock: 0,
|
|
950
952
|
normalizeRecording: { recording: UPLOAD_ID, converted: false },
|
|
951
953
|
splitRecording: { format: FORMAT, segments: [SEGMENT], durationMs: 1000 },
|
|
952
954
|
transcribeSegment: { index: 0, text: "hello" },
|
|
@@ -977,7 +979,7 @@ describe("transcribeSegment", () => {
|
|
|
977
979
|
// And the two that take the default, which is the other half of the claim.
|
|
978
980
|
// Asserted as PRESENT-with-no-budget rather than as `get(…) === undefined`,
|
|
979
981
|
// which a step the body never reached at all would also satisfy.
|
|
980
|
-
for (const name of ["
|
|
982
|
+
for (const name of ["mergeTranscript", "splitRecording"]) {
|
|
981
983
|
expect(budgets.has(name)).toBe(true);
|
|
982
984
|
expect(budgets.get(name)).toBeUndefined();
|
|
983
985
|
}
|
|
@@ -1759,3 +1761,115 @@ describe("nextPollDelay", () => {
|
|
|
1759
1761
|
expect(delay).toBe(5000);
|
|
1760
1762
|
});
|
|
1761
1763
|
});
|
|
1764
|
+
|
|
1765
|
+
/**
|
|
1766
|
+
* `transcribeBatch`, on the real replay engine.
|
|
1767
|
+
*
|
|
1768
|
+
* The batch flow is the one of this desk's three that reaches no ffmpeg — it
|
|
1769
|
+
* uploads, submits and polls — so it is the one a unit spec can drive end to
|
|
1770
|
+
* end. `transcribe` and `transcribeStream` both normalize the recording first,
|
|
1771
|
+
* and this repo's test environment has no ffmpeg (the `normalizeRecording` specs
|
|
1772
|
+
* above assert the FatalError that produces). Their durability is
|
|
1773
|
+
* `aai-cli`'s `dev-workflow.scenario.test.ts`'s to cover; what is asserted here
|
|
1774
|
+
* is the half that is honestly reachable, which is better than a body test
|
|
1775
|
+
* dressed up as a durability test.
|
|
1776
|
+
*
|
|
1777
|
+
* Two claims, and the second is the one no other tier can make:
|
|
1778
|
+
*
|
|
1779
|
+
* - **The recording is uploaded ONCE**, however many times the body is walked.
|
|
1780
|
+
* That is why the upload is a step of its own with a raised attempt budget.
|
|
1781
|
+
* - **`ctx.now()` is JOURNALED**, so `startedAt` — which the poll step subtracts
|
|
1782
|
+
* to report `elapsedMs` — is the instant the run really began and not the
|
|
1783
|
+
* instant the latest walk began. A body that read `Date.now()` there would
|
|
1784
|
+
* report a shrinking elapsed on every resume, which is `guard-invariants`
|
|
1785
|
+
* rule 30 and the whole reason the affordance exists.
|
|
1786
|
+
*/
|
|
1787
|
+
describe("the batch run is DURABLE", () => {
|
|
1788
|
+
const INPUT = { recording: UPLOAD_ID };
|
|
1789
|
+
|
|
1790
|
+
beforeEach(() => {
|
|
1791
|
+
installStubUploads({
|
|
1792
|
+
[UPLOAD_ID]: { bytes: new Uint8Array(4096), name: "standup.wav", type: "audio/wav" },
|
|
1793
|
+
});
|
|
1794
|
+
installStubReporter();
|
|
1795
|
+
vi.stubEnv("ASSEMBLYAI_API_KEY", "test-key");
|
|
1796
|
+
});
|
|
1797
|
+
|
|
1798
|
+
test("parks on the poll cadence with the upload already journaled", async () => {
|
|
1799
|
+
const provider = installStubTranscribe({ pendingPolls: 1, text: "We shipped it." });
|
|
1800
|
+
const started = Date.now();
|
|
1801
|
+
const run = await runWorkflow(transcribeBatch, INPUT, { name: "transcribeBatch" });
|
|
1802
|
+
|
|
1803
|
+
expect(run.status).toBe("running");
|
|
1804
|
+
expect(run.wakeAt).toBeGreaterThan(started);
|
|
1805
|
+
expect(run.steps.map((step) => step.key)).toEqual([
|
|
1806
|
+
"createJob#0",
|
|
1807
|
+
"pollTranscript#0",
|
|
1808
|
+
"uploadToProvider#0",
|
|
1809
|
+
]);
|
|
1810
|
+
// The clock the body read alongside the upload is journaled too, in its own
|
|
1811
|
+
// reserved key space — `run.reads`, not `run.steps`.
|
|
1812
|
+
expect(run.reads.map((read) => read.key)).toEqual(["now!0"]);
|
|
1813
|
+
expect(provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
1814
|
+
});
|
|
1815
|
+
|
|
1816
|
+
test("resumes past the poll without moving the recording again", async () => {
|
|
1817
|
+
const provider = installStubTranscribe({ pendingPolls: 1, text: "We shipped it." });
|
|
1818
|
+
const run = await runWorkflow(transcribeBatch, INPUT, { name: "transcribeBatch" });
|
|
1819
|
+
await run.advanceSleep();
|
|
1820
|
+
|
|
1821
|
+
expect(run.status).toBe("completed");
|
|
1822
|
+
expect(run.output).toMatchObject({
|
|
1823
|
+
transcript: "We shipped it.",
|
|
1824
|
+
// ONE, which is the difference this flow exists to show against the two
|
|
1825
|
+
// that segment.
|
|
1826
|
+
segments: 1,
|
|
1827
|
+
source: "standup.wav",
|
|
1828
|
+
});
|
|
1829
|
+
expect(run.deliveries).toBe(2);
|
|
1830
|
+
// The whole recording crossed the wire once, on the first walk.
|
|
1831
|
+
expect(provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
1832
|
+
expect(provider.calls.filter((call) => call.leg === "submit")).toHaveLength(1);
|
|
1833
|
+
});
|
|
1834
|
+
|
|
1835
|
+
test("reports the elapsed from the run's OWN start, not the resuming walk's", async () => {
|
|
1836
|
+
// The affordance's claim. `ctx.now()` is journaled under its own key, so the
|
|
1837
|
+
// second walk re-reads the first walk's instant — and `elapsedMs`, which the
|
|
1838
|
+
// poll step computes as `Date.now() - startedAt`, therefore counts the whole
|
|
1839
|
+
// run rather than the delivery it finished on.
|
|
1840
|
+
const provider = installStubTranscribe({ pendingPolls: 1, text: "We shipped it." });
|
|
1841
|
+
const run = await runWorkflow(transcribeBatch, INPUT, { name: "transcribeBatch" });
|
|
1842
|
+
await run.advanceSleep();
|
|
1843
|
+
|
|
1844
|
+
expect(run.status).toBe("completed");
|
|
1845
|
+
// A journaled clock cannot go backwards across a resume, and a re-read one
|
|
1846
|
+
// routinely would — the second walk starts later than the first.
|
|
1847
|
+
expect(run.output?.elapsedMs).toBeGreaterThanOrEqual(0);
|
|
1848
|
+
expect(provider.calls.filter((call) => call.leg === "poll")).toHaveLength(2);
|
|
1849
|
+
});
|
|
1850
|
+
|
|
1851
|
+
test("a worker that dies before the job is created re-uploads, because nothing settled", async () => {
|
|
1852
|
+
// The other side of exactly-once, and the honest one: a step whose entry was
|
|
1853
|
+
// never written IS re-run, because the journal is what makes a step skippable
|
|
1854
|
+
// and a crashed one left none.
|
|
1855
|
+
const provider = installStubTranscribe({ text: "We shipped it." });
|
|
1856
|
+
const run = await runWorkflow(transcribeBatch, INPUT, {
|
|
1857
|
+
name: "transcribeBatch",
|
|
1858
|
+
crashAt: "createJob",
|
|
1859
|
+
});
|
|
1860
|
+
|
|
1861
|
+
expect(run.crashed).toBe(true);
|
|
1862
|
+
expect(run.steps.map((step) => step.name)).toEqual(["uploadToProvider"]);
|
|
1863
|
+
// The clock the body read in the same `Promise.all` survived the crash with
|
|
1864
|
+
// it, which is what makes the resumed run's elapsed the WHOLE run's.
|
|
1865
|
+
expect(run.reads.map((read) => read.key)).toEqual(["now!0"]);
|
|
1866
|
+
expect(provider.calls.filter((call) => call.leg === "submit")).toHaveLength(0);
|
|
1867
|
+
|
|
1868
|
+
await run.restart();
|
|
1869
|
+
expect(run.status).toBe("completed");
|
|
1870
|
+
// The upload SURVIVED — it settled before the crash — and only the job
|
|
1871
|
+
// creation was re-issued.
|
|
1872
|
+
expect(provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
|
|
1873
|
+
expect(provider.calls.filter((call) => call.leg === "submit")).toHaveLength(1);
|
|
1874
|
+
});
|
|
1875
|
+
});
|
|
@@ -61,7 +61,7 @@ import {
|
|
|
61
61
|
stepTranscribeUploadClassified,
|
|
62
62
|
} from "@alexkroman1/aai/step-errors";
|
|
63
63
|
import { countWords, formatBytes } from "@alexkroman1/aai/utils";
|
|
64
|
-
import {
|
|
64
|
+
import type { Transcript } from "./transcribe.ts";
|
|
65
65
|
|
|
66
66
|
/** How long between polls of a submitted job. */
|
|
67
67
|
const POLL_INTERVAL_MS = 10_000;
|
|
@@ -88,7 +88,7 @@ export async function transcribeBatchFlow(
|
|
|
88
88
|
// one call here worth another attempt: it moves the whole recording, and a lost
|
|
89
89
|
// connection on a file this size is the expected failure.
|
|
90
90
|
const [startedAt, { audioUrl }] = await Promise.all([
|
|
91
|
-
ctx.
|
|
91
|
+
ctx.now(),
|
|
92
92
|
ctx.step("uploadToProvider", () => uploadToProvider(input.recording), { maxAttempts: 6 }),
|
|
93
93
|
]);
|
|
94
94
|
const job = await ctx.step("createJob", () => createJob(audioUrl));
|
|
@@ -98,7 +98,7 @@ export async function transcribeBatchFlow(
|
|
|
98
98
|
pollTranscript(input.recording, job.id, startedAt),
|
|
99
99
|
);
|
|
100
100
|
if (progress.done) return progress.transcript;
|
|
101
|
-
await ctx.sleep(POLL_INTERVAL_MS);
|
|
101
|
+
await ctx.sleep("poll", POLL_INTERVAL_MS);
|
|
102
102
|
}
|
|
103
103
|
// A plain throw: this is the BODY, where the fatal/retryable distinction has
|
|
104
104
|
// nothing to apply to — see `stream.ts`'s `abandon` for the same reasoning.
|
|
@@ -175,9 +175,9 @@ export async function pollTranscript(
|
|
|
175
175
|
// The provider's own measurement — the only one of the three flows that does
|
|
176
176
|
// not have to derive this from byte offsets.
|
|
177
177
|
durationMs: progress.transcript.durationMs,
|
|
178
|
-
// Wall clock, the same way both sync flows measure it —
|
|
179
|
-
//
|
|
180
|
-
//
|
|
178
|
+
// Wall clock, the same way both sync flows measure it — the body's
|
|
179
|
+
// `ctx.now()`, subtracted inside this step. For this flow it is mostly the
|
|
180
|
+
// provider's queue, which is exactly the thing a reader wants to see.
|
|
181
181
|
elapsedMs: Date.now() - startedAt,
|
|
182
182
|
words: countWords(transcript),
|
|
183
183
|
transcript,
|
|
@@ -182,7 +182,6 @@ import {
|
|
|
182
182
|
mergeTranscript,
|
|
183
183
|
type SegmentTranscript,
|
|
184
184
|
segmentConcurrency,
|
|
185
|
-
startClock,
|
|
186
185
|
transcribeSegment,
|
|
187
186
|
} from "./transcribe.ts";
|
|
188
187
|
import {
|
|
@@ -288,7 +287,9 @@ export type StreamPlan = {
|
|
|
288
287
|
* it and the bytes are still on their way.
|
|
289
288
|
*/
|
|
290
289
|
export async function transcribeStreamFlow(input: { recording: string }, ctx: WorkflowCtx) {
|
|
291
|
-
|
|
290
|
+
// `ctx.now()`, not a step: the engine journals the read under its own key, so
|
|
291
|
+
// every walk of this line sees the instant the first one did.
|
|
292
|
+
const startedAt = await ctx.now();
|
|
292
293
|
let plan: StreamPlan | undefined;
|
|
293
294
|
// Body state, and legal because every value in it came out of a journaled step
|
|
294
295
|
// result — a replay rebuilds the identical sets in the identical order.
|
|
@@ -380,7 +381,7 @@ export async function transcribeStreamFlow(input: { recording: string }, ctx: Wo
|
|
|
380
381
|
// Sleep until the next segment should HAVE landed, rather than for a fixed
|
|
381
382
|
// interval — see `nextPollDelay`. Both arguments are journaled step results,
|
|
382
383
|
// so a replay computes the same delay from the same two samples.
|
|
383
|
-
await ctx.sleep(nextPollDelay(at, previous, plan, done));
|
|
384
|
+
await ctx.sleep("poll", nextPollDelay(at, previous, plan, done));
|
|
384
385
|
previous = at;
|
|
385
386
|
}
|
|
386
387
|
|