@alexkroman1/aai-cli 9.2.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_bundler-CDuYl5Gb.mjs → _bundler-B31MqYaD.mjs} +7 -33
- package/dist/_bundler.d.ts +3 -41
- package/dist/{_dev-server-BAR8Ya43.mjs → _dev-server-Bo5dSBzQ.mjs} +140 -44
- package/dist/_dev-server.d.ts +1 -8
- package/dist/_dev-vite-config.d.ts +21 -0
- package/dist/{_init-BNotRgV1.mjs → _init-BURbdHNM.mjs} +1 -1
- package/dist/{_slug-api--ZBmIGH-.mjs → _slug-api-CfGQq1Tt.mjs} +1 -1
- package/dist/{_templates-4WcKOjS5.mjs → _templates-0Zr2z6yA.mjs} +37 -2
- package/dist/_templates.d.ts +29 -0
- package/dist/{build-DvDdSwAS.mjs → build-DKC2e9o_.mjs} +1 -2
- package/dist/cli.mjs +46 -22
- package/dist/{delete-8pjH8n8n.mjs → delete-X_CRgCf6.mjs} +1 -1
- package/dist/{deploy-CjU3e-3P.mjs → deploy-CchW5pmw.mjs} +2 -2
- package/dist/{dev-dFzdvwa7.mjs → dev-BfVl1uBA.mjs} +1 -1
- package/dist/{init-DK7GpIF9.mjs → init-BRfoc2EK.mjs} +2 -2
- package/dist/{logs-D2Yhqe2w.mjs → logs-hWnJ66Bl.mjs} +2 -2
- package/dist/scaffold/CLAUDE.md +162 -54
- package/dist/scaffold/package.json +4 -6
- package/dist/scaffold/server.mjs +12 -23
- package/dist/{secret-CwgDLpSy.mjs → secret-B-3T5tvr.mjs} +2 -2
- package/dist/{studio-D3-HWIfk.mjs → studio-wVWNLREn.mjs} +4 -3
- package/dist/templates/call-audit/agent.eval.test.ts +1 -1
- package/dist/templates/call-audit/agent.test.ts +80 -8
- package/dist/templates/call-audit/client.tsx +87 -3
- package/dist/templates/call-audit/workflows/audit.ts +52 -32
- package/dist/templates/call-audit/workflows/ingest.ts +15 -15
- package/dist/templates/call-audit/workflows/media.ts +1 -1
- package/dist/templates/call-audit/workflows/summarize.ts +0 -4
- package/dist/templates/link-digest/agent.eval.test.ts +4 -2
- package/dist/templates/link-digest/agent.test.ts +25 -5
- package/dist/templates/link-digest/client.tsx +65 -7
- package/dist/templates/link-digest/workflows/digest.ts +39 -29
- package/dist/templates/podcast-digest/agent.eval.test.ts +5 -5
- package/dist/templates/podcast-digest/agent.test.ts +108 -2
- package/dist/templates/podcast-digest/client.tsx +89 -8
- package/dist/templates/podcast-digest/workflows/digest.ts +67 -48
- package/dist/templates/podcast-digest/workflows/feeds.ts +1 -4
- package/dist/templates/podcast-digest/workflows/slack.ts +5 -6
- package/dist/templates/recap-workflow/agent.eval.test.ts +72 -8
- package/dist/templates/recap-workflow/agent.test.ts +323 -83
- package/dist/templates/recap-workflow/shared.ts +4 -3
- package/dist/templates/recap-workflow/workflows/recap.ts +322 -86
- package/dist/templates/recap-workflow/workflows/tokens.ts +21 -0
- package/dist/templates/redline/agent.eval.test.ts +1 -1
- package/dist/templates/redline/agent.test.ts +1 -1
- package/dist/templates/redline/client.tsx +83 -3
- package/dist/templates/redline/workflows/redline.ts +26 -23
- package/dist/templates/research-workflow/agent.eval.test.ts +9 -8
- package/dist/templates/research-workflow/agent.test.ts +32 -6
- package/dist/templates/research-workflow/agent.ts +4 -4
- package/dist/templates/research-workflow/shared.ts +4 -3
- package/dist/templates/research-workflow/workflows/prompts.ts +1 -1
- package/dist/templates/research-workflow/workflows/research.ts +35 -38
- package/dist/templates/spoken-summary/agent.eval.test.ts +3 -3
- package/dist/templates/spoken-summary/agent.test.ts +10 -8
- package/dist/templates/spoken-summary/client.tsx +72 -3
- package/dist/templates/spoken-summary/workflows/summarize.ts +25 -21
- package/dist/templates/spoken-summary/workflows/transcribe.ts +15 -18
- package/dist/templates/transcription-workflow/agent.eval.test.ts +1 -1
- package/dist/templates/transcription-workflow/agent.test.ts +626 -14
- package/dist/templates/transcription-workflow/agent.ts +6 -4
- package/dist/templates/transcription-workflow/client.tsx +106 -308
- package/dist/templates/transcription-workflow/recover.test.ts +141 -0
- package/dist/templates/transcription-workflow/recover.ts +130 -0
- package/dist/templates/transcription-workflow/run-panel.tsx +203 -0
- package/dist/templates/transcription-workflow/total-latency.tsx +144 -0
- package/dist/templates/transcription-workflow/workflows/batch.ts +21 -24
- package/dist/templates/transcription-workflow/workflows/downsample.ts +273 -0
- package/dist/templates/transcription-workflow/workflows/normalize.ts +17 -40
- package/dist/templates/transcription-workflow/workflows/stream.ts +177 -24
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +69 -32
- package/dist/templates/transcription-workflow/workflows/wav.ts +15 -4
- package/dist/worker-bundler.d.ts +0 -14
- package/dist/worker-bundler.mjs +149 -1
- package/dist/{workflow-BflATMmU.mjs → workflow-BBu5oEDw.mjs} +84 -10
- package/dist/workflow.d.ts +1 -0
- package/package.json +4 -6
- package/dist/_workflow-scan.d.ts +0 -108
- package/dist/worker-bundler-CCVEDjm1.mjs +0 -650
- package/dist/workflow-bundler.d.ts +0 -138
- package/dist/{_studio-CKrsixd0.mjs → _studio-B1waXMuR.mjs} +1 -1
|
@@ -24,6 +24,31 @@
|
|
|
24
24
|
* and a textarea holds a string where the workflow's schema wants `string[]`.
|
|
25
25
|
* `toInput` is where the two meet — and it is the only place, so the split lives
|
|
26
26
|
* in one function rather than in the field, the submit handler and the workflow.
|
|
27
|
+
*
|
|
28
|
+
* ## A reload used to lose the loop, which is minutes of model calls
|
|
29
|
+
*
|
|
30
|
+
* A `runId` names a run for as long as something holds it, and this page held it
|
|
31
|
+
* in React state — so a refresh lost it while the loop carried on writing,
|
|
32
|
+
* grading and revising without anywhere to report to. On a desk whose whole
|
|
33
|
+
* subject is a loop that runs several long-form model calls, that is the one
|
|
34
|
+
* failure worth two lines of wiring: `key` is the handle that survives a reload
|
|
35
|
+
* and `recover` is what reads it back, so the draft, the critique trail and the
|
|
36
|
+
* Clear button are all there again.
|
|
37
|
+
*
|
|
38
|
+
* **The key is opaque and lives in `sessionStorage`, and the brief is why.** A
|
|
39
|
+
* `?key=` parameter in the page's own URL would survive more — a new tab, a
|
|
40
|
+
* bookmark, a link sent to the person who asked for the piece — and that is
|
|
41
|
+
* exactly what it must not do here. There is no per-user filtering behind
|
|
42
|
+
* `find`, so the key IS the scoping mechanism, and a brief is the most private
|
|
43
|
+
* thing on this page: it is what somebody typed about their own product, their
|
|
44
|
+
* own incident or their own customers, and the critique trail beside it is
|
|
45
|
+
* working material nobody writes expecting an audience. The thing worth sending
|
|
46
|
+
* a colleague is the DRAFT, which is text on the page and travels by being
|
|
47
|
+
* copied; sending a run means sending the brief that produced it.
|
|
48
|
+
*
|
|
49
|
+
* Deriving the key from the brief is worse again: two people briefing the same
|
|
50
|
+
* thing would recover each other's runs, and the key would then carry what they
|
|
51
|
+
* typed into a lookup token the platform deliberately stopped logging.
|
|
27
52
|
*/
|
|
28
53
|
|
|
29
54
|
import "@alexkroman1/aai-ui/styles.css";
|
|
@@ -35,12 +60,14 @@ import {
|
|
|
35
60
|
page,
|
|
36
61
|
SubmitButton,
|
|
37
62
|
TextAreaField,
|
|
63
|
+
useRunKey,
|
|
38
64
|
useWorkflowSubmit,
|
|
39
65
|
WORKFLOW_STATUS_LABELS,
|
|
40
66
|
WorkflowFields,
|
|
41
67
|
WorkflowProgress,
|
|
42
68
|
type WorkflowRun,
|
|
43
69
|
} from "@alexkroman1/aai-ui";
|
|
70
|
+
import { useState } from "react";
|
|
44
71
|
import type { redline } from "./agent.ts";
|
|
45
72
|
|
|
46
73
|
/**
|
|
@@ -55,6 +82,21 @@ type Redline = WorkflowOutputOf<typeof redline>;
|
|
|
55
82
|
/** The workflow this page drives. Matches the key in `workflowApp({ workflows })`. */
|
|
56
83
|
const WORKFLOW = "redline";
|
|
57
84
|
|
|
85
|
+
/**
|
|
86
|
+
* What the desk says while the loop is turning — three situations, one line
|
|
87
|
+
* each.
|
|
88
|
+
*
|
|
89
|
+
* The reload case gets its own words deliberately: somebody who did not press
|
|
90
|
+
* the button is owed an explanation for a draft appearing in front of them, and
|
|
91
|
+
* the sentence a page reaches for instead ("you can close this tab") is the one
|
|
92
|
+
* that was true about the RUN and false about the page.
|
|
93
|
+
*/
|
|
94
|
+
function pendingNote(startedHere: boolean, found: boolean): string {
|
|
95
|
+
if (startedHere) return "Reloading is safe — this page will find the draft again.";
|
|
96
|
+
if (!found) return "Looking for a draft this tab started earlier…";
|
|
97
|
+
return "Still working on a draft this tab started earlier.";
|
|
98
|
+
}
|
|
99
|
+
|
|
58
100
|
/**
|
|
59
101
|
* The submitted form as the workflow's input schema wants it.
|
|
60
102
|
*
|
|
@@ -79,7 +121,19 @@ export function toInput(values: FormValues): WorkflowInputOf<typeof redline> {
|
|
|
79
121
|
}
|
|
80
122
|
|
|
81
123
|
function RedlineDesk() {
|
|
82
|
-
|
|
124
|
+
// This tab's handle on its own drafts — opaque, short, and remembered for the
|
|
125
|
+
// next load, which is what `recover` produces to find the run again.
|
|
126
|
+
const key = useRunKey();
|
|
127
|
+
// Did THIS load start the run? A reload cannot have, and that is the only way
|
|
128
|
+
// the page can tell "writing what you just briefed" from "picking up where
|
|
129
|
+
// you left off" — the hook reports the run, not who asked for it.
|
|
130
|
+
const [startedHere, setStartedHere] = useState(false);
|
|
131
|
+
const { submit, run, pending, error, reset } = useWorkflowSubmit<typeof redline>(WORKFLOW, {
|
|
132
|
+
// Neither half is useful alone: without the key there is nothing to find
|
|
133
|
+
// the run by, and without `recover` the key is only ever written.
|
|
134
|
+
key,
|
|
135
|
+
recover: true,
|
|
136
|
+
});
|
|
83
137
|
|
|
84
138
|
return (
|
|
85
139
|
<main className="mx-auto flex max-w-2xl flex-col gap-8 p-8">
|
|
@@ -91,7 +145,13 @@ function RedlineDesk() {
|
|
|
91
145
|
</p>
|
|
92
146
|
</header>
|
|
93
147
|
|
|
94
|
-
<Form
|
|
148
|
+
<Form
|
|
149
|
+
onSubmit={(values) => {
|
|
150
|
+
setStartedHere(true);
|
|
151
|
+
return submit(toInput(values));
|
|
152
|
+
}}
|
|
153
|
+
error={error}
|
|
154
|
+
>
|
|
95
155
|
{/* The scalars: brief, audience, rounds. Declared, not written. */}
|
|
96
156
|
<WorkflowFields workflow={WORKFLOW} />
|
|
97
157
|
{/* The array the schema declares and no generic control can render. */}
|
|
@@ -104,7 +164,27 @@ function RedlineDesk() {
|
|
|
104
164
|
<SubmitButton pending={pending}>Write it</SubmitButton>
|
|
105
165
|
</Form>
|
|
106
166
|
|
|
107
|
-
{
|
|
167
|
+
{/* `pending` covers the RUN rather than the request, and on a reload it is
|
|
168
|
+
also true while the run is being looked up by key — the stretch where a
|
|
169
|
+
form offering Submit would be inviting a second loop over the same
|
|
170
|
+
brief, which here is several long-form model calls of somebody's
|
|
171
|
+
money. */}
|
|
172
|
+
{pending && (
|
|
173
|
+
<p className="text-sm opacity-70">{pendingNote(startedHere, run !== undefined)}</p>
|
|
174
|
+
)}
|
|
175
|
+
|
|
176
|
+
{run && (
|
|
177
|
+
<RunPanel
|
|
178
|
+
run={run}
|
|
179
|
+
onClear={() => {
|
|
180
|
+
// The recovered run is dismissed as deliberately as one this load
|
|
181
|
+
// started: `reset()` is not undone by a second lookup (the lookup
|
|
182
|
+
// is a mount-time act), so Clear really does clear.
|
|
183
|
+
setStartedHere(false);
|
|
184
|
+
reset();
|
|
185
|
+
}}
|
|
186
|
+
/>
|
|
187
|
+
)}
|
|
108
188
|
</main>
|
|
109
189
|
);
|
|
110
190
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* The durable half of the redline desk: write, critique, revise — in a loop
|
|
3
3
|
* whose length the CRITIC decides.
|
|
4
4
|
*
|
|
5
|
-
* The rules a
|
|
5
|
+
* The rules a workflow body lives under are spelled out in
|
|
6
6
|
* `research-workflow/workflows/research.ts` and `link-digest/workflows/digest.ts`:
|
|
7
7
|
* the body is replayed from the top on every resume, so it holds no live handle
|
|
8
8
|
* and makes no undurable decision, and a step's arguments and result cross a
|
|
@@ -16,9 +16,10 @@
|
|
|
16
16
|
* different thing: `critique` returns a verdict, the body breaks on it, and a
|
|
17
17
|
* replay reads that verdict back out of the journal and takes the same branch.
|
|
18
18
|
* Deciding it any other way — a clock, a random draw, a re-read of something
|
|
19
|
-
* outside the run — would let a replay diverge
|
|
20
|
-
*
|
|
21
|
-
*
|
|
19
|
+
* outside the run — would let a replay diverge. A step is identified by its
|
|
20
|
+
* NAME plus the number of times that name has been reached in this run, so a
|
|
21
|
+
* branch that takes a different path on replay reads a journal entry that was
|
|
22
|
+
* written for a different call, rather than producing a slightly different essay.
|
|
22
23
|
*
|
|
23
24
|
* ## Why durability earns its keep here, specifically
|
|
24
25
|
*
|
|
@@ -31,10 +32,14 @@
|
|
|
31
32
|
* three functions read more tidily than one.
|
|
32
33
|
*/
|
|
33
34
|
|
|
35
|
+
import type { WorkflowCtx } from "@alexkroman1/aai";
|
|
34
36
|
import { report } from "@alexkroman1/aai/step";
|
|
35
|
-
import {
|
|
37
|
+
import {
|
|
38
|
+
FatalError,
|
|
39
|
+
stepGenerateClassified,
|
|
40
|
+
stepGenerateJsonClassified,
|
|
41
|
+
} from "@alexkroman1/aai/step-errors";
|
|
36
42
|
import { countWords } from "@alexkroman1/aai/utils";
|
|
37
|
-
import { FatalError } from "workflow";
|
|
38
43
|
import { z } from "zod";
|
|
39
44
|
import { CRITIC_SYSTEM, REVISER_SYSTEM, WRITER_SYSTEM } from "./prompts.ts";
|
|
40
45
|
|
|
@@ -93,15 +98,22 @@ export interface Round {
|
|
|
93
98
|
* the page's render model — and `WorkflowOutputOf<typeof redline>` in
|
|
94
99
|
* `client.tsx` is that type, derived rather than restated.
|
|
95
100
|
*/
|
|
96
|
-
export async function redlineFlow(input: RedlineInput) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
101
|
+
export async function redlineFlow(input: RedlineInput, ctx: WorkflowCtx) {
|
|
102
|
+
// The three `maxAttempts` below were `maxRetries` properties on the functions
|
|
103
|
+
// (3, 5, 3 — retries AFTER the first attempt, so 4, 6, 4 in all). The policy
|
|
104
|
+
// is an argument to the CALL now, which is where it belongs: the same function
|
|
105
|
+
// called from two places may deserve different patience.
|
|
106
|
+
let draft = await ctx.step("writeDraft", () => writeDraft(input), { maxAttempts: 4 });
|
|
100
107
|
const rounds: Round[] = [];
|
|
101
108
|
let shipped = false;
|
|
102
109
|
|
|
103
110
|
for (let round = 1; round <= input.rounds; round++) {
|
|
104
|
-
|
|
111
|
+
// ONE call site in a loop, which is exactly what `(name, occurrence)` step
|
|
112
|
+
// identity is for: this is `critiqueDraft#0`, `critiqueDraft#1`, … so each
|
|
113
|
+
// round journals separately and a resume replays the rounds already done.
|
|
114
|
+
const critique = await ctx.step("critiqueDraft", () => critiqueDraft(draft, input, round), {
|
|
115
|
+
maxAttempts: 6,
|
|
116
|
+
});
|
|
105
117
|
|
|
106
118
|
if (critique.verdict === "ship") {
|
|
107
119
|
// The break is decided by a STEP'S JOURNALED RESULT, which is what makes
|
|
@@ -111,7 +123,9 @@ export async function redlineFlow(input: RedlineInput) {
|
|
|
111
123
|
break;
|
|
112
124
|
}
|
|
113
125
|
|
|
114
|
-
draft = await reviseDraft(draft, critique, input, round)
|
|
126
|
+
draft = await ctx.step("reviseDraft", () => reviseDraft(draft, critique, input, round), {
|
|
127
|
+
maxAttempts: 4,
|
|
128
|
+
});
|
|
115
129
|
rounds.push({ round, critique, revisedWords: countWords(draft) });
|
|
116
130
|
}
|
|
117
131
|
|
|
@@ -127,8 +141,6 @@ export async function redlineFlow(input: RedlineInput) {
|
|
|
127
141
|
|
|
128
142
|
/** Their `generation_node`, first pass. */
|
|
129
143
|
export async function writeDraft(input: RedlineInput): Promise<string> {
|
|
130
|
-
"use step";
|
|
131
|
-
|
|
132
144
|
if (input.brief.trim().length < MIN_BRIEF_CHARS) {
|
|
133
145
|
// Fatal rather than retryable: the same brief is the same brief on every
|
|
134
146
|
// attempt, and four more model calls will not make it longer.
|
|
@@ -163,8 +175,6 @@ export async function critiqueDraft(
|
|
|
163
175
|
input: RedlineInput,
|
|
164
176
|
round: number,
|
|
165
177
|
): Promise<Critique> {
|
|
166
|
-
"use step";
|
|
167
|
-
|
|
168
178
|
await report(`Round ${round}: reading it back critically.`);
|
|
169
179
|
// `stepGenerateJson` owns the fence, the parse, the non-object case and the
|
|
170
180
|
// shape — and throws PLAINLY when any of them misses, unlike the fatal one
|
|
@@ -197,8 +207,6 @@ export async function reviseDraft(
|
|
|
197
207
|
input: RedlineInput,
|
|
198
208
|
round: number,
|
|
199
209
|
): Promise<string> {
|
|
200
|
-
"use step";
|
|
201
|
-
|
|
202
210
|
await report(`Round ${round}: revising.`);
|
|
203
211
|
const revised = await stepGenerateClassified(
|
|
204
212
|
[
|
|
@@ -239,8 +247,3 @@ export function clampScore(score: number): number {
|
|
|
239
247
|
// `RetryableError` carrying the delay the gateway itself named, which beats
|
|
240
248
|
// `RetryableError`'s own one-second default. Three templates each wrapped the
|
|
241
249
|
// raw `/step` call to say that; the wrapper is a suffix on the import now.
|
|
242
|
-
|
|
243
|
-
/** A rate limit — and a model that ignored the format — are both expected. */
|
|
244
|
-
critiqueDraft.maxRetries = 5;
|
|
245
|
-
writeDraft.maxRetries = 3;
|
|
246
|
-
reviseDraft.maxRetries = 3;
|
|
@@ -28,6 +28,13 @@
|
|
|
28
28
|
// And what no eval here can see at all: anything below the audio boundary —
|
|
29
29
|
// endpointing, barge-in, whether two sentences merged into one turn.
|
|
30
30
|
|
|
31
|
+
import agentDef from "virtual:aai/agent";
|
|
32
|
+
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
33
|
+
import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
|
|
34
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
35
|
+
import { expect } from "vitest";
|
|
36
|
+
import { z } from "zod";
|
|
37
|
+
import { research } from "./shared.ts";
|
|
31
38
|
/**
|
|
32
39
|
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
33
40
|
*
|
|
@@ -40,13 +47,7 @@
|
|
|
40
47
|
* The glob is written here rather than reached for from a shared helper because
|
|
41
48
|
* this file SHIPS — see `agent.test.ts`.
|
|
42
49
|
*/
|
|
43
|
-
import
|
|
44
|
-
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
45
|
-
import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
|
|
46
|
-
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
47
|
-
import { expect } from "vitest";
|
|
48
|
-
import { z } from "zod";
|
|
49
|
-
import { research } from "./shared.ts";
|
|
50
|
+
import { REVIEW_DELAY_MS } from "./workflows/research.ts";
|
|
50
51
|
|
|
51
52
|
/**
|
|
52
53
|
* The key the run's steps read with `requireStepEnv`.
|
|
@@ -249,7 +250,7 @@ describeEval(
|
|
|
249
250
|
// durable `sleep` rather than suspending, so what a case can honestly
|
|
250
251
|
// claim is that the body asked — and that is the assertion that fails
|
|
251
252
|
// if the suspension is ever deleted.
|
|
252
|
-
expect(run?.slept).toEqual([{ duration:
|
|
253
|
+
expect(run?.slept).toEqual([{ duration: REVIEW_DELAY_MS }]);
|
|
253
254
|
|
|
254
255
|
// Six model calls, all through the step slot: the fan-out's width came
|
|
255
256
|
// from a journaled stage rather than from anything the body recomputed.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* progress stream, the early wake) ask for what a voice reply can use.
|
|
14
14
|
*
|
|
15
15
|
* The STEPS are exercised separately, and directly: imported through vitest with
|
|
16
|
-
*
|
|
16
|
+
* a step is an ordinary exported async function,
|
|
17
17
|
* so its prompt handling, its parsing and its `FatalError` guards are all
|
|
18
18
|
* testable — while durability, suspension and replay are not. The body itself is
|
|
19
19
|
* not driven here for that reason; `aai-cli`'s `dev-workflow.scenario.test.ts`
|
|
@@ -21,9 +21,11 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import type { WorkflowClient } from "@alexkroman1/aai";
|
|
24
|
+
import { FatalError, RetryableError } from "@alexkroman1/aai/step-errors";
|
|
24
25
|
import {
|
|
25
26
|
createRunSnapshot,
|
|
26
27
|
createToolContext,
|
|
28
|
+
createWorkflowCtx,
|
|
27
29
|
parseSchemaInput,
|
|
28
30
|
type StubGatewayCall,
|
|
29
31
|
schemaInputIssues,
|
|
@@ -33,7 +35,6 @@ import { mockWorkflows, installStubGateway as stubGateway } from "@alexkroman1/a
|
|
|
33
35
|
import { visitWebpage, webSearch } from "@alexkroman1/aai/tools";
|
|
34
36
|
import type { WorkflowRunSnapshot } from "@alexkroman1/aai/workflow-api";
|
|
35
37
|
import { beforeEach, describe, expect, test, vi } from "vitest";
|
|
36
|
-
import { FatalError, RetryableError } from "workflow";
|
|
37
38
|
import { research } from "./shared.ts";
|
|
38
39
|
import {
|
|
39
40
|
countSources,
|
|
@@ -41,6 +42,7 @@ import {
|
|
|
41
42
|
findGaps,
|
|
42
43
|
investigate,
|
|
43
44
|
planAngles,
|
|
45
|
+
researchFlow,
|
|
44
46
|
writeBrief,
|
|
45
47
|
writeReport,
|
|
46
48
|
} from "./workflows/research.ts";
|
|
@@ -422,13 +424,37 @@ describe("the steps that research", () => {
|
|
|
422
424
|
expect(promptOf(calls, 2)).toContain("search is down");
|
|
423
425
|
});
|
|
424
426
|
|
|
425
|
-
test("investigate
|
|
426
|
-
|
|
427
|
+
test("both investigate waves are called with more attempts than the default", async () => {
|
|
428
|
+
// The retry policy is an argument to `ctx.step` now rather than a
|
|
429
|
+
// `maxRetries` property, so it is observable only at the CALL — and there
|
|
430
|
+
// are two calls, one per wave, which is exactly the kind of thing a property
|
|
431
|
+
// could not have said differently.
|
|
432
|
+
// `planAngles`' result is what the fan-out iterates, so it is supplied
|
|
433
|
+
// rather than run — the rest of the body needs no page and no model.
|
|
434
|
+
const ctx = createWorkflowCtx({
|
|
435
|
+
runSteps: false,
|
|
436
|
+
// Every step the body READS needs a value: with `runSteps: false` nothing
|
|
437
|
+
// runs, so this is the skeleton of a run rather than a run. That is the
|
|
438
|
+
// trade — no page, no model and no search, in exchange for spelling the
|
|
439
|
+
// shape out.
|
|
440
|
+
results: {
|
|
441
|
+
planAngles: ["Adoption", "Tooling"],
|
|
442
|
+
findGaps: ["Cost"],
|
|
443
|
+
investigate: { angle: "Adoption", findings: "f", sources: [] },
|
|
444
|
+
investigateGap: { angle: "Cost", findings: "f", sources: [] },
|
|
445
|
+
writeReport: { summary: "s", report: "r" },
|
|
446
|
+
},
|
|
447
|
+
});
|
|
448
|
+
await researchFlow({ topic: "Tool use", requestedBy: "Ada" }, ctx);
|
|
449
|
+
|
|
450
|
+
const investigations = ctx.steps.filter((step) => step.name.startsWith("investigate"));
|
|
451
|
+
expect(investigations.length).toBeGreaterThan(0);
|
|
452
|
+
for (const step of investigations) expect(step.maxAttempts).toBeGreaterThan(3);
|
|
427
453
|
});
|
|
428
454
|
|
|
429
|
-
test("a rate limit is RETRYABLE, so the
|
|
455
|
+
test("a rate limit is RETRYABLE, so the engine tries again", async () => {
|
|
430
456
|
// The message alone cannot say this — a 429 and a 401 read alike — so what
|
|
431
|
-
// is asserted is the class the
|
|
457
|
+
// is asserted is the class the engine actually branches on.
|
|
432
458
|
stubGateway([""], { status: 429 });
|
|
433
459
|
const err = await investigate(brief, "Tool use").catch((thrown: unknown) => thrown);
|
|
434
460
|
expect(RetryableError.is(err)).toBe(true);
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
*
|
|
49
49
|
* ## What is NOT here
|
|
50
50
|
*
|
|
51
|
-
* No
|
|
52
|
-
*
|
|
53
|
-
* `
|
|
54
|
-
*
|
|
51
|
+
* No step function and no waitpoint. The body composes them with `ctx.step`
|
|
52
|
+
* and suspends with `ctx.sleep`; the functions themselves live in
|
|
53
|
+
* `workflows/research.ts`. What `agent.ts` owns is declaring the workflow and
|
|
54
|
+
* the two tools that start and read runs.
|
|
55
55
|
*
|
|
56
56
|
* ## The research is real, and it really searches the web
|
|
57
57
|
*
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
* so "both halves import the declaration" needs the declaration to have a home
|
|
11
11
|
* that is neither half.
|
|
12
12
|
*
|
|
13
|
-
* The
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* The BODY stays in `workflows/research.ts` by CONVENTION rather than by
|
|
14
|
+
* mechanism — nothing scans that directory any more, and a body reached with a
|
|
15
|
+
* `ctx` is durable wherever it is written. Keeping it there is what makes the
|
|
16
|
+
* declaration, the tools and the body findable from one another.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import { workflow } from "@alexkroman1/aai";
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*
|
|
26
26
|
* They are ADAPTED rather than copied: theirs are written for a LangGraph agent
|
|
27
27
|
* that calls tools by name and returns a long markdown report to a reader, and
|
|
28
|
-
* ours are written for a
|
|
28
|
+
* ours are written for a step that calls `webSearch` itself,
|
|
29
29
|
* returns JSON a later step consumes, and ends at a voice agent reading two
|
|
30
30
|
* sentences down a phone. What survives verbatim is the part that is the actual
|
|
31
31
|
* finding: the numbered stop rules, "repeat the useful text rather than
|
|
@@ -49,11 +49,11 @@
|
|
|
49
49
|
* researcher CONCLUDED, which is exactly what the step returns.
|
|
50
50
|
*/
|
|
51
51
|
|
|
52
|
+
import type { WorkflowCtx } from "@alexkroman1/aai";
|
|
52
53
|
import { mapConcurrent, report } from "@alexkroman1/aai/step";
|
|
53
54
|
import { stepGenerateClassified, stepGenerateJsonClassified } from "@alexkroman1/aai/step-errors";
|
|
54
55
|
import { visitWebpage, webSearch } from "@alexkroman1/aai/tools";
|
|
55
56
|
import { errorMessage, isToolFailure, plural } from "@alexkroman1/aai/utils";
|
|
56
|
-
import { sleep } from "workflow";
|
|
57
57
|
import { z } from "zod";
|
|
58
58
|
import {
|
|
59
59
|
BRIEF_SUMMARY_SYSTEM,
|
|
@@ -68,9 +68,8 @@ import {
|
|
|
68
68
|
/**
|
|
69
69
|
* Angles investigated at once. The far side of every one is a rate limit.
|
|
70
70
|
*
|
|
71
|
-
* Inside
|
|
72
|
-
*
|
|
73
|
-
* not the concurrency" in `mapConcurrent`.
|
|
71
|
+
* Inside `DEFAULT_STEP_CONCURRENCY` (`aai-runtime`, 16), so this width is what
|
|
72
|
+
* really runs — see "The WINDOW is not the concurrency" in `mapConcurrent`.
|
|
74
73
|
*/
|
|
75
74
|
const ANGLE_CONCURRENCY = 2;
|
|
76
75
|
|
|
@@ -82,7 +81,7 @@ const ANGLE_CONCURRENCY = 2;
|
|
|
82
81
|
* use; what makes either affordable is that the run is SUSPENDED rather than
|
|
83
82
|
* blocked, so the sandbox is free to exit and the run resumes when it comes due.
|
|
84
83
|
*/
|
|
85
|
-
const
|
|
84
|
+
export const REVIEW_DELAY_MS = 30_000;
|
|
86
85
|
|
|
87
86
|
/** Most angles a wave may carry, whatever the supervisor asks for. */
|
|
88
87
|
const MAX_ANGLES = 4;
|
|
@@ -208,35 +207,46 @@ export type Findings = {
|
|
|
208
207
|
* is for. `agent.ts` starts this with `notify`, so the agent says so when it
|
|
209
208
|
* lands rather than waiting to be asked.
|
|
210
209
|
*
|
|
211
|
-
* The `sleep` on top is the review wait — the one suspension in the template,
|
|
212
|
-
* and what `file_it_now` skips with `wakeUp`.
|
|
210
|
+
* The `ctx.sleep` on top is the review wait — the one suspension in the template,
|
|
211
|
+
* and what `file_it_now` skips with `ctx.workflows.wakeUp`.
|
|
213
212
|
*/
|
|
214
|
-
export async function researchFlow(
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
const
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
//
|
|
213
|
+
export async function researchFlow(
|
|
214
|
+
input: { topic: string; requestedBy: string },
|
|
215
|
+
ctx: WorkflowCtx,
|
|
216
|
+
) {
|
|
217
|
+
const brief = await ctx.step("writeBrief", () => writeBrief(input.topic));
|
|
218
|
+
const angles = await ctx.step("planAngles", () => planAngles(brief));
|
|
219
|
+
|
|
220
|
+
// One step per angle, bounded, in an order a replay reproduces exactly —
|
|
221
|
+
// `mapConcurrent` hands out items from a monotonic cursor, so the Nth call
|
|
222
|
+
// ISSUED is item N whatever order they settle in, and the Nth call is
|
|
223
|
+
// `investigate#N`. A failed angle fails the RUN: its finished siblings are
|
|
224
|
+
// already journaled, so the resume replays them for free and re-issues only
|
|
225
|
+
// what is missing, where catching here would file a report with a silent hole
|
|
226
|
+
// in it. `maxAttempts: 5` was `investigate.maxRetries = 4`.
|
|
224
227
|
const first = await mapConcurrent(angles, ANGLE_CONCURRENCY, (angle) =>
|
|
225
|
-
investigate(brief, angle),
|
|
228
|
+
ctx.step("investigate", () => investigate(brief, angle), { maxAttempts: 5 }),
|
|
226
229
|
);
|
|
227
230
|
|
|
228
231
|
// The supervisor's second look. Usually empty — a second wave costs the caller
|
|
229
232
|
// minutes, and the prompt says so.
|
|
230
|
-
const gaps = await findGaps(brief, first);
|
|
231
|
-
|
|
233
|
+
const gaps = await ctx.step("findGaps", () => findGaps(brief, first));
|
|
234
|
+
// A DIFFERENT step name from the first wave, though it calls the same
|
|
235
|
+
// function. Two waves under one name would share one occurrence counter, which
|
|
236
|
+
// is replay-safe (the waves are sequential, so the order is fixed) and reads
|
|
237
|
+
// terribly in a run's history: `investigate#7` would be the second wave's
|
|
238
|
+
// first angle with nothing saying so. The name is what an operator reads.
|
|
239
|
+
const second = await mapConcurrent(gaps, ANGLE_CONCURRENCY, (angle) =>
|
|
240
|
+
ctx.step("investigateGap", () => investigate(brief, angle), { maxAttempts: 5 }),
|
|
241
|
+
);
|
|
232
242
|
|
|
233
243
|
const notes = [...first, ...second];
|
|
234
|
-
const written = await writeReport(input.topic, brief, notes);
|
|
244
|
+
const written = await ctx.step("writeReport", () => writeReport(input.topic, brief, notes));
|
|
235
245
|
|
|
236
246
|
// Suspended, not blocked. On resume the body re-runs from the top and every
|
|
237
247
|
// step above returns its journaled result rather than researching again —
|
|
238
|
-
// which is also what `file_it_now` ends early, through `wakeUp`.
|
|
239
|
-
await sleep(
|
|
248
|
+
// which is also what `file_it_now` ends early, through `ctx.workflows.wakeUp`.
|
|
249
|
+
await ctx.sleep(REVIEW_DELAY_MS);
|
|
240
250
|
|
|
241
251
|
// Whatever this returns is what `ctx.workflows.get(runId)` reports as `output`
|
|
242
252
|
// on a completed run — so it is what the agent reads back, and what the
|
|
@@ -247,7 +257,7 @@ export async function researchFlow(input: { topic: string; requestedBy: string }
|
|
|
247
257
|
report: written.report,
|
|
248
258
|
sources: countSources(notes),
|
|
249
259
|
angles: notes.map((note) => note.angle),
|
|
250
|
-
filedAt: await file(input.requestedBy, input.topic),
|
|
260
|
+
filedAt: await ctx.step("file", () => file(input.requestedBy, input.topic)),
|
|
251
261
|
} satisfies Findings & { filedAt: string };
|
|
252
262
|
}
|
|
253
263
|
|
|
@@ -260,8 +270,6 @@ export async function researchFlow(input: { topic: string; requestedBy: string }
|
|
|
260
270
|
* later model call.
|
|
261
271
|
*/
|
|
262
272
|
export async function writeBrief(topic: string): Promise<Brief> {
|
|
263
|
-
"use step";
|
|
264
|
-
|
|
265
273
|
await report(`Working out what "${topic}" is really asking.`);
|
|
266
274
|
const parsed = await stepGenerateJsonClassified(
|
|
267
275
|
`Research request, as the caller said it: ${topic}`,
|
|
@@ -278,8 +286,6 @@ export async function writeBrief(topic: string): Promise<Brief> {
|
|
|
278
286
|
* asking the model again and getting a different one.
|
|
279
287
|
*/
|
|
280
288
|
export async function planAngles(brief: Brief): Promise<string[]> {
|
|
281
|
-
"use step";
|
|
282
|
-
|
|
283
289
|
const parsed = await stepGenerateJsonClassified(briefText(brief), {
|
|
284
290
|
system: PLAN_SYSTEM,
|
|
285
291
|
schema: AnglesReply,
|
|
@@ -303,8 +309,6 @@ export async function planAngles(brief: Brief): Promise<string[]> {
|
|
|
303
309
|
* at the end, which is where it becomes small enough to journal.
|
|
304
310
|
*/
|
|
305
311
|
export async function investigate(brief: Brief, angle: string): Promise<Note> {
|
|
306
|
-
"use step";
|
|
307
|
-
|
|
308
312
|
await report(`Looking into: ${angle}`);
|
|
309
313
|
const seen: string[] = [];
|
|
310
314
|
const sources: Source[] = [];
|
|
@@ -331,8 +335,7 @@ export async function investigate(brief: Brief, angle: string): Promise<Note> {
|
|
|
331
335
|
return await compress(angle, seen, sources);
|
|
332
336
|
}
|
|
333
337
|
|
|
334
|
-
/** Retries beyond the default: the far side is a search engine and a model. */
|
|
335
|
-
investigate.maxRetries = 4;
|
|
338
|
+
/** UNUSED-ORPHAN-MARKER Retries beyond the default: the far side is a search engine and a model. */
|
|
336
339
|
|
|
337
340
|
/**
|
|
338
341
|
* The supervisor's second look.
|
|
@@ -342,8 +345,6 @@ investigate.maxRetries = 4;
|
|
|
342
345
|
* caller who is told "still working" for twenty minutes.
|
|
343
346
|
*/
|
|
344
347
|
export async function findGaps(brief: Brief, notes: readonly Note[]): Promise<string[]> {
|
|
345
|
-
"use step";
|
|
346
|
-
|
|
347
348
|
if (notes.length === 0) return [];
|
|
348
349
|
const parsed = await stepGenerateJsonClassified(
|
|
349
350
|
`${briefText(brief)}\n\nWhat came back:\n${notes.map(noteText).join("\n\n")}`,
|
|
@@ -370,8 +371,6 @@ export async function writeReport(
|
|
|
370
371
|
brief: Brief,
|
|
371
372
|
notes: readonly Note[],
|
|
372
373
|
): Promise<{ report: string; summary: string }> {
|
|
373
|
-
"use step";
|
|
374
|
-
|
|
375
374
|
await report(`Writing up ${notes.length} ${plural(notes.length, "angle")}.`);
|
|
376
375
|
const written = await stepGenerateClassified(
|
|
377
376
|
`${briefText(brief)}\n\nFindings:\n${notes.map(noteText).join("\n\n")}`,
|
|
@@ -391,8 +390,6 @@ export async function writeReport(
|
|
|
391
390
|
* parameters carry `_` for the same reason.
|
|
392
391
|
*/
|
|
393
392
|
export async function file(_requestedBy: string, _topic: string): Promise<string> {
|
|
394
|
-
"use step";
|
|
395
|
-
|
|
396
393
|
await report("Filing the findings.");
|
|
397
394
|
return "filed";
|
|
398
395
|
}
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
// about the transcript and the summary rather than about audible audio.
|
|
42
42
|
//
|
|
43
43
|
// WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
|
|
44
|
-
// in the path, a
|
|
44
|
+
// in the path, a workflow body is an ordinary async function — no
|
|
45
45
|
// journal, no replay, and no per-step retry, so a rate-limited live run FAILS
|
|
46
46
|
// where a deployed one would have ridden it out, and the resume-replays-the-id
|
|
47
47
|
// property that makes speak-and-store ONE step is argued here rather than
|
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
57
57
|
import { expect } from "vitest";
|
|
58
58
|
import agentDef, { spokenSummary } from "./agent.ts";
|
|
59
|
-
import {
|
|
59
|
+
import { POLL_INTERVAL_MS } from "./workflows/transcribe.ts";
|
|
60
60
|
|
|
61
61
|
/** The id every case uploads the recording under. */
|
|
62
62
|
const UPLOAD_ID = "upl_eval";
|
|
@@ -271,7 +271,7 @@ describeWorkflowEval(
|
|
|
271
271
|
expect(run.output?.transcript).toBe(TRANSCRIPT);
|
|
272
272
|
// Two waits for three polls: asked for and — this being an eval rather
|
|
273
273
|
// than a deployment — recorded rather than taken.
|
|
274
|
-
expect(run.slept).toEqual([{ duration:
|
|
274
|
+
expect(run.slept).toEqual([{ duration: POLL_INTERVAL_MS }, { duration: POLL_INTERVAL_MS }]);
|
|
275
275
|
expect(provider.calls.filter((call) => call.leg === "poll")).toHaveLength(3);
|
|
276
276
|
// The expensive half happened ONCE, which is the whole reason the upload
|
|
277
277
|
// and the submit are separate steps.
|