@alexkroman1/aai-cli 13.1.0 → 13.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +1 -1
- package/dist/{_agent-BTwZJu4P.mjs → _agent-Clbq3a4h.mjs} +2 -2
- package/dist/{_bundler-C31rlcYz.mjs → _bundler-DEkjO1d8.mjs} +3 -3
- package/dist/{_config-CmJOFsAP.mjs → _config-ZY2euhdh.mjs} +7 -4
- package/dist/_config.d.ts +25 -1
- package/dist/{_dev-server-DWvkNoIS.mjs → _dev-server-C13xuGB1.mjs} +13 -13
- package/dist/_e2e-test-utils.d.ts +15 -7
- package/dist/{_init-BURbdHNM.mjs → _init-Ci4QGjp8.mjs} +58 -10
- package/dist/{_output-CKkmWs7i.mjs → _output-DbJt9gsl.mjs} +34 -3
- package/dist/_output.d.ts +17 -1
- package/dist/{_preflight-Co1G9Jww.mjs → _preflight-B2laWPdU.mjs} +1 -1
- package/dist/{_server-common-6e0QI4mq.mjs → _server-common-BV4_CrP3.mjs} +3 -3
- package/dist/{_slug-api-CfGQq1Tt.mjs → _slug-api-DlKpj91X.mjs} +3 -3
- package/dist/{_studio-B1waXMuR.mjs → _studio-DTi3EdXQ.mjs} +3 -3
- package/dist/{_templates-0Zr2z6yA.mjs → _templates-V4l5sZJy.mjs} +2 -2
- package/dist/{_typecheck-gate-BXvUNnfs.mjs → _typecheck-gate-sV4N5F93.mjs} +3 -3
- package/dist/{_ui-u7T4YooX.mjs → _ui-DCt4qZrk.mjs} +1 -1
- package/dist/{_utils-B8QmtFhK.mjs → _utils-Dhd058D9.mjs} +1 -1
- package/dist/{_vite-env-D9Qj6KPX.mjs → _vite-env-kejF0p2L.mjs} +1 -1
- package/dist/{_workflow-determinism-f9J-5Zlj.mjs → _workflow-determinism-C9l8uC2R.mjs} +1 -1
- package/dist/{build-BLlLOrng.mjs → build-CWyFGe-W.mjs} +11 -8
- package/dist/cli.d.ts +2 -1
- package/dist/cli.mjs +120 -46
- package/dist/{client-bundler-j5c46x0G.mjs → client-bundler-CT-_26c5.mjs} +4 -4
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-X_CRgCf6.mjs → delete-RqlQZ6Y_.mjs} +6 -6
- package/dist/{deploy-BzZ-r4iM.mjs → deploy-B7f3CK7v.mjs} +13 -13
- package/dist/{dev-BDykFvcc.mjs → dev-DamXieW_.mjs} +5 -5
- package/dist/{eval-0qEOdm0l.mjs → eval-h3DTVfRw.mjs} +27 -11
- package/dist/eval.d.ts +2 -0
- package/dist/{init-BtJI306C.mjs → init-D-vFrV8h.mjs} +7 -7
- package/dist/{login-DeUETobb.mjs → login-CDiemNmC.mjs} +5 -5
- package/dist/{logs-hWnJ66Bl.mjs → logs-DGwPNCzB.mjs} +4 -4
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/.env.example +17 -1
- package/dist/scaffold/package.json +6 -5
- package/dist/secret-V1w7y9-j.mjs +162 -0
- package/dist/secret.d.ts +41 -8
- package/dist/{studio-BxAS_FQQ.mjs → studio-BpMCfUNx.mjs} +9 -9
- package/dist/templates/briefing-desk/agent.test.ts +7 -3
- package/dist/templates/call-audit/agent.eval.test.ts +14 -11
- package/dist/templates/call-audit/agent.test.ts +34 -7
- package/dist/templates/call-audit/client.tsx +14 -17
- package/dist/templates/call-audit/workflows/ingest.ts +10 -1
- package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
- package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
- package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
- package/dist/templates/embedded-assets/agent.test.ts +7 -1
- package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
- package/dist/templates/health-assistant/agent.test.ts +8 -4
- package/dist/templates/link-digest/agent.eval.test.ts +24 -15
- package/dist/templates/link-digest/agent.test.ts +7 -1
- package/dist/templates/link-digest/client.tsx +17 -21
- package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
- package/dist/templates/night-owl/agent.eval.test.ts +30 -15
- package/dist/templates/night-owl/agent.test.ts +6 -1
- package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
- package/dist/templates/pipeline-simple/agent.test.ts +31 -14
- package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
- package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
- package/dist/templates/podcast-digest/agent.test.ts +13 -3
- package/dist/templates/podcast-digest/client.tsx +12 -15
- package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
- package/dist/templates/recap-workflow/agent.test.ts +22 -9
- package/dist/templates/redline/agent.eval.test.ts +32 -24
- package/dist/templates/redline/agent.test.ts +7 -1
- package/dist/templates/redline/client.tsx +12 -15
- package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
- package/dist/templates/research-workflow/agent.test.ts +7 -1
- package/dist/templates/retail/agent.eval.test.ts +18 -34
- package/dist/templates/retail/registry.test.ts +99 -58
- package/dist/templates/simple/agent.test.ts +39 -6
- package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
- package/dist/templates/spoken-summary/agent.test.ts +21 -7
- package/dist/templates/spoken-summary/client.tsx +14 -18
- package/dist/templates/support-line/agent.eval.test.ts +23 -26
- package/dist/templates/transcription-workflow/agent.test.ts +26 -8
- package/dist/templates/transcription-workflow/client.tsx +19 -26
- package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
- package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
- package/dist/test-guenJUxi.mjs +255 -0
- package/dist/test.d.ts +97 -31
- package/dist/typecheck.mjs +2 -2
- package/dist/worker-bundler.mjs +3 -3
- package/dist/{workflow-BBu5oEDw.mjs → workflow-BZyKLkhV.mjs} +5 -5
- package/package.json +10 -10
- package/dist/bin.d.mts +0 -2
- package/dist/secret-B-3T5tvr.mjs +0 -61
- package/dist/test-CiLab-AA.mjs +0 -182
- package/dist/tsdown.config.d.ts +0 -2
|
@@ -13,25 +13,42 @@ describe("pipeline-simple template", () => {
|
|
|
13
13
|
expect(() => toAgentConfig(agentDef)).not.toThrow();
|
|
14
14
|
});
|
|
15
15
|
|
|
16
|
-
test("
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
expect(agentDef.
|
|
22
|
-
expect(agentDef.
|
|
16
|
+
test("exports an agent the platform can name", () => {
|
|
17
|
+
// Not the literal. `aai init <project>` scaffolds this file verbatim, so a
|
|
18
|
+
// pinned name is a test about the TEMPLATE inside somebody else's project —
|
|
19
|
+
// and renaming the agent is the first thing they will do. What has to hold
|
|
20
|
+
// is that there is a name and the conversion carries it through.
|
|
21
|
+
expect(agentDef.name).toBeTruthy();
|
|
22
|
+
expect(toAgentConfig(agentDef).name).toBe(agentDef.name);
|
|
23
23
|
});
|
|
24
24
|
|
|
25
|
-
test("
|
|
26
|
-
|
|
25
|
+
test("declares at least one stage itself, which is what makes it a pipeline", () => {
|
|
26
|
+
// WHICH stage is deliberately not asserted: `agent.ts` invites you to move
|
|
27
|
+
// the declaration ("Declare only the stage you want elsewhere"), so naming
|
|
28
|
+
// one here would redden on the edit the comment above it asks for.
|
|
29
|
+
const declared = (["stt", "llm", "tts"] as const).filter(
|
|
30
|
+
(stage) => agentDef[stage] !== undefined,
|
|
31
|
+
);
|
|
32
|
+
expect(declared.length).toBeGreaterThan(0);
|
|
27
33
|
});
|
|
28
34
|
|
|
29
|
-
test("
|
|
35
|
+
test("a stage you declare survives, and every stage you leave unset defaults", () => {
|
|
36
|
+
// The template's whole subject, stated so it survives a swap: the config a
|
|
37
|
+
// deploy carries agrees with the def wherever the def has an opinion, and
|
|
38
|
+
// fills the AssemblyAI default wherever it does not.
|
|
30
39
|
const config = toAgentConfig(agentDef);
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
40
|
+
if (config.mode !== "pipeline") {
|
|
41
|
+
// Switched the def to `s2s`? Then there is no cascade to fill — S2S
|
|
42
|
+
// REPLACES the pipeline rather than joining it, which the last describe
|
|
43
|
+
// below is the worked example for.
|
|
44
|
+
expect(config.mode).toBe("s2s");
|
|
45
|
+
expect(config.stt).toBeUndefined();
|
|
46
|
+
expect(config.tts).toBeUndefined();
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
for (const stage of ["stt", "llm", "tts"] as const) {
|
|
50
|
+
expect(config[stage]?.kind, stage).toBe(agentDef[stage]?.kind ?? "assemblyai");
|
|
51
|
+
}
|
|
35
52
|
});
|
|
36
53
|
});
|
|
37
54
|
|
|
@@ -13,7 +13,7 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
|
|
|
13
13
|
// SCRIPTED model (its `stubReply`), which still boots this agent, still
|
|
14
14
|
// resolves `tools/`, and still executes the tool a script names — so a stub run
|
|
15
15
|
// proves the wiring and proves nothing about what the agent chose.
|
|
16
|
-
import { lastStateIn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
|
|
16
|
+
import { lastStateIn, statesIn, toolNames, toolResultIn } from "@alexkroman1/aai-runtime/eval";
|
|
17
17
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
18
18
|
import { expect } from "vitest";
|
|
19
19
|
import { z } from "zod";
|
|
@@ -44,9 +44,14 @@ const ProjectedOrder = z.object({
|
|
|
44
44
|
*/
|
|
45
45
|
const lastPushedView = (events: readonly SessionEvent[]) => lastStateIn(events, ProjectedOrder);
|
|
46
46
|
|
|
47
|
-
/**
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
/**
|
|
48
|
+
* Every cart the session pushed, in stream order — `statesIn` is `lastStateIn`'s
|
|
49
|
+
* plural half, and takes the schema for the same reason.
|
|
50
|
+
*
|
|
51
|
+
* The SEQUENCE is the stronger claim: not "the cart is not placed now" but "no
|
|
52
|
+
* frame the customer ever saw showed it placed".
|
|
53
|
+
*/
|
|
54
|
+
const pushedViews = (events: readonly SessionEvent[]) => statesIn(events, ProjectedOrder);
|
|
50
55
|
|
|
51
56
|
describeEval(agentDef, (test) => {
|
|
52
57
|
test(
|
|
@@ -58,7 +63,7 @@ describeEval(agentDef, (test) => {
|
|
|
58
63
|
|
|
59
64
|
// One tool, and the right one: quoting a price without adding the pizza,
|
|
60
65
|
// or adding it twice, are both real findings.
|
|
61
|
-
expect(turn.toolCalls
|
|
66
|
+
expect(toolNames(turn.toolCalls)).toEqual(["add_pizza"]);
|
|
62
67
|
const call = turn.toolCalls[0]!;
|
|
63
68
|
const args = call.args as { size: string; crust: string; toppings: string[] };
|
|
64
69
|
expect(args.size).toBe("large");
|
|
@@ -106,7 +111,7 @@ describeEval(agentDef, (test) => {
|
|
|
106
111
|
// turn leaves the model with nothing to address.
|
|
107
112
|
const turn = await session.say("Actually, make that a large.");
|
|
108
113
|
|
|
109
|
-
expect(turn.toolCalls
|
|
114
|
+
expect(toolNames(turn.toolCalls)).toEqual(["update_pizza"]);
|
|
110
115
|
const call = turn.toolCalls[0]!;
|
|
111
116
|
expect(call.args).toMatchObject({ pizza_id: 1, size: "large" });
|
|
112
117
|
|
|
@@ -24,7 +24,13 @@
|
|
|
24
24
|
|
|
25
25
|
/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
|
|
26
26
|
import agentDef from "virtual:aai/agent";
|
|
27
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
describeToolCalls,
|
|
29
|
+
describeTurn,
|
|
30
|
+
type EvalSession,
|
|
31
|
+
lastStateIn,
|
|
32
|
+
toolNames,
|
|
33
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
28
34
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
29
35
|
import { expect } from "vitest";
|
|
30
36
|
import { z } from "zod";
|
|
@@ -57,8 +63,6 @@ const ProjectedPlan = z.object({
|
|
|
57
63
|
*/
|
|
58
64
|
const planState = (session: EvalSession) => lastStateIn(session.events(), ProjectedPlan);
|
|
59
65
|
|
|
60
|
-
const named = (calls: readonly { name: string }[]): string[] => calls.map((call) => call.name);
|
|
61
|
-
|
|
62
66
|
describeEval(agentDef, (test) => {
|
|
63
67
|
test(
|
|
64
68
|
"the stage the desk reports is the flow's, not a guess at the plan",
|
|
@@ -122,7 +126,10 @@ describeEval(agentDef, (test) => {
|
|
|
122
126
|
"I want to work out whether it is cheaper to take the train or fly from London to Edinburgh next month.",
|
|
123
127
|
);
|
|
124
128
|
const started = session.toolCalls().find((call) => call.name === "start_plan");
|
|
125
|
-
|
|
129
|
+
// The whole SESSION's calls, not one turn's: the plan may be started on
|
|
130
|
+
// either utterance, and "expected undefined to be defined" says nothing
|
|
131
|
+
// about a desk that talked instead. `describeToolCalls` is that sentence.
|
|
132
|
+
expect(started, describeToolCalls(session.toolCalls())).toBeDefined();
|
|
126
133
|
const planned = planState(session);
|
|
127
134
|
// The tool's own result rides in the message: a planner that FAILED (a
|
|
128
135
|
// gateway error, a schema the provider would not honour) writes nothing,
|
|
@@ -131,8 +138,8 @@ describeEval(agentDef, (test) => {
|
|
|
131
138
|
expect(planned?.plan.length ?? 0).toBeGreaterThan(0);
|
|
132
139
|
|
|
133
140
|
const worked = await session.say("Yes, go ahead and start on it.");
|
|
134
|
-
const calls =
|
|
135
|
-
expect(calls,
|
|
141
|
+
const calls = toolNames(worked.toolCalls).filter((name) => name === "work_next_step").length;
|
|
142
|
+
expect(calls, describeTurn(worked)).toBeGreaterThan(0);
|
|
136
143
|
|
|
137
144
|
const after = planState(session);
|
|
138
145
|
const done = after?.done ?? [];
|
|
@@ -180,7 +187,7 @@ describeEval(agentDef, (test) => {
|
|
|
180
187
|
const revised = turn.toolCalls.find((call) => call.name === "revise_plan");
|
|
181
188
|
expect(
|
|
182
189
|
revised,
|
|
183
|
-
|
|
190
|
+
`${describeTurn(turn)} — the caller changed the objective, ` +
|
|
184
191
|
"so this is `revise_plan`, not a plan rewritten by hand",
|
|
185
192
|
).toBeDefined();
|
|
186
193
|
expect(revised?.args.instruction).toBeTruthy();
|
|
@@ -103,15 +103,25 @@ const slackInput = (slackWebhookUrl: string) => ({
|
|
|
103
103
|
|
|
104
104
|
describe("the declaration", () => {
|
|
105
105
|
test("is a workflow app with one workflow and a static page", () => {
|
|
106
|
-
|
|
107
|
-
|
|
106
|
+
// Not the agent's name: renaming it is an invited edit, and pinning the
|
|
107
|
+
// literal here would fail a build in somebody else's project.
|
|
108
|
+
expect(agentDef.name).toBeTruthy();
|
|
109
|
+
// `toContain` rather than an exact key list: adding a second workflow is an
|
|
110
|
+
// invited edit and must not redden a test the author did not write. The
|
|
111
|
+
// NAME is still pinned, deliberately — the page starts a run by this
|
|
112
|
+
// string, so renaming the key is a runtime 400 rather than a compile
|
|
113
|
+
// error, and this pin is the only thing that says so. Rename it here and
|
|
114
|
+
// in `client.tsx` together.
|
|
115
|
+
expect(Object.keys(agentDef.workflows ?? {})).toContain("dailyDigest");
|
|
108
116
|
expect(agentDef.workflows?.dailyDigest).toBe(dailyDigest);
|
|
109
117
|
});
|
|
110
118
|
|
|
111
119
|
test("names the one credential its steps read", () => {
|
|
112
120
|
// A workflow app declares no providers, so nothing else in the config can
|
|
113
121
|
// name this — which is what makes a deploy able to check for it.
|
|
114
|
-
|
|
122
|
+
// `toContain`, so a step of your own that reads a second credential can be
|
|
123
|
+
// declared beside this one without failing a test you did not write.
|
|
124
|
+
expect(agentDef.requiredEnv).toContain("ASSEMBLYAI_API_KEY");
|
|
115
125
|
});
|
|
116
126
|
});
|
|
117
127
|
|
|
@@ -17,12 +17,13 @@
|
|
|
17
17
|
* minutes, this one loses the only Stop button a thirty-digest commitment has,
|
|
18
18
|
* and the run is invisible from then on to everything but `curl`.
|
|
19
19
|
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* Wake and Cancel buttons bound to
|
|
20
|
+
* A correlation KEY is the handle that survives, and `useWorkflowSubmit` looks
|
|
21
|
+
* that key's newest run up as it mounts, so a later load lands on the same
|
|
22
|
+
* count, the same newest line, and the same Wake and Cancel buttons bound to
|
|
23
|
+
* the same run.
|
|
23
24
|
*
|
|
24
|
-
* **
|
|
25
|
-
*
|
|
25
|
+
* **This is the one workflow app that passes a `key` of its own, and it is
|
|
26
|
+
* `useRunKey({ storage: "local" })`.** Its siblings let the hook mint one into
|
|
26
27
|
* `sessionStorage`, which dies with the tab and covers exactly the interruption
|
|
27
28
|
* they have — a reload, a same-tab navigation, a crashed tab. A schedule
|
|
28
29
|
* outlives all of that by design: closing the browser on Tuesday and coming
|
|
@@ -42,9 +43,9 @@
|
|
|
42
43
|
* URL would carry a credential into a lookup token — which is why the
|
|
43
44
|
* platform stopped writing keys to the operator's log.
|
|
44
45
|
*
|
|
45
|
-
* A real app with accounts passes the account's own id instead, and
|
|
46
|
-
* schedule follows the person to a new device — a promise only a login
|
|
47
|
-
* keep.
|
|
46
|
+
* A real app with accounts passes the account's own id as `key` instead, and
|
|
47
|
+
* then the schedule follows the person to a new device — a promise only a login
|
|
48
|
+
* can keep.
|
|
48
49
|
*
|
|
49
50
|
* ## The form is DECLARED, not written
|
|
50
51
|
*
|
|
@@ -135,15 +136,11 @@ export function App() {
|
|
|
135
136
|
// `run.output` instead of `unknown`. `error` is the agent's own sentence for a
|
|
136
137
|
// rejected input — better copy than anything this page could write, and the
|
|
137
138
|
// reason there is no `try`/`catch` here.
|
|
139
|
+
// The key REPLACES the tab-scoped one the hook would mint; the lookup that
|
|
140
|
+
// reads it back on the next load happens either way.
|
|
138
141
|
const { submitForm, run, pending, error, wake, cancel } = useWorkflowSubmit<typeof dailyDigest>(
|
|
139
142
|
WORKFLOW,
|
|
140
|
-
{
|
|
141
|
-
// Recorded with the run, and read back on the next load. Neither half is
|
|
142
|
-
// useful alone: without the key there is nothing to find the run by, and
|
|
143
|
-
// without `recover` the key is only ever written.
|
|
144
|
-
key,
|
|
145
|
-
recover: true,
|
|
146
|
-
},
|
|
143
|
+
{ key },
|
|
147
144
|
);
|
|
148
145
|
|
|
149
146
|
return (
|
|
@@ -62,8 +62,15 @@
|
|
|
62
62
|
* this file SHIPS — see `agent.test.ts`.
|
|
63
63
|
*/
|
|
64
64
|
import agentDef from "virtual:aai/agent";
|
|
65
|
+
import { stubGatewayRoute } from "@alexkroman1/aai/testing";
|
|
65
66
|
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
66
|
-
import
|
|
67
|
+
import {
|
|
68
|
+
describeToolCalls,
|
|
69
|
+
type EvalToolCall,
|
|
70
|
+
type EvalWorkflows,
|
|
71
|
+
toolResultIn,
|
|
72
|
+
toolResultsIn,
|
|
73
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
67
74
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
68
75
|
import { expect } from "vitest";
|
|
69
76
|
import { z } from "zod";
|
|
@@ -116,17 +123,24 @@ type ScriptedProvider = {
|
|
|
116
123
|
* rather than answering an empty 200: a step calling something nobody expected
|
|
117
124
|
* is a finding, where an empty body reads as a provider that said nothing.
|
|
118
125
|
*
|
|
126
|
+
* The model leg is `stubGatewayRoute`'s, first, because it is the one leg whose
|
|
127
|
+
* shape this file cannot check: the completion envelope is a WIRE shape, so a
|
|
128
|
+
* field typed one off does not fail — `stepGenerate` reads no content, reports
|
|
129
|
+
* an empty completion, and the case blames the recap. The reader routes off the
|
|
130
|
+
* SDK's own completions PATH and answers `undefined` for everything else, which
|
|
131
|
+
* is what lets it sit in front of the three batch legs below it.
|
|
132
|
+
*
|
|
119
133
|
* `hold` keeps the FIRST poll pending, and it is the only way to observe a run
|
|
120
134
|
* that is still going: a durable `sleep` is skipped here, so an unheld run
|
|
121
135
|
* burns its whole poll loop in milliseconds.
|
|
122
136
|
*/
|
|
123
137
|
function stubProvider(options: { hold?: boolean; ending?: Ending } = {}): ScriptedProvider {
|
|
124
138
|
const gate = Promise.withResolvers<void>();
|
|
139
|
+
const model = stubGatewayRoute(RECAP_JSON);
|
|
125
140
|
let polls = 0;
|
|
126
141
|
const stub = installStubStepFetch(async (request) => {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
}
|
|
142
|
+
const recapped = model.route(request);
|
|
143
|
+
if (recapped) return recapped;
|
|
130
144
|
if (request.method === "POST") return { body: { id: TRANSCRIPT_ID, status: "queued" } };
|
|
131
145
|
// The compensation. A real DELETE removes the transcript from the account,
|
|
132
146
|
// which is what makes "a failed run leaves nothing behind" a claim rather
|
|
@@ -158,21 +172,32 @@ const Cancelled = z.object({ cancelled: z.boolean(), note: z.string() });
|
|
|
158
172
|
*
|
|
159
173
|
* Parsed rather than regexed: a tool result reaches the event stream as a
|
|
160
174
|
* serialized string, and a shape that stopped matching should fail HERE naming
|
|
161
|
-
* the field instead of handing the next assertion `undefined`.
|
|
175
|
+
* the field instead of handing the next assertion `undefined`. That is
|
|
176
|
+
* `toolResultsIn`'s job, and the half it does better than the filter-and-map
|
|
177
|
+
* this was: a call with no result THROWS naming its position, where
|
|
178
|
+
* `one.result !== undefined` dropped it — so a tool that was called and never
|
|
179
|
+
* returned left a shorter list and a case that read the calls it did get.
|
|
162
180
|
*/
|
|
163
|
-
function recapStarts(calls: readonly EvalToolCall[]): z.infer<typeof RecapStart>[] {
|
|
164
|
-
return calls
|
|
165
|
-
.filter((one) => one.name === "request_recap" && one.result !== undefined)
|
|
166
|
-
.map((one) => RecapStart.parse(JSON.parse(String(one.result))));
|
|
181
|
+
function recapStarts(calls: readonly EvalToolCall[]): readonly z.infer<typeof RecapStart>[] {
|
|
182
|
+
return toolResultsIn(calls, "request_recap", RecapStart);
|
|
167
183
|
}
|
|
168
184
|
|
|
169
|
-
/**
|
|
185
|
+
/**
|
|
186
|
+
* The run id the FIRST `request_recap` of this turn reported.
|
|
187
|
+
*
|
|
188
|
+
* `recapStarts` rather than `toolResultIn`: a turn is allowed more than one call
|
|
189
|
+
* here — the case below says so in as many words, a desk that asks twice being
|
|
190
|
+
* the model's business — and the singular reader refuses a second one on
|
|
191
|
+
* purpose. What is asserted about the extras is that each was REFUSED with the
|
|
192
|
+
* run this found.
|
|
193
|
+
*/
|
|
170
194
|
function startedRunId(calls: readonly EvalToolCall[]): string {
|
|
171
195
|
const [first] = recapStarts(calls);
|
|
172
196
|
if (first === undefined) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
197
|
+
// `describeToolCalls` is the harness's own sentence for this — including
|
|
198
|
+
// "called no tools", which is the answer a desk that asked a question
|
|
199
|
+
// instead of acting gives and the one an empty list reads as truncation.
|
|
200
|
+
throw new Error(`no request_recap in this turn: ${describeToolCalls(calls)}`);
|
|
176
201
|
}
|
|
177
202
|
return first.runId;
|
|
178
203
|
}
|
|
@@ -207,14 +232,19 @@ const START_TURN = [
|
|
|
207
232
|
* Not tidiness: the scripted provider is unpublished when the test that
|
|
208
233
|
* installed it finishes, so a body still mid-flight would make its next request
|
|
209
234
|
* against whatever the next case publishes — or against the real provider, with
|
|
210
|
-
* a real key.
|
|
211
|
-
*
|
|
212
|
-
*
|
|
213
|
-
*
|
|
235
|
+
* a real key. `close()` says so on stderr (`EvalRunAbandoned`) when a case
|
|
236
|
+
* forgets, which is a report rather than a fix — the wait is `settleAll`'s and
|
|
237
|
+
* the RELEASE is this template's, because what holds the run in flight is this
|
|
238
|
+
* file's own gate and nothing in the harness can open one.
|
|
239
|
+
*
|
|
240
|
+
* A run drained here COMPLETES rather than failing, and the last thing it does
|
|
241
|
+
* on the way is delete its own transcript: the retention gate's window closes
|
|
242
|
+
* with nobody having answered, which is the safe default. See the header, and
|
|
243
|
+
* the case that pins it.
|
|
214
244
|
*/
|
|
215
245
|
async function drain(workflows: EvalWorkflows | undefined, provider: ScriptedProvider) {
|
|
216
246
|
provider.release();
|
|
217
|
-
|
|
247
|
+
await workflows?.settleAll();
|
|
218
248
|
}
|
|
219
249
|
|
|
220
250
|
describeEval(
|
|
@@ -328,8 +358,7 @@ describeEval(
|
|
|
328
358
|
const runId = startedRunId(started.toolCalls);
|
|
329
359
|
const turn = await session.say("Forget it — cancel that, please.");
|
|
330
360
|
|
|
331
|
-
const
|
|
332
|
-
const answer = Cancelled.parse(JSON.parse(String(cancel?.result)));
|
|
361
|
+
const answer = toolResultIn(turn.toolCalls, "cancel_recap", Cancelled);
|
|
333
362
|
expect(answer.cancelled).toBe(true);
|
|
334
363
|
// The sentence is a documented promise of this template, not a
|
|
335
364
|
// decoration: cancellation is NOT cooperative here, so the transcript
|
|
@@ -116,7 +116,13 @@ function finishedOutput(over: { kept?: boolean; answered?: boolean } = {}) {
|
|
|
116
116
|
|
|
117
117
|
describe("the agent declares its workflow", () => {
|
|
118
118
|
test("under the name ctx.workflows.start resolves it by", () => {
|
|
119
|
-
|
|
119
|
+
// `toContain` rather than an exact key list: a second workflow is an
|
|
120
|
+
// invited edit and must not redden a test the author did not write. The
|
|
121
|
+
// NAME is still pinned, deliberately — this key is a STRING to everything
|
|
122
|
+
// outside this file (the REST route, `ctx.workflows.get`, a schedule), so
|
|
123
|
+
// renaming it is a runtime 404 rather than a compile error, and nothing
|
|
124
|
+
// else says so.
|
|
125
|
+
expect(Object.keys(agentDef.workflows ?? {})).toContain("recap");
|
|
120
126
|
expect(agentDef.workflows?.recap).toBe(recap);
|
|
121
127
|
});
|
|
122
128
|
|
|
@@ -143,14 +149,21 @@ describe("the agent declares its workflow", () => {
|
|
|
143
149
|
});
|
|
144
150
|
|
|
145
151
|
test("discovers every tool in tools/, by file name", () => {
|
|
146
|
-
// Discovered, not declared: every name here is a file in `tools/`.
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
152
|
+
// Discovered, not declared: every name here is a file in `tools/`. Asserted
|
|
153
|
+
// with `arrayContaining`, because dropping a file into `tools/` is the
|
|
154
|
+
// cheapest edit this template invites and an exact list would turn it into
|
|
155
|
+
// a failing test in somebody else's project. What still fails is a tool
|
|
156
|
+
// going MISSING — which is what a broken discovery looks like, and it looks
|
|
157
|
+
// identical to a template that never had tools.
|
|
158
|
+
expect(Object.keys(agentDef.tools)).toEqual(
|
|
159
|
+
expect.arrayContaining([
|
|
160
|
+
"cancel_recap",
|
|
161
|
+
"keep_transcript",
|
|
162
|
+
"recap_progress",
|
|
163
|
+
"recap_status",
|
|
164
|
+
"request_recap",
|
|
165
|
+
]),
|
|
166
|
+
);
|
|
154
167
|
});
|
|
155
168
|
});
|
|
156
169
|
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
// journal, no replay, and no per-step retry, so a rate-limited live run FAILS
|
|
30
30
|
// where a deployed one would have ridden it out. The tier that really resumes a
|
|
31
31
|
// run is `aai-cli`'s `dev-workflow.scenario.test.ts`.
|
|
32
|
+
import { stubGatewayRoute } from "@alexkroman1/aai/testing";
|
|
32
33
|
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
33
34
|
import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
34
35
|
import { expect } from "vitest";
|
|
@@ -52,32 +53,36 @@ const critique = (verdict: "ship" | "revise", score = 8): string =>
|
|
|
52
53
|
notes: verdict === "ship" ? [] : ["Say what happens to the handover", "Name the start date"],
|
|
53
54
|
});
|
|
54
55
|
|
|
55
|
-
/** One gateway reply, in the envelope `stepGenerate` reads. */
|
|
56
|
-
const reply = (content: string) => ({ body: { choices: [{ message: { content } }] } });
|
|
57
|
-
|
|
58
56
|
/**
|
|
59
57
|
* Answer the gateway with `contents`, in order, and record what each stage asked.
|
|
60
58
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
59
|
+
* `stubGatewayRoute` owns both halves this file used to hand-write. The
|
|
60
|
+
* ENVELOPE, because it is a WIRE shape and a field typed one off does not fail —
|
|
61
|
+
* `stepGenerate` reads no content, reports an empty completion, and the case
|
|
62
|
+
* blames the loop. And the CURSOR: the last reply repeats, because a loop cannot
|
|
63
|
+
* know how many calls it will make, so a script that said one thing forever
|
|
64
|
+
* could only ever drive it into its budget and one that ran out mid-loop would
|
|
65
|
+
* fail on the script rather than on the code. Both of those are now the SDK's
|
|
66
|
+
* one copy rather than this file's second.
|
|
67
|
+
*
|
|
68
|
+
* `installStubStepFetch` rather than `installStubGateway`: `stepGenerate` goes
|
|
69
|
+
* through the published `stepFetch` slot, and a published slot BEATS a stubbed
|
|
70
|
+
* global, so stubbing the global here would test a path production does not
|
|
71
|
+
* take. Anything that is not a completion request THROWS — every step in this
|
|
72
|
+
* body is a model call, so a request the route does not recognise is a finding,
|
|
73
|
+
* and answering it with a reply anyway is how a stage that started dialling
|
|
74
|
+
* something else would pass.
|
|
67
75
|
*/
|
|
68
76
|
function scriptGateway(contents: readonly string[]) {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
77
|
+
const model = stubGatewayRoute(contents);
|
|
78
|
+
installStubStepFetch((request) => {
|
|
79
|
+
const answered = model.route(request);
|
|
80
|
+
if (answered === undefined) {
|
|
81
|
+
throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
|
|
82
|
+
}
|
|
83
|
+
return answered;
|
|
74
84
|
});
|
|
75
|
-
return
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/** Every prompt the gateway was sent, in call order. */
|
|
79
|
-
function promptsOf(fetched: ReturnType<typeof scriptGateway>): string[] {
|
|
80
|
-
return fetched.calls.map((call) => String(call.body ?? ""));
|
|
85
|
+
return model;
|
|
81
86
|
}
|
|
82
87
|
|
|
83
88
|
describeWorkflowEval(agentDef, (test) => {
|
|
@@ -140,7 +145,7 @@ describeWorkflowEval(agentDef, (test) => {
|
|
|
140
145
|
// running, and asking it and then accepting whatever it says is not evidence
|
|
141
146
|
// about the budget. What this pins is the loop's arithmetic — the half a
|
|
142
147
|
// live case cannot reach.
|
|
143
|
-
const
|
|
148
|
+
const model = scriptGateway([
|
|
144
149
|
DRAFT,
|
|
145
150
|
critique("revise", 4),
|
|
146
151
|
`${DRAFT} It starts on the first Monday of the month.`,
|
|
@@ -170,12 +175,15 @@ describeWorkflowEval(agentDef, (test) => {
|
|
|
170
175
|
).toBe(true);
|
|
171
176
|
// One draft plus a critique-and-revise pair per round. A loop that critiqued
|
|
172
177
|
// twice, or revised the round it shipped, changes this number.
|
|
173
|
-
expect(
|
|
178
|
+
expect(model.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
|
|
174
179
|
|
|
175
180
|
// `briefBlock` is what keeps the three stages from drifting apart, and this is
|
|
176
181
|
// the assertion behind that claim: the writer, the critic AND the reviser were
|
|
177
|
-
// all shown the same brief and the same must-cover point.
|
|
178
|
-
|
|
182
|
+
// all shown the same brief and the same must-cover point. Read off the
|
|
183
|
+
// recorded `prompt` — the USER message — rather than off the raw request
|
|
184
|
+
// body, which is the whole serialized request and would let a `model` id or a
|
|
185
|
+
// `temperature` satisfy one of these `toContain`s.
|
|
186
|
+
const prompts = model.calls.map((call) => call.prompt);
|
|
179
187
|
expect(prompts).toHaveLength(1 + 2 * MAX_ROUNDS);
|
|
180
188
|
for (const prompt of prompts) {
|
|
181
189
|
expect(prompt).toContain("quokka");
|
|
@@ -46,7 +46,13 @@ describe("the agent declares itself a workflow app", () => {
|
|
|
46
46
|
test("under the name the page starts a run by", () => {
|
|
47
47
|
// `useWorkflowSubmit("redline")` in client.tsx names this key. Nothing else
|
|
48
48
|
// records it, so a rename here is a 400 there rather than a compile error.
|
|
49
|
-
|
|
49
|
+
// `toContain` rather than an exact key list: adding a second workflow is an
|
|
50
|
+
// invited edit and must not redden a test the author did not write. The
|
|
51
|
+
// NAME is still pinned, deliberately — the page starts a run by this
|
|
52
|
+
// string, so renaming the key is a runtime 400 rather than a compile
|
|
53
|
+
// error, and this pin is the only thing that says so. Rename it here and
|
|
54
|
+
// in `client.tsx` together.
|
|
55
|
+
expect(Object.keys(agentDef.workflows ?? {})).toContain("redline");
|
|
50
56
|
expect(agentDef.workflows?.redline).toBe(redline);
|
|
51
57
|
});
|
|
52
58
|
|
|
@@ -31,11 +31,13 @@
|
|
|
31
31
|
* in React state — so a refresh lost it while the loop carried on writing,
|
|
32
32
|
* grading and revising without anywhere to report to. On a desk whose whole
|
|
33
33
|
* subject is a loop that runs several long-form model calls, that is the one
|
|
34
|
-
* failure
|
|
35
|
-
*
|
|
36
|
-
* Clear button are
|
|
34
|
+
* failure the hook now covers on its own: `useWorkflowSubmit` records every run
|
|
35
|
+
* under a correlation KEY it mints for this page and asks for that key's newest
|
|
36
|
+
* run as it mounts, so the draft, the critique trail and the Clear button are
|
|
37
|
+
* all there again with nothing written here.
|
|
37
38
|
*
|
|
38
|
-
* **The key is opaque and lives in `sessionStorage`, and the brief is
|
|
39
|
+
* **The key it mints is opaque and lives in `sessionStorage`, and the brief is
|
|
40
|
+
* why this page wants exactly that one.** A
|
|
39
41
|
* `?key=` parameter in the page's own URL would survive more — a new tab, a
|
|
40
42
|
* bookmark, a link sent to the person who asked for the piece — and that is
|
|
41
43
|
* exactly what it must not do here. There is no per-user filtering behind
|
|
@@ -48,7 +50,9 @@
|
|
|
48
50
|
*
|
|
49
51
|
* Deriving the key from the brief is worse again: two people briefing the same
|
|
50
52
|
* thing would recover each other's runs, and the key would then carry what they
|
|
51
|
-
* typed into a lookup token the platform deliberately stopped logging.
|
|
53
|
+
* typed into a lookup token the platform deliberately stopped logging. Both are
|
|
54
|
+
* things a page could still ask for by passing its own `key`, and this one has
|
|
55
|
+
* no reason to.
|
|
52
56
|
*/
|
|
53
57
|
|
|
54
58
|
import "@alexkroman1/aai-ui/styles.css";
|
|
@@ -60,7 +64,6 @@ import {
|
|
|
60
64
|
page,
|
|
61
65
|
SubmitButton,
|
|
62
66
|
TextAreaField,
|
|
63
|
-
useRunKey,
|
|
64
67
|
useWorkflowSubmit,
|
|
65
68
|
WORKFLOW_STATUS_LABELS,
|
|
66
69
|
WorkflowFields,
|
|
@@ -121,19 +124,13 @@ export function toInput(values: FormValues): WorkflowInputOf<typeof redline> {
|
|
|
121
124
|
}
|
|
122
125
|
|
|
123
126
|
function RedlineDesk() {
|
|
124
|
-
// This tab's handle on its own drafts — opaque, short, and remembered for the
|
|
125
|
-
// next load, which is what `recover` produces to find the run again.
|
|
126
|
-
const key = useRunKey();
|
|
127
127
|
// Did THIS load start the run? A reload cannot have, and that is the only way
|
|
128
128
|
// the page can tell "writing what you just briefed" from "picking up where
|
|
129
129
|
// you left off" — the hook reports the run, not who asked for it.
|
|
130
130
|
const [startedHere, setStartedHere] = useState(false);
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
key,
|
|
135
|
-
recover: true,
|
|
136
|
-
});
|
|
131
|
+
// The reload is covered by the hook's own key — see the module doc for why
|
|
132
|
+
// this desk wants the tab-scoped one it mints rather than a key of its own.
|
|
133
|
+
const { submit, run, pending, error, reset } = useWorkflowSubmit<typeof redline>(WORKFLOW);
|
|
137
134
|
|
|
138
135
|
return (
|
|
139
136
|
<main className="mx-auto flex max-w-2xl flex-col gap-8 p-8">
|