@alexkroman1/aai-cli 13.1.0 → 13.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/bin.mjs +1 -1
  2. package/dist/{_agent-BTwZJu4P.mjs → _agent-Clbq3a4h.mjs} +2 -2
  3. package/dist/{_bundler-C31rlcYz.mjs → _bundler-DEkjO1d8.mjs} +3 -3
  4. package/dist/{_config-CmJOFsAP.mjs → _config-ZY2euhdh.mjs} +7 -4
  5. package/dist/_config.d.ts +25 -1
  6. package/dist/{_dev-server-DWvkNoIS.mjs → _dev-server-C13xuGB1.mjs} +13 -13
  7. package/dist/_e2e-test-utils.d.ts +15 -7
  8. package/dist/{_init-BURbdHNM.mjs → _init-Ci4QGjp8.mjs} +58 -10
  9. package/dist/{_output-CKkmWs7i.mjs → _output-DbJt9gsl.mjs} +34 -3
  10. package/dist/_output.d.ts +17 -1
  11. package/dist/{_preflight-Co1G9Jww.mjs → _preflight-B2laWPdU.mjs} +1 -1
  12. package/dist/{_server-common-6e0QI4mq.mjs → _server-common-BV4_CrP3.mjs} +3 -3
  13. package/dist/{_slug-api-CfGQq1Tt.mjs → _slug-api-DlKpj91X.mjs} +3 -3
  14. package/dist/{_studio-B1waXMuR.mjs → _studio-DTi3EdXQ.mjs} +3 -3
  15. package/dist/{_templates-0Zr2z6yA.mjs → _templates-V4l5sZJy.mjs} +2 -2
  16. package/dist/{_typecheck-gate-BXvUNnfs.mjs → _typecheck-gate-sV4N5F93.mjs} +3 -3
  17. package/dist/{_ui-u7T4YooX.mjs → _ui-DCt4qZrk.mjs} +1 -1
  18. package/dist/{_utils-B8QmtFhK.mjs → _utils-Dhd058D9.mjs} +1 -1
  19. package/dist/{_vite-env-D9Qj6KPX.mjs → _vite-env-kejF0p2L.mjs} +1 -1
  20. package/dist/{_workflow-determinism-f9J-5Zlj.mjs → _workflow-determinism-C9l8uC2R.mjs} +1 -1
  21. package/dist/{build-BLlLOrng.mjs → build-CWyFGe-W.mjs} +11 -8
  22. package/dist/cli.d.ts +2 -1
  23. package/dist/cli.mjs +120 -46
  24. package/dist/{client-bundler-j5c46x0G.mjs → client-bundler-CT-_26c5.mjs} +4 -4
  25. package/dist/client-bundler.mjs +1 -1
  26. package/dist/{delete-X_CRgCf6.mjs → delete-RqlQZ6Y_.mjs} +6 -6
  27. package/dist/{deploy-BzZ-r4iM.mjs → deploy-B7f3CK7v.mjs} +13 -13
  28. package/dist/{dev-BDykFvcc.mjs → dev-DamXieW_.mjs} +5 -5
  29. package/dist/{eval-0qEOdm0l.mjs → eval-h3DTVfRw.mjs} +27 -11
  30. package/dist/eval.d.ts +2 -0
  31. package/dist/{init-BtJI306C.mjs → init-D-vFrV8h.mjs} +7 -7
  32. package/dist/{login-DeUETobb.mjs → login-CDiemNmC.mjs} +5 -5
  33. package/dist/{logs-hWnJ66Bl.mjs → logs-DGwPNCzB.mjs} +4 -4
  34. package/dist/project-config.mjs +1 -1
  35. package/dist/scaffold/.env.example +17 -1
  36. package/dist/scaffold/package.json +6 -5
  37. package/dist/secret-V1w7y9-j.mjs +162 -0
  38. package/dist/secret.d.ts +41 -8
  39. package/dist/{studio-BxAS_FQQ.mjs → studio-BpMCfUNx.mjs} +9 -9
  40. package/dist/templates/briefing-desk/agent.test.ts +7 -3
  41. package/dist/templates/call-audit/agent.eval.test.ts +14 -11
  42. package/dist/templates/call-audit/agent.test.ts +34 -7
  43. package/dist/templates/call-audit/client.tsx +14 -17
  44. package/dist/templates/call-audit/workflows/ingest.ts +10 -1
  45. package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
  46. package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
  47. package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
  48. package/dist/templates/embedded-assets/agent.test.ts +7 -1
  49. package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
  50. package/dist/templates/health-assistant/agent.test.ts +8 -4
  51. package/dist/templates/link-digest/agent.eval.test.ts +24 -15
  52. package/dist/templates/link-digest/agent.test.ts +7 -1
  53. package/dist/templates/link-digest/client.tsx +17 -21
  54. package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
  55. package/dist/templates/night-owl/agent.eval.test.ts +30 -15
  56. package/dist/templates/night-owl/agent.test.ts +6 -1
  57. package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
  58. package/dist/templates/pipeline-simple/agent.test.ts +31 -14
  59. package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
  60. package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
  61. package/dist/templates/podcast-digest/agent.test.ts +13 -3
  62. package/dist/templates/podcast-digest/client.tsx +12 -15
  63. package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
  64. package/dist/templates/recap-workflow/agent.test.ts +22 -9
  65. package/dist/templates/redline/agent.eval.test.ts +32 -24
  66. package/dist/templates/redline/agent.test.ts +7 -1
  67. package/dist/templates/redline/client.tsx +12 -15
  68. package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
  69. package/dist/templates/research-workflow/agent.test.ts +7 -1
  70. package/dist/templates/retail/agent.eval.test.ts +18 -34
  71. package/dist/templates/retail/registry.test.ts +99 -58
  72. package/dist/templates/simple/agent.test.ts +39 -6
  73. package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
  74. package/dist/templates/spoken-summary/agent.test.ts +21 -7
  75. package/dist/templates/spoken-summary/client.tsx +14 -18
  76. package/dist/templates/support-line/agent.eval.test.ts +23 -26
  77. package/dist/templates/transcription-workflow/agent.test.ts +26 -8
  78. package/dist/templates/transcription-workflow/client.tsx +19 -26
  79. package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
  80. package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
  81. package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
  82. package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
  83. package/dist/test-guenJUxi.mjs +255 -0
  84. package/dist/test.d.ts +97 -31
  85. package/dist/typecheck.mjs +2 -2
  86. package/dist/worker-bundler.mjs +3 -3
  87. package/dist/{workflow-BBu5oEDw.mjs → workflow-BZyKLkhV.mjs} +5 -5
  88. package/package.json +10 -10
  89. package/dist/bin.d.mts +0 -2
  90. package/dist/secret-B-3T5tvr.mjs +0 -61
  91. package/dist/test-CiLab-AA.mjs +0 -182
  92. package/dist/tsdown.config.d.ts +0 -2
@@ -13,25 +13,42 @@ describe("pipeline-simple template", () => {
13
13
  expect(() => toAgentConfig(agentDef)).not.toThrow();
14
14
  });
15
15
 
16
- test("declares only the LLM stage on the def", () => {
17
- // The template's point: swap one stage, leave the rest unset. The
18
- // unset stages are filled with the AssemblyAI defaults at parse time.
19
- expect(agentDef.name).toBe("pipeline-simple");
20
- expect(agentDef.llm).toBeDefined();
21
- expect(agentDef.stt).toBeUndefined();
22
- expect(agentDef.tts).toBeUndefined();
16
+ test("exports an agent the platform can name", () => {
17
+ // Not the literal. `aai init <project>` scaffolds this file verbatim, so a
18
+ // pinned name is a test about the TEMPLATE inside somebody else's project —
19
+ // and renaming the agent is the first thing they will do. What has to hold
20
+ // is that there is a name and the conversion carries it through.
21
+ expect(agentDef.name).toBeTruthy();
22
+ expect(toAgentConfig(agentDef).name).toBe(agentDef.name);
23
23
  });
24
24
 
25
- test("LLM descriptor is Anthropic", () => {
26
- expect(agentDef.llm?.kind).toBe("anthropic");
25
+ test("declares at least one stage itself, which is what makes it a pipeline", () => {
26
+ // WHICH stage is deliberately not asserted: `agent.ts` invites you to move
27
+ // the declaration ("Declare only the stage you want elsewhere"), so naming
28
+ // one here would redden on the edit the comment above it asks for.
29
+ const declared = (["stt", "llm", "tts"] as const).filter(
30
+ (stage) => agentDef[stage] !== undefined,
31
+ );
32
+ expect(declared.length).toBeGreaterThan(0);
27
33
  });
28
34
 
29
- test("unset stages fill to AssemblyAI in the deployable config", () => {
35
+ test("a stage you declare survives, and every stage you leave unset defaults", () => {
36
+ // The template's whole subject, stated so it survives a swap: the config a
37
+ // deploy carries agrees with the def wherever the def has an opinion, and
38
+ // fills the AssemblyAI default wherever it does not.
30
39
  const config = toAgentConfig(agentDef);
31
- expect(config.mode).toBe("pipeline");
32
- expect(config.stt?.kind).toBe("assemblyai");
33
- expect(config.tts?.kind).toBe("assemblyai");
34
- expect(config.llm?.kind).toBe("anthropic");
40
+ if (config.mode !== "pipeline") {
41
+ // Switched the def to `s2s`? Then there is no cascade to fill — S2S
42
+ // REPLACES the pipeline rather than joining it, which the last describe
43
+ // below is the worked example for.
44
+ expect(config.mode).toBe("s2s");
45
+ expect(config.stt).toBeUndefined();
46
+ expect(config.tts).toBeUndefined();
47
+ return;
48
+ }
49
+ for (const stage of ["stt", "llm", "tts"] as const) {
50
+ expect(config[stage]?.kind, stage).toBe(agentDef[stage]?.kind ?? "assemblyai");
51
+ }
35
52
  });
36
53
  });
37
54
 
@@ -13,7 +13,7 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
13
13
  // SCRIPTED model (its `stubReply`), which still boots this agent, still
14
14
  // resolves `tools/`, and still executes the tool a script names — so a stub run
15
15
  // proves the wiring and proves nothing about what the agent chose.
16
- import { lastStateIn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
16
+ import { lastStateIn, statesIn, toolNames, toolResultIn } from "@alexkroman1/aai-runtime/eval";
17
17
  import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
18
18
  import { expect } from "vitest";
19
19
  import { z } from "zod";
@@ -44,9 +44,14 @@ const ProjectedOrder = z.object({
44
44
  */
45
45
  const lastPushedView = (events: readonly SessionEvent[]) => lastStateIn(events, ProjectedOrder);
46
46
 
47
- /** Every cart the session pushed, in stream order. */
48
- const pushedViews = (events: readonly SessionEvent[]) =>
49
- events.flatMap((e) => (e.type === "state.updated" ? [ProjectedOrder.parse(e.state)] : []));
47
+ /**
48
+ * Every cart the session pushed, in stream order — `statesIn` is `lastStateIn`'s
49
+ * plural half, and takes the schema for the same reason.
50
+ *
51
+ * The SEQUENCE is the stronger claim: not "the cart is not placed now" but "no
52
+ * frame the customer ever saw showed it placed".
53
+ */
54
+ const pushedViews = (events: readonly SessionEvent[]) => statesIn(events, ProjectedOrder);
50
55
 
51
56
  describeEval(agentDef, (test) => {
52
57
  test(
@@ -58,7 +63,7 @@ describeEval(agentDef, (test) => {
58
63
 
59
64
  // One tool, and the right one: quoting a price without adding the pizza,
60
65
  // or adding it twice, are both real findings.
61
- expect(turn.toolCalls.map((c) => c.name)).toEqual(["add_pizza"]);
66
+ expect(toolNames(turn.toolCalls)).toEqual(["add_pizza"]);
62
67
  const call = turn.toolCalls[0]!;
63
68
  const args = call.args as { size: string; crust: string; toppings: string[] };
64
69
  expect(args.size).toBe("large");
@@ -106,7 +111,7 @@ describeEval(agentDef, (test) => {
106
111
  // turn leaves the model with nothing to address.
107
112
  const turn = await session.say("Actually, make that a large.");
108
113
 
109
- expect(turn.toolCalls.map((c) => c.name)).toEqual(["update_pizza"]);
114
+ expect(toolNames(turn.toolCalls)).toEqual(["update_pizza"]);
110
115
  const call = turn.toolCalls[0]!;
111
116
  expect(call.args).toMatchObject({ pizza_id: 1, size: "large" });
112
117
 
@@ -24,7 +24,13 @@
24
24
 
25
25
  /** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
26
26
  import agentDef from "virtual:aai/agent";
27
- import { type EvalSession, lastStateIn } from "@alexkroman1/aai-runtime/eval";
27
+ import {
28
+ describeToolCalls,
29
+ describeTurn,
30
+ type EvalSession,
31
+ lastStateIn,
32
+ toolNames,
33
+ } from "@alexkroman1/aai-runtime/eval";
28
34
  import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
29
35
  import { expect } from "vitest";
30
36
  import { z } from "zod";
@@ -57,8 +63,6 @@ const ProjectedPlan = z.object({
57
63
  */
58
64
  const planState = (session: EvalSession) => lastStateIn(session.events(), ProjectedPlan);
59
65
 
60
- const named = (calls: readonly { name: string }[]): string[] => calls.map((call) => call.name);
61
-
62
66
  describeEval(agentDef, (test) => {
63
67
  test(
64
68
  "the stage the desk reports is the flow's, not a guess at the plan",
@@ -122,7 +126,10 @@ describeEval(agentDef, (test) => {
122
126
  "I want to work out whether it is cheaper to take the train or fly from London to Edinburgh next month.",
123
127
  );
124
128
  const started = session.toolCalls().find((call) => call.name === "start_plan");
125
- expect(started, `tools called: ${named(session.toolCalls()).join(", ")}`).toBeDefined();
129
+ // The whole SESSION's calls, not one turn's: the plan may be started on
130
+ // either utterance, and "expected undefined to be defined" says nothing
131
+ // about a desk that talked instead. `describeToolCalls` is that sentence.
132
+ expect(started, describeToolCalls(session.toolCalls())).toBeDefined();
126
133
  const planned = planState(session);
127
134
  // The tool's own result rides in the message: a planner that FAILED (a
128
135
  // gateway error, a schema the provider would not honour) writes nothing,
@@ -131,8 +138,8 @@ describeEval(agentDef, (test) => {
131
138
  expect(planned?.plan.length ?? 0).toBeGreaterThan(0);
132
139
 
133
140
  const worked = await session.say("Yes, go ahead and start on it.");
134
- const calls = named(worked.toolCalls).filter((name) => name === "work_next_step").length;
135
- expect(calls, `tools called: [${named(worked.toolCalls).join(", ")}]`).toBeGreaterThan(0);
141
+ const calls = toolNames(worked.toolCalls).filter((name) => name === "work_next_step").length;
142
+ expect(calls, describeTurn(worked)).toBeGreaterThan(0);
136
143
 
137
144
  const after = planState(session);
138
145
  const done = after?.done ?? [];
@@ -180,7 +187,7 @@ describeEval(agentDef, (test) => {
180
187
  const revised = turn.toolCalls.find((call) => call.name === "revise_plan");
181
188
  expect(
182
189
  revised,
183
- `tools called: ${named(turn.toolCalls).join(", ")} — the caller changed the objective, ` +
190
+ `${describeTurn(turn)} — the caller changed the objective, ` +
184
191
  "so this is `revise_plan`, not a plan rewritten by hand",
185
192
  ).toBeDefined();
186
193
  expect(revised?.args.instruction).toBeTruthy();
@@ -103,15 +103,25 @@ const slackInput = (slackWebhookUrl: string) => ({
103
103
 
104
104
  describe("the declaration", () => {
105
105
  test("is a workflow app with one workflow and a static page", () => {
106
- expect(agentDef.name).toBe("Podcast Digest");
107
- expect(Object.keys(agentDef.workflows ?? {})).toEqual(["dailyDigest"]);
106
+ // Not the agent's name: renaming it is an invited edit, and pinning the
107
+ // literal here would fail a build in somebody else's project.
108
+ expect(agentDef.name).toBeTruthy();
109
+ // `toContain` rather than an exact key list: adding a second workflow is an
110
+ // invited edit and must not redden a test the author did not write. The
111
+ // NAME is still pinned, deliberately — the page starts a run by this
112
+ // string, so renaming the key is a runtime 400 rather than a compile
113
+ // error, and this pin is the only thing that says so. Rename it here and
114
+ // in `client.tsx` together.
115
+ expect(Object.keys(agentDef.workflows ?? {})).toContain("dailyDigest");
108
116
  expect(agentDef.workflows?.dailyDigest).toBe(dailyDigest);
109
117
  });
110
118
 
111
119
  test("names the one credential its steps read", () => {
112
120
  // A workflow app declares no providers, so nothing else in the config can
113
121
  // name this — which is what makes a deploy able to check for it.
114
- expect(agentDef.requiredEnv).toEqual(["ASSEMBLYAI_API_KEY"]);
122
+ // `toContain`, so a step of your own that reads a second credential can be
123
+ // declared beside this one without failing a test you did not write.
124
+ expect(agentDef.requiredEnv).toContain("ASSEMBLYAI_API_KEY");
115
125
  });
116
126
  });
117
127
 
@@ -17,12 +17,13 @@
17
17
  * minutes, this one loses the only Stop button a thirty-digest commitment has,
18
18
  * and the run is invisible from then on to everything but `curl`.
19
19
  *
20
- * `key` is the handle that survives and `recover: true` is what reads it back,
21
- * so a later load lands on the same count, the same newest line, and the same
22
- * Wake and Cancel buttons bound to the same run.
20
+ * A correlation KEY is the handle that survives, and `useWorkflowSubmit` looks
21
+ * that key's newest run up as it mounts, so a later load lands on the same
22
+ * count, the same newest line, and the same Wake and Cancel buttons bound to
23
+ * the same run.
23
24
  *
24
- * **The key is `useRunKey({ storage: "local" })`, and this is the one workflow
25
- * app that passes that option.** Its siblings take the default,
25
+ * **This is the one workflow app that passes a `key` of its own, and it is
26
+ * `useRunKey({ storage: "local" })`.** Its siblings let the hook mint one into
26
27
  * `sessionStorage`, which dies with the tab and covers exactly the interruption
27
28
  * they have — a reload, a same-tab navigation, a crashed tab. A schedule
28
29
  * outlives all of that by design: closing the browser on Tuesday and coming
@@ -42,9 +43,9 @@
42
43
  * URL would carry a credential into a lookup token — which is why the
43
44
  * platform stopped writing keys to the operator's log.
44
45
  *
45
- * A real app with accounts passes the account's own id instead, and then the
46
- * schedule follows the person to a new device — a promise only a login can
47
- * keep.
46
+ * A real app with accounts passes the account's own id as `key` instead, and
47
+ * then the schedule follows the person to a new device — a promise only a login
48
+ * can keep.
48
49
  *
49
50
  * ## The form is DECLARED, not written
50
51
  *
@@ -135,15 +136,11 @@ export function App() {
135
136
  // `run.output` instead of `unknown`. `error` is the agent's own sentence for a
136
137
  // rejected input — better copy than anything this page could write, and the
137
138
  // reason there is no `try`/`catch` here.
139
+ // The key REPLACES the tab-scoped one the hook would mint; the lookup that
140
+ // reads it back on the next load happens either way.
138
141
  const { submitForm, run, pending, error, wake, cancel } = useWorkflowSubmit<typeof dailyDigest>(
139
142
  WORKFLOW,
140
- {
141
- // Recorded with the run, and read back on the next load. Neither half is
142
- // useful alone: without the key there is nothing to find the run by, and
143
- // without `recover` the key is only ever written.
144
- key,
145
- recover: true,
146
- },
143
+ { key },
147
144
  );
148
145
 
149
146
  return (
@@ -62,8 +62,15 @@
62
62
  * this file SHIPS — see `agent.test.ts`.
63
63
  */
64
64
  import agentDef from "virtual:aai/agent";
65
+ import { stubGatewayRoute } from "@alexkroman1/aai/testing";
65
66
  import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
66
- import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
67
+ import {
68
+ describeToolCalls,
69
+ type EvalToolCall,
70
+ type EvalWorkflows,
71
+ toolResultIn,
72
+ toolResultsIn,
73
+ } from "@alexkroman1/aai-runtime/eval";
67
74
  import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
68
75
  import { expect } from "vitest";
69
76
  import { z } from "zod";
@@ -116,17 +123,24 @@ type ScriptedProvider = {
116
123
  * rather than answering an empty 200: a step calling something nobody expected
117
124
  * is a finding, where an empty body reads as a provider that said nothing.
118
125
  *
126
+ * The model leg is `stubGatewayRoute`'s, first, because it is the one leg whose
127
+ * shape this file cannot check: the completion envelope is a WIRE shape, so a
128
+ * field typed one off does not fail — `stepGenerate` reads no content, reports
129
+ * an empty completion, and the case blames the recap. The reader routes off the
130
+ * SDK's own completions PATH and answers `undefined` for everything else, which
131
+ * is what lets it sit in front of the three batch legs below it.
132
+ *
119
133
  * `hold` keeps the FIRST poll pending, and it is the only way to observe a run
120
134
  * that is still going: a durable `sleep` is skipped here, so an unheld run
121
135
  * burns its whole poll loop in milliseconds.
122
136
  */
123
137
  function stubProvider(options: { hold?: boolean; ending?: Ending } = {}): ScriptedProvider {
124
138
  const gate = Promise.withResolvers<void>();
139
+ const model = stubGatewayRoute(RECAP_JSON);
125
140
  let polls = 0;
126
141
  const stub = installStubStepFetch(async (request) => {
127
- if (request.url.includes("/chat/completions")) {
128
- return { body: { choices: [{ message: { content: RECAP_JSON } }] } };
129
- }
142
+ const recapped = model.route(request);
143
+ if (recapped) return recapped;
130
144
  if (request.method === "POST") return { body: { id: TRANSCRIPT_ID, status: "queued" } };
131
145
  // The compensation. A real DELETE removes the transcript from the account,
132
146
  // which is what makes "a failed run leaves nothing behind" a claim rather
@@ -158,21 +172,32 @@ const Cancelled = z.object({ cancelled: z.boolean(), note: z.string() });
158
172
  *
159
173
  * Parsed rather than regexed: a tool result reaches the event stream as a
160
174
  * serialized string, and a shape that stopped matching should fail HERE naming
161
- * the field instead of handing the next assertion `undefined`.
175
+ * the field instead of handing the next assertion `undefined`. That is
176
+ * `toolResultsIn`'s job, and the half it does better than the filter-and-map
177
+ * this was: a call with no result THROWS naming its position, where
178
+ * `one.result !== undefined` dropped it — so a tool that was called and never
179
+ * returned left a shorter list and a case that read the calls it did get.
162
180
  */
163
- function recapStarts(calls: readonly EvalToolCall[]): z.infer<typeof RecapStart>[] {
164
- return calls
165
- .filter((one) => one.name === "request_recap" && one.result !== undefined)
166
- .map((one) => RecapStart.parse(JSON.parse(String(one.result))));
181
+ function recapStarts(calls: readonly EvalToolCall[]): readonly z.infer<typeof RecapStart>[] {
182
+ return toolResultsIn(calls, "request_recap", RecapStart);
167
183
  }
168
184
 
169
- /** The run id the first `request_recap` of this turn reported. */
185
+ /**
186
+ * The run id the FIRST `request_recap` of this turn reported.
187
+ *
188
+ * `recapStarts` rather than `toolResultIn`: a turn is allowed more than one call
189
+ * here — the case below says so in as many words, a desk that asks twice being
190
+ * the model's business — and the singular reader refuses a second one on
191
+ * purpose. What is asserted about the extras is that each was REFUSED with the
192
+ * run this found.
193
+ */
170
194
  function startedRunId(calls: readonly EvalToolCall[]): string {
171
195
  const [first] = recapStarts(calls);
172
196
  if (first === undefined) {
173
- throw new Error(
174
- `the desk called no request_recap: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
175
- );
197
+ // `describeToolCalls` is the harness's own sentence for this — including
198
+ // "called no tools", which is the answer a desk that asked a question
199
+ // instead of acting gives and the one an empty list reads as truncation.
200
+ throw new Error(`no request_recap in this turn: ${describeToolCalls(calls)}`);
176
201
  }
177
202
  return first.runId;
178
203
  }
@@ -207,14 +232,19 @@ const START_TURN = [
207
232
  * Not tidiness: the scripted provider is unpublished when the test that
208
233
  * installed it finishes, so a body still mid-flight would make its next request
209
234
  * against whatever the next case publishes — or against the real provider, with
210
- * a real key. A run drained here COMPLETES rather than failing, and the last
211
- * thing it does on the way is delete its own transcript: the retention gate's
212
- * window closes with nobody having answered, which is the safe default. See the
213
- * header, and the case that pins it.
235
+ * a real key. `close()` says so on stderr (`EvalRunAbandoned`) when a case
236
+ * forgets, which is a report rather than a fix the wait is `settleAll`'s and
237
+ * the RELEASE is this template's, because what holds the run in flight is this
238
+ * file's own gate and nothing in the harness can open one.
239
+ *
240
+ * A run drained here COMPLETES rather than failing, and the last thing it does
241
+ * on the way is delete its own transcript: the retention gate's window closes
242
+ * with nobody having answered, which is the safe default. See the header, and
243
+ * the case that pins it.
214
244
  */
215
245
  async function drain(workflows: EvalWorkflows | undefined, provider: ScriptedProvider) {
216
246
  provider.release();
217
- for (const run of await (workflows?.runs() ?? [])) await workflows?.settle(run.runId, recap);
247
+ await workflows?.settleAll();
218
248
  }
219
249
 
220
250
  describeEval(
@@ -328,8 +358,7 @@ describeEval(
328
358
  const runId = startedRunId(started.toolCalls);
329
359
  const turn = await session.say("Forget it — cancel that, please.");
330
360
 
331
- const cancel = turn.toolCalls.find((one) => one.name === "cancel_recap");
332
- const answer = Cancelled.parse(JSON.parse(String(cancel?.result)));
361
+ const answer = toolResultIn(turn.toolCalls, "cancel_recap", Cancelled);
333
362
  expect(answer.cancelled).toBe(true);
334
363
  // The sentence is a documented promise of this template, not a
335
364
  // decoration: cancellation is NOT cooperative here, so the transcript
@@ -116,7 +116,13 @@ function finishedOutput(over: { kept?: boolean; answered?: boolean } = {}) {
116
116
 
117
117
  describe("the agent declares its workflow", () => {
118
118
  test("under the name ctx.workflows.start resolves it by", () => {
119
- expect(Object.keys(agentDef.workflows ?? {})).toEqual(["recap"]);
119
+ // `toContain` rather than an exact key list: a second workflow is an
120
+ // invited edit and must not redden a test the author did not write. The
121
+ // NAME is still pinned, deliberately — this key is a STRING to everything
122
+ // outside this file (the REST route, `ctx.workflows.get`, a schedule), so
123
+ // renaming it is a runtime 404 rather than a compile error, and nothing
124
+ // else says so.
125
+ expect(Object.keys(agentDef.workflows ?? {})).toContain("recap");
120
126
  expect(agentDef.workflows?.recap).toBe(recap);
121
127
  });
122
128
 
@@ -143,14 +149,21 @@ describe("the agent declares its workflow", () => {
143
149
  });
144
150
 
145
151
  test("discovers every tool in tools/, by file name", () => {
146
- // Discovered, not declared: every name here is a file in `tools/`.
147
- expect(Object.keys(agentDef.tools).sort()).toEqual([
148
- "cancel_recap",
149
- "keep_transcript",
150
- "recap_progress",
151
- "recap_status",
152
- "request_recap",
153
- ]);
152
+ // Discovered, not declared: every name here is a file in `tools/`. Asserted
153
+ // with `arrayContaining`, because dropping a file into `tools/` is the
154
+ // cheapest edit this template invites and an exact list would turn it into
155
+ // a failing test in somebody else's project. What still fails is a tool
156
+ // going MISSING — which is what a broken discovery looks like, and it looks
157
+ // identical to a template that never had tools.
158
+ expect(Object.keys(agentDef.tools)).toEqual(
159
+ expect.arrayContaining([
160
+ "cancel_recap",
161
+ "keep_transcript",
162
+ "recap_progress",
163
+ "recap_status",
164
+ "request_recap",
165
+ ]),
166
+ );
154
167
  });
155
168
  });
156
169
 
@@ -29,6 +29,7 @@
29
29
  // journal, no replay, and no per-step retry, so a rate-limited live run FAILS
30
30
  // where a deployed one would have ridden it out. The tier that really resumes a
31
31
  // run is `aai-cli`'s `dev-workflow.scenario.test.ts`.
32
+ import { stubGatewayRoute } from "@alexkroman1/aai/testing";
32
33
  import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
33
34
  import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
34
35
  import { expect } from "vitest";
@@ -52,32 +53,36 @@ const critique = (verdict: "ship" | "revise", score = 8): string =>
52
53
  notes: verdict === "ship" ? [] : ["Say what happens to the handover", "Name the start date"],
53
54
  });
54
55
 
55
- /** One gateway reply, in the envelope `stepGenerate` reads. */
56
- const reply = (content: string) => ({ body: { choices: [{ message: { content } }] } });
57
-
58
56
  /**
59
57
  * Answer the gateway with `contents`, in order, and record what each stage asked.
60
58
  *
61
- * The last reply repeats, matching `stubGateway`'s convention a loop cannot
62
- * know how many calls it will make, and a script that ran out mid-loop would
63
- * fail on the script rather than on the code. `installStubStepFetch` rather than
64
- * `installStubGateway`: `stepGenerate` goes through the published `stepFetch`
65
- * slot, and a published slot BEATS a stubbed global, so stubbing the global here
66
- * would test a path production does not take.
59
+ * `stubGatewayRoute` owns both halves this file used to hand-write. The
60
+ * ENVELOPE, because it is a WIRE shape and a field typed one off does not fail —
61
+ * `stepGenerate` reads no content, reports an empty completion, and the case
62
+ * blames the loop. And the CURSOR: the last reply repeats, because a loop cannot
63
+ * know how many calls it will make, so a script that said one thing forever
64
+ * could only ever drive it into its budget and one that ran out mid-loop would
65
+ * fail on the script rather than on the code. Both of those are now the SDK's
66
+ * one copy rather than this file's second.
67
+ *
68
+ * `installStubStepFetch` rather than `installStubGateway`: `stepGenerate` goes
69
+ * through the published `stepFetch` slot, and a published slot BEATS a stubbed
70
+ * global, so stubbing the global here would test a path production does not
71
+ * take. Anything that is not a completion request THROWS — every step in this
72
+ * body is a model call, so a request the route does not recognise is a finding,
73
+ * and answering it with a reply anyway is how a stage that started dialling
74
+ * something else would pass.
67
75
  */
68
76
  function scriptGateway(contents: readonly string[]) {
69
- let next = 0;
70
- const fetched = installStubStepFetch(() => {
71
- const content = contents.at(Math.min(next, contents.length - 1)) ?? "";
72
- next += 1;
73
- return reply(content);
77
+ const model = stubGatewayRoute(contents);
78
+ installStubStepFetch((request) => {
79
+ const answered = model.route(request);
80
+ if (answered === undefined) {
81
+ throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
82
+ }
83
+ return answered;
74
84
  });
75
- return fetched;
76
- }
77
-
78
- /** Every prompt the gateway was sent, in call order. */
79
- function promptsOf(fetched: ReturnType<typeof scriptGateway>): string[] {
80
- return fetched.calls.map((call) => String(call.body ?? ""));
85
+ return model;
81
86
  }
82
87
 
83
88
  describeWorkflowEval(agentDef, (test) => {
@@ -140,7 +145,7 @@ describeWorkflowEval(agentDef, (test) => {
140
145
  // running, and asking it and then accepting whatever it says is not evidence
141
146
  // about the budget. What this pins is the loop's arithmetic — the half a
142
147
  // live case cannot reach.
143
- const fetched = scriptGateway([
148
+ const model = scriptGateway([
144
149
  DRAFT,
145
150
  critique("revise", 4),
146
151
  `${DRAFT} It starts on the first Monday of the month.`,
@@ -170,12 +175,15 @@ describeWorkflowEval(agentDef, (test) => {
170
175
  ).toBe(true);
171
176
  // One draft plus a critique-and-revise pair per round. A loop that critiqued
172
177
  // twice, or revised the round it shipped, changes this number.
173
- expect(fetched.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
178
+ expect(model.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
174
179
 
175
180
  // `briefBlock` is what keeps the three stages from drifting apart, and this is
176
181
  // the assertion behind that claim: the writer, the critic AND the reviser were
177
- // all shown the same brief and the same must-cover point.
178
- const prompts = promptsOf(fetched);
182
+ // all shown the same brief and the same must-cover point. Read off the
183
+ // recorded `prompt` — the USER message — rather than off the raw request
184
+ // body, which is the whole serialized request and would let a `model` id or a
185
+ // `temperature` satisfy one of these `toContain`s.
186
+ const prompts = model.calls.map((call) => call.prompt);
179
187
  expect(prompts).toHaveLength(1 + 2 * MAX_ROUNDS);
180
188
  for (const prompt of prompts) {
181
189
  expect(prompt).toContain("quokka");
@@ -46,7 +46,13 @@ describe("the agent declares itself a workflow app", () => {
46
46
  test("under the name the page starts a run by", () => {
47
47
  // `useWorkflowSubmit("redline")` in client.tsx names this key. Nothing else
48
48
  // records it, so a rename here is a 400 there rather than a compile error.
49
- expect(Object.keys(agentDef.workflows ?? {})).toEqual(["redline"]);
49
+ // `toContain` rather than an exact key list: adding a second workflow is an
50
+ // invited edit and must not redden a test the author did not write. The
51
+ // NAME is still pinned, deliberately — the page starts a run by this
52
+ // string, so renaming the key is a runtime 400 rather than a compile
53
+ // error, and this pin is the only thing that says so. Rename it here and
54
+ // in `client.tsx` together.
55
+ expect(Object.keys(agentDef.workflows ?? {})).toContain("redline");
50
56
  expect(agentDef.workflows?.redline).toBe(redline);
51
57
  });
52
58
 
@@ -31,11 +31,13 @@
31
31
  * in React state — so a refresh lost it while the loop carried on writing,
32
32
  * grading and revising without anywhere to report to. On a desk whose whole
33
33
  * subject is a loop that runs several long-form model calls, that is the one
34
- * failure worth two lines of wiring: `key` is the handle that survives a reload
35
- * and `recover` is what reads it back, so the draft, the critique trail and the
36
- * Clear button are all there again.
34
+ * failure the hook now covers on its own: `useWorkflowSubmit` records every run
35
+ * under a correlation KEY it mints for this page and asks for that key's newest
36
+ * run as it mounts, so the draft, the critique trail and the Clear button are
37
+ * all there again with nothing written here.
37
38
  *
38
- * **The key is opaque and lives in `sessionStorage`, and the brief is why.** A
39
+ * **The key it mints is opaque and lives in `sessionStorage`, and the brief is
40
+ * why this page wants exactly that one.** A
39
41
  * `?key=` parameter in the page's own URL would survive more — a new tab, a
40
42
  * bookmark, a link sent to the person who asked for the piece — and that is
41
43
  * exactly what it must not do here. There is no per-user filtering behind
@@ -48,7 +50,9 @@
48
50
  *
49
51
  * Deriving the key from the brief is worse again: two people briefing the same
50
52
  * thing would recover each other's runs, and the key would then carry what they
51
- * typed into a lookup token the platform deliberately stopped logging.
53
+ * typed into a lookup token the platform deliberately stopped logging. Both are
54
+ * things a page could still ask for by passing its own `key`, and this one has
55
+ * no reason to.
52
56
  */
53
57
 
54
58
  import "@alexkroman1/aai-ui/styles.css";
@@ -60,7 +64,6 @@ import {
60
64
  page,
61
65
  SubmitButton,
62
66
  TextAreaField,
63
- useRunKey,
64
67
  useWorkflowSubmit,
65
68
  WORKFLOW_STATUS_LABELS,
66
69
  WorkflowFields,
@@ -121,19 +124,13 @@ export function toInput(values: FormValues): WorkflowInputOf<typeof redline> {
121
124
  }
122
125
 
123
126
  function RedlineDesk() {
124
- // This tab's handle on its own drafts — opaque, short, and remembered for the
125
- // next load, which is what `recover` produces to find the run again.
126
- const key = useRunKey();
127
127
  // Did THIS load start the run? A reload cannot have, and that is the only way
128
128
  // the page can tell "writing what you just briefed" from "picking up where
129
129
  // you left off" — the hook reports the run, not who asked for it.
130
130
  const [startedHere, setStartedHere] = useState(false);
131
- const { submit, run, pending, error, reset } = useWorkflowSubmit<typeof redline>(WORKFLOW, {
132
- // Neither half is useful alone: without the key there is nothing to find
133
- // the run by, and without `recover` the key is only ever written.
134
- key,
135
- recover: true,
136
- });
131
+ // The reload is covered by the hook's own key see the module doc for why
132
+ // this desk wants the tab-scoped one it mints rather than a key of its own.
133
+ const { submit, run, pending, error, reset } = useWorkflowSubmit<typeof redline>(WORKFLOW);
137
134
 
138
135
  return (
139
136
  <main className="mx-auto flex max-w-2xl flex-col gap-8 p-8">