@alexkroman1/aai-cli 8.2.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_dev-env.d.ts +15 -2
- package/dist/{_dev-server-Dt3TCjhj.mjs → _dev-server-BAR8Ya43.mjs} +182 -55
- package/dist/_dev-server.d.ts +5 -30
- package/dist/_dev-typecheck.d.ts +38 -0
- package/dist/_dev-vite-config.d.ts +74 -0
- package/dist/_e2e-test-utils.d.ts +33 -1
- package/dist/_init-BNotRgV1.mjs +163 -0
- package/dist/_resource-commands.d.ts +0 -1
- package/dist/{_slug-api-z-USxPtL.mjs → _slug-api--ZBmIGH-.mjs} +1 -1
- package/dist/{_studio-DR5yzZ1T.mjs → _studio-CKrsixd0.mjs} +2 -1
- package/dist/_studio.d.ts +3 -1
- package/dist/{build-BBxf8vom.mjs → build-DvDdSwAS.mjs} +1 -1
- package/dist/cli.mjs +26 -97
- package/dist/{delete-Dl_LSfvI.mjs → delete-8pjH8n8n.mjs} +3 -2
- package/dist/{deploy-DRC8jWr8.mjs → deploy-CjU3e-3P.mjs} +4 -4
- package/dist/{dev-QuuOOUtv.mjs → dev-dFzdvwa7.mjs} +3 -2
- package/dist/dev.d.ts +2 -0
- package/dist/{eval-DfQ8cm5p.mjs → eval-DwNwdqmS.mjs} +1 -1
- package/dist/{init-CUo87gcI.mjs → init-DK7GpIF9.mjs} +7 -6
- package/dist/{logs-CQ2T46dr.mjs → logs-D2Yhqe2w.mjs} +2 -2
- package/dist/scaffold/.env.example +7 -4
- package/dist/scaffold/CLAUDE.md +152 -97
- package/dist/scaffold/global.d.ts +12 -0
- package/dist/scaffold/package.json +4 -4
- package/dist/scaffold/server.mjs +49 -2
- package/dist/scaffold/tsconfig.json +1 -0
- package/dist/scaffold/vitest.config.ts +24 -0
- package/dist/{secret-Ct34kdgU.mjs → secret-CwgDLpSy.mjs} +2 -2
- package/dist/{studio-CzTzPqe1.mjs → studio-D3-HWIfk.mjs} +7 -5
- package/dist/templates/briefing-desk/agent.test.ts +3 -15
- package/dist/templates/briefing-desk/shared.ts +3 -3
- package/dist/templates/call-audit/agent.eval.test.ts +8 -1
- package/dist/templates/call-audit/agent.test.ts +0 -1
- package/dist/templates/call-audit/agent.ts +4 -3
- package/dist/templates/call-audit/client.tsx +4 -7
- package/dist/templates/call-audit/workflows/audit.ts +10 -1
- package/dist/templates/code-interpreter/agent.eval.test.ts +14 -43
- package/dist/templates/dispatch-center/agent.eval.test.ts +16 -26
- package/dist/templates/embedded-assets/agent.eval.test.ts +6 -16
- package/dist/templates/embedded-assets/agent.test.ts +3 -16
- package/dist/templates/health-assistant/agent.eval.test.ts +5 -15
- package/dist/templates/health-assistant/agent.test.ts +3 -14
- package/dist/templates/infocom-adventure/agent.eval.test.ts +6 -16
- package/dist/templates/infocom-adventure/agent.test.ts +3 -17
- package/dist/templates/link-digest/agent.test.ts +0 -14
- package/dist/templates/link-digest/agent.ts +9 -5
- package/dist/templates/link-digest/client.tsx +8 -6
- package/dist/templates/math-buddy/agent.eval.test.ts +11 -40
- package/dist/templates/night-owl/agent.eval.test.ts +15 -44
- package/dist/templates/night-owl/agent.test.ts +2 -15
- package/dist/templates/personal-finance/agent.eval.test.ts +10 -39
- package/dist/templates/pipeline-simple/agent.test.ts +73 -0
- package/dist/templates/pizza-ordering/agent.eval.test.ts +2 -17
- package/dist/templates/pizza-ordering/agent.test.ts +2 -17
- package/dist/templates/plan-and-execute/agent.eval.test.ts +2 -12
- package/dist/templates/plan-and-execute/agent.test.ts +3 -23
- package/dist/templates/plan-and-execute/shared.ts +1 -1
- package/dist/templates/podcast-digest/agent.test.ts +0 -1
- package/dist/templates/podcast-digest/client.tsx +3 -6
- package/dist/templates/podcast-digest/workflows/digest.ts +16 -8
- package/dist/templates/recap-workflow/agent.eval.test.ts +7 -14
- package/dist/templates/recap-workflow/agent.test.ts +3 -18
- package/dist/templates/recap-workflow/agent.ts +7 -5
- package/dist/templates/redline/agent.test.ts +0 -13
- package/dist/templates/redline/agent.ts +3 -2
- package/dist/templates/redline/client.tsx +9 -4
- package/dist/templates/research-workflow/agent.eval.test.ts +7 -14
- package/dist/templates/research-workflow/agent.test.ts +3 -15
- package/dist/templates/research-workflow/agent.ts +7 -5
- package/dist/templates/research-workflow/workflows/research.ts +8 -2
- package/dist/templates/retail/agent.eval.test.ts +10 -25
- package/dist/templates/retail/registry.test.ts +16 -13
- package/dist/templates/solo-rpg/agent.eval.test.ts +5 -15
- package/dist/templates/solo-rpg/agent.test.ts +17 -124
- package/dist/templates/solo-rpg/shared.ts +13 -61
- package/dist/templates/spoken-summary/agent.test.ts +0 -1
- package/dist/templates/spoken-summary/agent.ts +5 -4
- package/dist/templates/spoken-summary/client.tsx +4 -7
- package/dist/templates/support-line/agent.eval.test.ts +2 -12
- package/dist/templates/support-line/agent.test.ts +3 -22
- package/dist/templates/transcription-workflow/agent.ts +4 -4
- package/dist/templates/transcription-workflow/client.tsx +5 -5
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +10 -1
- package/dist/templates/travel-concierge/agent.eval.test.ts +5 -15
- package/dist/templates/travel-concierge/agent.test.ts +3 -23
- package/dist/templates/web-researcher/agent.eval.test.ts +2 -5
- package/dist/test-CvwgeVSQ.mjs +181 -0
- package/dist/test.d.ts +33 -2
- package/package.json +4 -4
- package/dist/_init-BhEfQ6Yi.mjs +0 -93
- package/dist/storage-i_79ZwAR.mjs +0 -66
- package/dist/storage.d.ts +0 -22
- package/dist/templates/solo-rpg/tools/load_game.ts +0 -37
- package/dist/templates/solo-rpg/tools/save_game.ts +0 -32
- package/dist/test-BvyT6sAc.mjs +0 -104
|
@@ -12,16 +12,11 @@
|
|
|
12
12
|
// different agent; and `run_code` refuses unless the EVAL supplies an executor,
|
|
13
13
|
// which this suite does — so the cases below assert the answer the code came
|
|
14
14
|
// back with as well as the code the tutor wrote.
|
|
15
|
-
|
|
16
|
-
import
|
|
17
|
-
import {
|
|
18
|
-
import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
|
|
15
|
+
|
|
16
|
+
import agentDef from "virtual:aai/agent";
|
|
17
|
+
import { createVmRunCode, toolResultsIn } from "@alexkroman1/aai-runtime/eval";
|
|
19
18
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
20
19
|
import { expect } from "vitest";
|
|
21
|
-
import authored from "./agent.ts";
|
|
22
|
-
import systemPrompt from "./system-prompt.md?raw";
|
|
23
|
-
|
|
24
|
-
const agentDef = withSystemPrompt(authored, systemPrompt);
|
|
25
20
|
|
|
26
21
|
/** The code every `run_code` call in this turn carried, joined. */
|
|
27
22
|
const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
|
|
@@ -35,37 +30,13 @@ const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string,
|
|
|
35
30
|
*
|
|
36
31
|
* The builtin refuses without one — the Modal container is the security
|
|
37
32
|
* boundary, and off-platform there is none — so a case could assert the CALL and
|
|
38
|
-
* the code it carried, and never what the code came back with.
|
|
39
|
-
* context with a capturing `console.log
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* A template eval imports from `@alexkroman1/aai-runtime/eval` and
|
|
45
|
-
* `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
|
|
46
|
-
* node-reaching module graph into this project's TypeScript program, which is
|
|
47
|
-
* three errors in runtime files no eval ever calls. `RunCodeExecutor` is
|
|
48
|
-
* re-exported from `/eval` for exactly that reason.
|
|
33
|
+
* the code it carried, and never what the code came back with.
|
|
34
|
+
* `createVmRunCode()` is a `node:vm` context with a capturing `console.log`,
|
|
35
|
+
* which is enough here: what runs is arithmetic, not a program. It is NOT a
|
|
36
|
+
* sandbox and does not pretend to be one; a deployed agent still gets the
|
|
37
|
+
* refusal.
|
|
49
38
|
*/
|
|
50
|
-
const runCode
|
|
51
|
-
const lines: string[] = [];
|
|
52
|
-
const log = (...args: unknown[]): void => {
|
|
53
|
-
lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
|
|
54
|
-
};
|
|
55
|
-
try {
|
|
56
|
-
runInNewContext(code, { console: { log } }, { timeout: 1000 });
|
|
57
|
-
} catch (err) {
|
|
58
|
-
return { error: errorMessage(err) };
|
|
59
|
-
}
|
|
60
|
-
return lines.join("\n");
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
/** What every `run_code` call in this turn PRINTED, joined. */
|
|
64
|
-
const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
|
|
65
|
-
turn.toolCalls
|
|
66
|
-
.filter((c) => c.name === "run_code")
|
|
67
|
-
.map((c) => c.result ?? "")
|
|
68
|
-
.join("\n");
|
|
39
|
+
const runCode = createVmRunCode();
|
|
69
40
|
|
|
70
41
|
describeEval(
|
|
71
42
|
agentDef,
|
|
@@ -84,7 +55,7 @@ describeEval(
|
|
|
84
55
|
expect(code).toMatch(/1\.60|1\.61|0\.621|8\.04/);
|
|
85
56
|
// And the factor was applied rather than merely mentioned: five miles is
|
|
86
57
|
// 8.0467 km, so whatever rounding the tutor chose the answer starts 8.0.
|
|
87
|
-
const output =
|
|
58
|
+
const output = toolResultsIn(turn.toolCalls, "run_code").join("\n");
|
|
88
59
|
expect(output, `run_code printed: ${output}`).toMatch(/8\.0/);
|
|
89
60
|
},
|
|
90
61
|
{ live: true },
|
|
@@ -108,7 +79,7 @@ describeEval(
|
|
|
108
79
|
// face of a twenty-sided die. `Math.random` in the code says the tutor
|
|
109
80
|
// asked for a roll; this says it GOT one — a `run_code` that refused
|
|
110
81
|
// prints a sentence with no dice in it at all.
|
|
111
|
-
const output =
|
|
82
|
+
const output = toolResultsIn(turn.toolCalls, "run_code").join("\n");
|
|
112
83
|
const rolled = [...output.matchAll(/\d+/g)].map((m) => Number(m[0]));
|
|
113
84
|
expect(rolled.length, `run_code printed: ${output}`).toBeGreaterThanOrEqual(3);
|
|
114
85
|
for (const face of rolled) {
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
/// <reference types="vite/client" />
|
|
2
|
-
|
|
3
1
|
// An EVAL: does the companion actually pick from its own shelf?
|
|
4
2
|
//
|
|
5
3
|
// `agent.test.ts` calls `recommend` directly, which settles what the tool does
|
|
@@ -12,24 +10,6 @@
|
|
|
12
10
|
// SCRIPTED model (its `stubReply`): the real session, the real slot, the real
|
|
13
11
|
// tool, a fake reply. That proves the wiring and nothing about the choice.
|
|
14
12
|
|
|
15
|
-
import { runInNewContext } from "node:vm";
|
|
16
|
-
import { withSystemPrompt } from "@alexkroman1/aai/manifest";
|
|
17
|
-
import type { SessionEvent } from "@alexkroman1/aai/protocol";
|
|
18
|
-
import { withDiscoveredTools } from "@alexkroman1/aai/testing";
|
|
19
|
-
import { errorMessage } from "@alexkroman1/aai/utils";
|
|
20
|
-
import {
|
|
21
|
-
customEventsIn,
|
|
22
|
-
lastStateIn,
|
|
23
|
-
type RunCodeExecutor,
|
|
24
|
-
toolResultIn,
|
|
25
|
-
} from "@alexkroman1/aai-runtime/eval";
|
|
26
|
-
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
27
|
-
import { expect } from "vitest";
|
|
28
|
-
import { z } from "zod";
|
|
29
|
-
import authoredAgent from "./agent.ts";
|
|
30
|
-
import { CATEGORIES, MOODS } from "./shared.ts";
|
|
31
|
-
import systemPrompt from "./system-prompt.md?raw";
|
|
32
|
-
|
|
33
13
|
/**
|
|
34
14
|
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
35
15
|
*
|
|
@@ -44,10 +24,18 @@ import systemPrompt from "./system-prompt.md?raw";
|
|
|
44
24
|
* different agent than the one that deploys, and every tool-choice claim below
|
|
45
25
|
* then passes or fails for the wrong reason.
|
|
46
26
|
*/
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
27
|
+
import agentDef from "virtual:aai/agent";
|
|
28
|
+
import type { SessionEvent } from "@alexkroman1/aai/protocol";
|
|
29
|
+
import {
|
|
30
|
+
createVmRunCode,
|
|
31
|
+
customEventsIn,
|
|
32
|
+
lastStateIn,
|
|
33
|
+
toolResultIn,
|
|
34
|
+
} from "@alexkroman1/aai-runtime/eval";
|
|
35
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
36
|
+
import { expect } from "vitest";
|
|
37
|
+
import { z } from "zod";
|
|
38
|
+
import { CATEGORIES, MOODS } from "./shared.ts";
|
|
51
39
|
|
|
52
40
|
/**
|
|
53
41
|
* One `recommend` answer, and the whole projection, as the wire carries them.
|
|
@@ -84,27 +72,10 @@ const nudges = (events: readonly SessionEvent[]) => customEventsIn(events, "wind
|
|
|
84
72
|
* boundary, and off-platform there is none — which left this template's
|
|
85
73
|
* headline feature assertable as a CALL and never as a number. A `node:vm`
|
|
86
74
|
* context with a capturing `console.log` is what a developer would reach for on
|
|
87
|
-
* their own machine, and
|
|
88
|
-
* model wrote, not a program.
|
|
89
|
-
*
|
|
90
|
-
* A template eval imports from `@alexkroman1/aai-runtime/eval` and
|
|
91
|
-
* `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
|
|
92
|
-
* node-reaching module graph into this project's TypeScript program, which is
|
|
93
|
-
* three errors in runtime files no eval ever calls. `RunCodeExecutor` is
|
|
94
|
-
* re-exported from `/eval` for exactly that reason.
|
|
75
|
+
* their own machine, and `createVmRunCode()` is exactly that: the code under
|
|
76
|
+
* test is arithmetic the model wrote, not a program.
|
|
95
77
|
*/
|
|
96
|
-
const runCode
|
|
97
|
-
const lines: string[] = [];
|
|
98
|
-
const log = (...args: unknown[]): void => {
|
|
99
|
-
lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
|
|
100
|
-
};
|
|
101
|
-
try {
|
|
102
|
-
runInNewContext(code, { console: { log } }, { timeout: 1000 });
|
|
103
|
-
} catch (err) {
|
|
104
|
-
return { error: errorMessage(err) };
|
|
105
|
-
}
|
|
106
|
-
return lines.join("\n");
|
|
107
|
-
};
|
|
78
|
+
const runCode = createVmRunCode();
|
|
108
79
|
|
|
109
80
|
/** Two-digit, for the clock arithmetic below. */
|
|
110
81
|
const pad = (n: number): string => String(n).padStart(2, "0");
|
|
@@ -1,28 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/** The def a DEPLOYED agent runs: authored, plus what `tools/` declares. */
|
|
2
|
+
import agentDef from "virtual:aai/agent";
|
|
3
3
|
import { toAgentConfig } from "@alexkroman1/aai/manifest";
|
|
4
4
|
import {
|
|
5
5
|
createToolContext,
|
|
6
6
|
parseToolInput,
|
|
7
7
|
toolInputIssues,
|
|
8
8
|
toolRunner,
|
|
9
|
-
withDiscoveredTools,
|
|
10
9
|
} from "@alexkroman1/aai/testing";
|
|
11
10
|
import { describe, expect, test } from "vitest";
|
|
12
|
-
import authoredAgent from "./agent.ts";
|
|
13
11
|
import { CATEGORIES, MOODS, nightProjection, nightSlot } from "./shared.ts";
|
|
14
12
|
|
|
15
|
-
/**
|
|
16
|
-
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
17
|
-
*
|
|
18
|
-
* The glob is written HERE rather than reached for from a shared helper because
|
|
19
|
-
* this file SHIPS — a scaffolded project has no repo helper to import.
|
|
20
|
-
*/
|
|
21
|
-
const agentDef = withDiscoveredTools(
|
|
22
|
-
authoredAgent,
|
|
23
|
-
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
24
|
-
);
|
|
25
|
-
|
|
26
13
|
/**
|
|
27
14
|
* `runTool` takes the context in the ARGUMENTS' place when a tool needs none,
|
|
28
15
|
* so `toolRunner`'s second parameter takes either — which is why it is one
|
|
@@ -14,16 +14,11 @@
|
|
|
14
14
|
// the answer as well as the code that was submitted. `fetch_json` is
|
|
15
15
|
// unaffected — it makes a real request, so the currency case really does reach a
|
|
16
16
|
// live rates API.
|
|
17
|
-
|
|
18
|
-
import
|
|
19
|
-
import {
|
|
20
|
-
import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
|
|
17
|
+
|
|
18
|
+
import agentDef from "virtual:aai/agent";
|
|
19
|
+
import { createVmRunCode, toolResultsIn } from "@alexkroman1/aai-runtime/eval";
|
|
21
20
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
22
21
|
import { expect } from "vitest";
|
|
23
|
-
import authored from "./agent.ts";
|
|
24
|
-
import systemPrompt from "./system-prompt.md?raw";
|
|
25
|
-
|
|
26
|
-
const agentDef = withSystemPrompt(authored, systemPrompt);
|
|
27
22
|
|
|
28
23
|
type Turn = { toolCalls: readonly { name: string; args: Record<string, unknown> }[] };
|
|
29
24
|
|
|
@@ -43,37 +38,13 @@ const fetchedUrls = (turn: Turn) =>
|
|
|
43
38
|
*
|
|
44
39
|
* The builtin refuses without one — the Modal container is the security
|
|
45
40
|
* boundary, and off-platform there is none — so a case could assert the CALL and
|
|
46
|
-
* the code it carried, and never what the code came back with.
|
|
47
|
-
* context with a capturing `console.log
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* A template eval imports from `@alexkroman1/aai-runtime/eval` and
|
|
53
|
-
* `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
|
|
54
|
-
* node-reaching module graph into this project's TypeScript program, which is
|
|
55
|
-
* three errors in runtime files no eval ever calls. `RunCodeExecutor` is
|
|
56
|
-
* re-exported from `/eval` for exactly that reason.
|
|
41
|
+
* the code it carried, and never what the code came back with.
|
|
42
|
+
* `createVmRunCode()` is a `node:vm` context with a capturing `console.log`,
|
|
43
|
+
* which is enough here: what runs is arithmetic, not a program. It is NOT a
|
|
44
|
+
* sandbox and does not pretend to be one; a deployed agent still gets the
|
|
45
|
+
* refusal.
|
|
57
46
|
*/
|
|
58
|
-
const runCode
|
|
59
|
-
const lines: string[] = [];
|
|
60
|
-
const log = (...args: unknown[]): void => {
|
|
61
|
-
lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
|
|
62
|
-
};
|
|
63
|
-
try {
|
|
64
|
-
runInNewContext(code, { console: { log } }, { timeout: 1000 });
|
|
65
|
-
} catch (err) {
|
|
66
|
-
return { error: errorMessage(err) };
|
|
67
|
-
}
|
|
68
|
-
return lines.join("\n");
|
|
69
|
-
};
|
|
70
|
-
|
|
71
|
-
/** What every `run_code` call in this turn PRINTED, joined. */
|
|
72
|
-
const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
|
|
73
|
-
turn.toolCalls
|
|
74
|
-
.filter((c) => c.name === "run_code")
|
|
75
|
-
.map((c) => c.result ?? "")
|
|
76
|
-
.join("\n");
|
|
47
|
+
const runCode = createVmRunCode();
|
|
77
48
|
|
|
78
49
|
describeEval(
|
|
79
50
|
agentDef,
|
|
@@ -98,7 +69,7 @@ describeEval(
|
|
|
98
69
|
// Without an executor `run_code` answers with a refusal, so every claim
|
|
99
70
|
// above is satisfied by an agent that then divides in its head — which is
|
|
100
71
|
// the failure this template's whole run_code rule exists to prevent.
|
|
101
|
-
const output =
|
|
72
|
+
const output = toolResultsIn(turn.toolCalls, "run_code").join("\n");
|
|
102
73
|
expect(output, `run_code printed: ${output}`).toMatch(/\b36(\.0+)?\b/);
|
|
103
74
|
},
|
|
104
75
|
{ live: true },
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
+
import { agent } from "@alexkroman1/aai";
|
|
1
2
|
import { toAgentConfig } from "@alexkroman1/aai/manifest";
|
|
3
|
+
import { assemblyAIS2s, openaiS2s } from "@alexkroman1/aai/s2s";
|
|
4
|
+
import { assemblyAIStt, deepgramStt, elevenLabsStt, sonioxStt } from "@alexkroman1/aai/stt";
|
|
5
|
+
import { ASSEMBLYAI_TTS_VOICES, assemblyAITts, cartesiaTts, rimeTts } from "@alexkroman1/aai/tts";
|
|
2
6
|
import { describe, expect, test } from "vitest";
|
|
3
7
|
import agentDef from "./agent.ts";
|
|
4
8
|
|
|
@@ -30,3 +34,72 @@ describe("pipeline-simple template", () => {
|
|
|
30
34
|
expect(config.llm?.kind).toBe("anthropic");
|
|
31
35
|
});
|
|
32
36
|
});
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The other stage swaps, as worked examples.
|
|
40
|
+
*
|
|
41
|
+
* `agent.ts` above declares one — the LLM — because an agent should declare one
|
|
42
|
+
* thing. But the rest of the provider surface had NO example anywhere: zero of
|
|
43
|
+
* the 26 templates set `stt:`, `tts:` or `s2s:`, so `@alexkroman1/aai/stt` and
|
|
44
|
+
* `/s2s` were 100% unexercised while carrying a semver promise. That is either
|
|
45
|
+
* an API nobody needs or an API nobody has run, and neither is a good place to
|
|
46
|
+
* leave a published surface.
|
|
47
|
+
*
|
|
48
|
+
* They live in the spec rather than in `agent.ts` for a reason worth copying:
|
|
49
|
+
* a deployed agent needs a key per vendor it names, and the all-AssemblyAI
|
|
50
|
+
* default is what makes the starter run the moment it is deployed. Declaring a
|
|
51
|
+
* stage here shows the shape and costs the template nothing.
|
|
52
|
+
*/
|
|
53
|
+
describe("swapping any other stage", () => {
|
|
54
|
+
test("STT: four providers, each one field on `agent()`", () => {
|
|
55
|
+
for (const [stt, kind] of [
|
|
56
|
+
[assemblyAIStt({ region: "eu" }), "assemblyai"],
|
|
57
|
+
[deepgramStt({ language: "en" }), "deepgram"],
|
|
58
|
+
[elevenLabsStt(), "elevenlabs"],
|
|
59
|
+
[sonioxStt({ languages: ["en", "es"] }), "soniox"],
|
|
60
|
+
] as const) {
|
|
61
|
+
const config = toAgentConfig(agent({ name: "Line", stt }));
|
|
62
|
+
expect(config.stt?.kind).toBe(kind);
|
|
63
|
+
// Whatever the STT stage is, the two it does not touch still default.
|
|
64
|
+
expect(config.llm?.kind).toBe("assemblyai");
|
|
65
|
+
expect(config.tts?.kind).toBe("assemblyai");
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("TTS: three providers, and the AssemblyAI voice catalog is typed", () => {
|
|
70
|
+
for (const [tts, kind] of [
|
|
71
|
+
[assemblyAITts({ voice: "jane" }), "assemblyai"],
|
|
72
|
+
[cartesiaTts(), "cartesia"],
|
|
73
|
+
[rimeTts(), "rime"],
|
|
74
|
+
] as const) {
|
|
75
|
+
expect(toAgentConfig(agent({ name: "Line", tts })).tts?.kind).toBe(kind);
|
|
76
|
+
}
|
|
77
|
+
// A voice outside the catalog still compiles — it may be one shipped after
|
|
78
|
+
// this release — and `agentConfigWarnings` is what says so. See its doc.
|
|
79
|
+
expect(Object.keys(ASSEMBLYAI_TTS_VOICES).length).toBeGreaterThan(0);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("S2S: an explicit opt-in, and it REPLACES the pipeline rather than joining it", () => {
|
|
83
|
+
for (const [s2s, kind] of [
|
|
84
|
+
[assemblyAIS2s(), "assemblyai"],
|
|
85
|
+
[openaiS2s({ voice: "alloy" }), "openai-realtime"],
|
|
86
|
+
] as const) {
|
|
87
|
+
const config = toAgentConfig(agent({ name: "Line", s2s }));
|
|
88
|
+
expect(config.s2s?.kind).toBe(kind);
|
|
89
|
+
expect(config.mode).toBe("s2s");
|
|
90
|
+
// No cascade is filled in: speech-to-speech has no separate STT or TTS,
|
|
91
|
+
// which is the whole difference and the reason `s2s` is never a default.
|
|
92
|
+
expect(config.stt).toBeUndefined();
|
|
93
|
+
expect(config.tts).toBeUndefined();
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("mixing S2S with a pipeline stage is refused, by the type AND by the config", () => {
|
|
98
|
+
// The type refuses it outright: `agent({ s2s, tts })` does not compile, which
|
|
99
|
+
// is why this reaches the runtime rule by spreading instead. Both halves
|
|
100
|
+
// matter — the second is what catches a raw `export default {...}` that
|
|
101
|
+
// never went through `agent()`.
|
|
102
|
+
const s2sAgent = agent({ name: "Line", s2s: assemblyAIS2s() });
|
|
103
|
+
expect(() => toAgentConfig({ ...s2sAgent, tts: cartesiaTts() })).toThrow();
|
|
104
|
+
});
|
|
105
|
+
});
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/** The def a DEPLOYED agent runs: authored, plus what `tools/` declares. */
|
|
2
|
+
import agentDef from "virtual:aai/agent";
|
|
3
3
|
import type { SessionEvent } from "@alexkroman1/aai/protocol";
|
|
4
|
-
import { withDiscoveredTools } from "@alexkroman1/aai/testing";
|
|
5
4
|
// An EVAL: does the order-taker actually take the order?
|
|
6
5
|
//
|
|
7
6
|
// `agent.test.ts` drives the six tools directly, which settles what each one
|
|
@@ -18,22 +17,8 @@ import { lastStateIn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
|
|
|
18
17
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
19
18
|
import { expect } from "vitest";
|
|
20
19
|
import { z } from "zod";
|
|
21
|
-
import authoredAgent from "./agent.ts";
|
|
22
20
|
import { MENU } from "./shared.ts";
|
|
23
21
|
|
|
24
|
-
/**
|
|
25
|
-
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
26
|
-
*
|
|
27
|
-
* The glob is written HERE rather than reached for from a shared helper because
|
|
28
|
-
* this file SHIPS — a scaffolded project has no repo helper to import. Without
|
|
29
|
-
* it the eval would drive an agent with no tools at all, which is the one
|
|
30
|
-
* failure mode a green eval must not be able to hide.
|
|
31
|
-
*/
|
|
32
|
-
const agentDef = withDiscoveredTools(
|
|
33
|
-
authoredAgent,
|
|
34
|
-
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
35
|
-
);
|
|
36
|
-
|
|
37
22
|
/**
|
|
38
23
|
* What the BROWSER is sent, as this eval reads it.
|
|
39
24
|
*
|
|
@@ -1,28 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/** The def a DEPLOYED agent runs: authored, plus what `tools/` declares. */
|
|
2
|
+
import agentDef from "virtual:aai/agent";
|
|
3
3
|
import type { ToolContext } from "@alexkroman1/aai";
|
|
4
4
|
import {
|
|
5
5
|
createToolContext,
|
|
6
6
|
parseToolInput,
|
|
7
7
|
toolInputIssues,
|
|
8
8
|
toolRunner,
|
|
9
|
-
withDiscoveredTools,
|
|
10
9
|
} from "@alexkroman1/aai/testing";
|
|
11
10
|
import { describe, expect, test } from "vitest";
|
|
12
|
-
import authoredAgent from "./agent.ts";
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
16
|
-
*
|
|
17
|
-
* The glob is written HERE rather than reached for from a shared helper because
|
|
18
|
-
* this file SHIPS: it is what a scaffolded project runs, so it may not import
|
|
19
|
-
* anything outside its own template, and `import.meta.glob` is expanded against
|
|
20
|
-
* the file containing it either way. This is the pattern a user writes.
|
|
21
|
-
*/
|
|
22
|
-
const agentDef = withDiscoveredTools(
|
|
23
|
-
authoredAgent,
|
|
24
|
-
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
25
|
-
);
|
|
26
11
|
|
|
27
12
|
import {
|
|
28
13
|
calculateTotal,
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
/// <reference types="vite/client" />
|
|
2
|
-
|
|
3
1
|
// An EVAL: does the desk really plan, work one step, and replan? Run it with
|
|
4
2
|
// `aai eval`.
|
|
5
3
|
//
|
|
@@ -24,20 +22,12 @@
|
|
|
24
22
|
// product questions from its own knowledge and skips the tools the prompt exists
|
|
25
23
|
// to route it through, so a case run that way measures nothing it claims to.
|
|
26
24
|
|
|
27
|
-
|
|
28
|
-
import
|
|
25
|
+
/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
|
|
26
|
+
import agentDef from "virtual:aai/agent";
|
|
29
27
|
import { type EvalSession, lastStateIn } from "@alexkroman1/aai-runtime/eval";
|
|
30
28
|
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
31
29
|
import { expect } from "vitest";
|
|
32
30
|
import { z } from "zod";
|
|
33
|
-
import authoredAgent from "./agent.ts";
|
|
34
|
-
import systemPrompt from "./system-prompt.md?raw";
|
|
35
|
-
|
|
36
|
-
/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
|
|
37
|
-
const agentDef = withSystemPrompt(
|
|
38
|
-
withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
|
|
39
|
-
systemPrompt,
|
|
40
|
-
);
|
|
41
31
|
|
|
42
32
|
/**
|
|
43
33
|
* What the BROWSER is sent, as this eval reads it.
|
|
@@ -1,28 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/** The def a DEPLOYED agent runs: authored, plus what `tools/` declares. */
|
|
2
|
+
import agentDef from "virtual:aai/agent";
|
|
3
3
|
import type { ToolContext } from "@alexkroman1/aai";
|
|
4
|
-
import {
|
|
5
|
-
createToolContext,
|
|
6
|
-
okPosition,
|
|
7
|
-
stubGenerate,
|
|
8
|
-
toolRunner,
|
|
9
|
-
withDiscoveredTools,
|
|
10
|
-
} from "@alexkroman1/aai/testing";
|
|
4
|
+
import { createToolContext, okPosition, stubGenerate, toolRunner } from "@alexkroman1/aai/testing";
|
|
11
5
|
import { describe, expect, test } from "vitest";
|
|
12
|
-
import authoredAgent from "./agent.ts";
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
16
|
-
*
|
|
17
|
-
* The glob is written HERE rather than reached for from a shared helper because
|
|
18
|
-
* this file SHIPS: it is what a scaffolded project runs, so it may not import
|
|
19
|
-
* anything outside its own template, and `import.meta.glob` is expanded against
|
|
20
|
-
* the file containing it either way. This is the pattern a user writes.
|
|
21
|
-
*/
|
|
22
|
-
const agentDef = withDiscoveredTools(
|
|
23
|
-
authoredAgent,
|
|
24
|
-
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
25
|
-
);
|
|
26
6
|
|
|
27
7
|
import { executeStep, MAX_STEP_SEARCHES, normalizeAct, planNode } from "./procedure.ts";
|
|
28
8
|
import { EXECUTOR_SYSTEM, PLANNER_SYSTEM, REPLANNER_SYSTEM, REVISE_SYSTEM } from "./prompts.ts";
|
|
@@ -175,7 +175,7 @@ export const SEARCH_RESULTS = 4;
|
|
|
175
175
|
export const liveSearch: SearchFn = async (query) => {
|
|
176
176
|
const results = await webSearch<{ results?: { title?: string; url?: string }[] }>({
|
|
177
177
|
query,
|
|
178
|
-
|
|
178
|
+
maxResults: SEARCH_RESULTS,
|
|
179
179
|
});
|
|
180
180
|
// A REFUSED search is not an empty web, and `webSearch` answers with
|
|
181
181
|
// `{ error }` rather than throwing — so an unnarrowed `?? []` below would tell
|
|
@@ -92,7 +92,6 @@ const slackInput = (slackWebhookUrl: string) => ({
|
|
|
92
92
|
describe("the declaration", () => {
|
|
93
93
|
test("is a workflow app with one workflow and a static page", () => {
|
|
94
94
|
expect(agentDef.name).toBe("Podcast Digest");
|
|
95
|
-
expect(agentDef.page).toBe("static");
|
|
96
95
|
expect(Object.keys(agentDef.workflows ?? {})).toEqual(["dailyDigest"]);
|
|
97
96
|
expect(agentDef.workflows?.dailyDigest).toBe(dailyDigest);
|
|
98
97
|
});
|
|
@@ -59,12 +59,8 @@ import "@alexkroman1/aai-ui/styles.css";
|
|
|
59
59
|
// ERASED at build time, so naming the agent's own type costs the browser bundle
|
|
60
60
|
// nothing — and it is what stops this file restating a shape `workflows/
|
|
61
61
|
// digest.ts` already declares.
|
|
62
|
-
import type { WorkflowOutputOf } from "@alexkroman1/aai/workflow-api";
|
|
63
62
|
import type { dailyDigest } from "./agent.ts";
|
|
64
63
|
|
|
65
|
-
/** What a completed run reports, derived from the workflow rather than restated. */
|
|
66
|
-
type Digest = WorkflowOutputOf<typeof dailyDigest>;
|
|
67
|
-
|
|
68
64
|
/** The workflow this page drives. Matches the key in `workflowApp({ workflows })`. */
|
|
69
65
|
const WORKFLOW = "dailyDigest";
|
|
70
66
|
|
|
@@ -73,7 +69,8 @@ export function App() {
|
|
|
73
69
|
// `run.output` instead of `unknown`. `error` is the agent's own sentence for a
|
|
74
70
|
// rejected input — better copy than anything this page could write, and the
|
|
75
71
|
// reason there is no `try`/`catch` here.
|
|
76
|
-
const {
|
|
72
|
+
const { submitForm, run, pending, error, wake, cancel } =
|
|
73
|
+
useWorkflowSubmit<typeof dailyDigest>(WORKFLOW);
|
|
77
74
|
|
|
78
75
|
return (
|
|
79
76
|
<main className="mx-auto flex max-w-2xl flex-col gap-6 p-8">
|
|
@@ -81,7 +78,7 @@ export function App() {
|
|
|
81
78
|
|
|
82
79
|
{/* `submit()` resolves as soon as the run EXISTS — deliberately not when it
|
|
83
80
|
finishes, which here could be a month away. */}
|
|
84
|
-
<Form onSubmit={
|
|
81
|
+
<Form onSubmit={submitForm} error={error}>
|
|
85
82
|
<WorkflowFields workflow={WORKFLOW} />
|
|
86
83
|
<SubmitButton pending={pending}>
|
|
87
84
|
{pending ? "Digest scheduled" : "Start digest"}
|
|
@@ -29,12 +29,12 @@
|
|
|
29
29
|
* ## Storage is not optional here, unlike everywhere else
|
|
30
30
|
*
|
|
31
31
|
* Every other template says "enable the database for durability, but it is fine
|
|
32
|
-
* to build without it".
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* work either way; leave it on `days`
|
|
37
|
-
*
|
|
32
|
+
* to build without it". Deployed, that is settled for you: the platform keeps
|
|
33
|
+
* runs on its own database, so a multi-day sleep survives. Under `aai dev`
|
|
34
|
+
* without a `DATABASE_URL` the run lives in the process, and a process does not
|
|
35
|
+
* survive a multi-day sleep — build with `intervalUnit: "minutes"` and you will
|
|
36
|
+
* see it work either way; leave it on `days` locally and the second digest never
|
|
37
|
+
* arrives.
|
|
38
38
|
*
|
|
39
39
|
* ## Batch polling, which is this file's one genuinely new mechanism
|
|
40
40
|
*
|
|
@@ -90,10 +90,18 @@ export const MAX_POLL_ATTEMPTS = 180;
|
|
|
90
90
|
*/
|
|
91
91
|
const MAX_TRANSCRIPT_CHARS = 18_000;
|
|
92
92
|
|
|
93
|
-
/**
|
|
93
|
+
/**
|
|
94
|
+
* Submissions in flight at once — polite to the provider, still parallel.
|
|
95
|
+
*
|
|
96
|
+
* Inside the workflow world's default worker concurrency (three on the
|
|
97
|
+
* `DATABASE_URL` path), so this width is what really runs — see "The WINDOW is
|
|
98
|
+
* not the concurrency" in `mapConcurrent`.
|
|
99
|
+
*/
|
|
94
100
|
const SUBMIT_CONCURRENCY = 2;
|
|
95
101
|
|
|
96
|
-
/** Status checks in flight at once. Cheaper calls, so a wider gate
|
|
102
|
+
/** Status checks in flight at once. Cheaper calls, so a wider gate — and three is
|
|
103
|
+
* also where the world's default worker concurrency caps it, so this is the
|
|
104
|
+
* widest a stock deployment executes. */
|
|
97
105
|
const POLL_CONCURRENCY = 3;
|
|
98
106
|
|
|
99
107
|
/**
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
/// <reference types="vite/client" />
|
|
2
|
-
|
|
3
1
|
// An EVAL: does this desk actually behave? Run it with `aai eval`.
|
|
4
2
|
//
|
|
5
3
|
// `agent.test.ts` drives the five tools against a STUBBED `ctx.workflows`, the
|
|
@@ -30,14 +28,6 @@
|
|
|
30
28
|
// model stays live and is what a live run measures.
|
|
31
29
|
//
|
|
32
30
|
// And what no eval here can see: anything below the audio boundary.
|
|
33
|
-
import { withDiscoveredTools } from "@alexkroman1/aai/testing";
|
|
34
|
-
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
35
|
-
import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
|
|
36
|
-
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
37
|
-
import { expect } from "vitest";
|
|
38
|
-
import { z } from "zod";
|
|
39
|
-
import authoredAgent from "./agent.ts";
|
|
40
|
-
import { recap, SAMPLE_RECORDING } from "./shared.ts";
|
|
41
31
|
|
|
42
32
|
/**
|
|
43
33
|
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
|
|
@@ -50,10 +40,13 @@ import { recap, SAMPLE_RECORDING } from "./shared.ts";
|
|
|
50
40
|
* The glob is written here rather than reached for from a shared helper because
|
|
51
41
|
* this file SHIPS — see `agent.test.ts`.
|
|
52
42
|
*/
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
43
|
+
import agentDef from "virtual:aai/agent";
|
|
44
|
+
import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
|
|
45
|
+
import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
|
|
46
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
47
|
+
import { expect } from "vitest";
|
|
48
|
+
import { z } from "zod";
|
|
49
|
+
import { recap, SAMPLE_RECORDING } from "./shared.ts";
|
|
57
50
|
|
|
58
51
|
/**
|
|
59
52
|
* The key every step reads with `requireStepEnv` — the one name `agent.ts`
|