@alexkroman1/aai-cli 14.0.0 → 15.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_artifacts-BJOYGQPp.mjs +21 -0
- package/dist/_artifacts.d.ts +16 -0
- package/dist/_build-target.d.ts +172 -0
- package/dist/{_bundler-DolUCMxu.mjs → _bundler-DM0d0M7m.mjs} +1 -1
- package/dist/{_dev-server-CSMqF8PN.mjs → _dev-server-BzWB6-4y.mjs} +9 -6
- package/dist/_e2e-test-utils.d.ts +1 -1
- package/dist/{_init-CQ8idAwo.mjs → _init-Bsi3DZNJ.mjs} +1 -1
- package/dist/_server-common-De0haHr9.mjs +70 -0
- package/dist/_server-common.d.ts +20 -1
- package/dist/{_templates-CK4oKoeX.mjs → _templates-CIlJ3Vay.mjs} +1 -1
- package/dist/_templates.d.ts +1 -1
- package/dist/_vercel-output.d.ts +63 -0
- package/dist/build-BhEaxBPu.mjs +481 -0
- package/dist/build.d.ts +18 -10
- package/dist/cli.mjs +51 -23
- package/dist/{client-bundler-BJgREAh6.mjs → client-bundler-6mTLs6ny.mjs} +4 -4
- package/dist/client-bundler.d.ts +1 -1
- package/dist/client-bundler.mjs +1 -1
- package/dist/{deploy-uAJ4NukN.mjs → deploy-CGqPU5U-.mjs} +2 -2
- package/dist/{dev-DApPSaE_.mjs → dev-Bx9gYBHM.mjs} +1 -1
- package/dist/{eval-BK47A_K5.mjs → eval-B3I7FqN9.mjs} +1 -1
- package/dist/{init-DukDxECd.mjs → init-CQbj3ycf.mjs} +49 -39
- package/dist/init.d.ts +19 -10
- package/dist/scaffold/CLAUDE.md +73 -22
- package/dist/scaffold/package.json +6 -6
- package/dist/start.d.ts +112 -0
- package/dist/start.mjs +156 -0
- package/dist/{studio-CpHlNHUZ.mjs → studio-C_zuRC_z.mjs} +2 -2
- package/dist/templates/briefing-desk/agent.eval.test.ts +156 -0
- package/dist/templates/code-interpreter/agent.test.ts +103 -0
- package/dist/templates/link-digest/client.tsx +55 -3
- package/dist/templates/math-buddy/agent.test.ts +126 -0
- package/dist/templates/personal-finance/agent.test.ts +127 -0
- package/dist/templates/support-line/agent.ts +8 -0
- package/dist/templates/travel-concierge/routing.ts +64 -55
- package/dist/templates/travel-concierge/tools/cancel_action.ts +3 -1
- package/dist/templates/travel-concierge/tools/complete_or_escalate.ts +3 -1
- package/dist/templates/travel-concierge/tools/confirm_action.ts +3 -1
- package/dist/templates/web-researcher/agent.test.ts +130 -0
- package/dist/worker-bundler.d.ts +1 -1
- package/dist/worker-bundler.mjs +7 -7
- package/package.json +9 -4
- package/dist/_server-common-vILJp3it.mjs +0 -43
- package/dist/build-Mxk8gWvX.mjs +0 -108
- package/dist/scaffold/server.mjs +0 -308
package/dist/start.d.ts
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `aai start` — serve a BUILT agent from a plain Node process.
|
|
3
|
+
*
|
|
4
|
+
* The deployment counterpart of `aai dev`: no file watching, no Vite, no
|
|
5
|
+
* typecheck, no source evaluation. It loads the artifact `aai build` left at
|
|
6
|
+
* {@link WORKER_ARTIFACT_REL} — the same one `aai publish` uploads and the
|
|
7
|
+
* managed platform runs — so a self-hosted agent and a deployed one cannot
|
|
8
|
+
* behave differently.
|
|
9
|
+
*
|
|
10
|
+
* ## Why this is a command rather than a file in every project
|
|
11
|
+
*
|
|
12
|
+
* It used to be `scaffold/server.mjs`, ~300 lines of boot that `aai init`
|
|
13
|
+
* copied into every scaffolded project: worker load, env resolution, schema
|
|
14
|
+
* DDL, client-directory probing, error classification, listen, signal
|
|
15
|
+
* handlers. Shipping that as source made each of those a fact a USER's
|
|
16
|
+
* repository asserted, so improving any of them reached only projects
|
|
17
|
+
* scaffolded afterwards, and an existing project silently kept the old
|
|
18
|
+
* behaviour with nothing to report the drift.
|
|
19
|
+
*
|
|
20
|
+
* Every framework that solved this solved it the same way: the boot belongs to
|
|
21
|
+
* the framework and the project holds none of it. Next has `next start` and
|
|
22
|
+
* generates a `server.js` for `output: "standalone"` rather than asking anyone
|
|
23
|
+
* to write one. Nitro's default production preset is `node-server`, emitted
|
|
24
|
+
* into the build directory. The custom server is a documented opt-out, not the
|
|
25
|
+
* default everyone inherits — and here that opt-out is
|
|
26
|
+
* {@link createProjectServer}, which builds the server and binds nothing.
|
|
27
|
+
*
|
|
28
|
+
* ## Why the CLI rather than the runtime
|
|
29
|
+
*
|
|
30
|
+
* Booting a project needs three things at once: the runtime, the project's
|
|
31
|
+
* `.aai/` layout, and the prebuilt browser client. Only this package depends on
|
|
32
|
+
* all three — `aai-runtime` may not import `@alexkroman1/aai-ui`, and
|
|
33
|
+
* konsistent's `runtime-package-boundary` carries the install-weight argument
|
|
34
|
+
* for why that stays true. `client-dir.ts`'s own module doc records that this
|
|
35
|
+
* composition has always lived in an ENTRY POINT; this is that entry point,
|
|
36
|
+
* owned by the framework instead of copied into each project.
|
|
37
|
+
*
|
|
38
|
+
* The cost, stated because it is real: `npm start` needs `@alexkroman1/aai-cli`
|
|
39
|
+
* installed, so a production image carries a build toolchain it does not run.
|
|
40
|
+
* Next makes the same trade — `next` is a `dependency`, not a `devDependency` —
|
|
41
|
+
* and it is why the scaffold moves this package to `dependencies`.
|
|
42
|
+
*/
|
|
43
|
+
import type { AgentDef } from "@alexkroman1/aai";
|
|
44
|
+
import { type AgentServer } from "@alexkroman1/aai-runtime";
|
|
45
|
+
export { CLIENT_ARTIFACT_REL } from "./_artifacts.ts";
|
|
46
|
+
/** The port `aai start` binds when neither an argument nor `PORT` says otherwise. */
|
|
47
|
+
export declare const DEFAULT_START_PORT = 3000;
|
|
48
|
+
/** Options for {@link createProjectServer} and {@link executeStart}. */
|
|
49
|
+
export interface StartOptions {
|
|
50
|
+
/** Project root — the directory holding `agent.ts` and `.aai/`. */
|
|
51
|
+
cwd: string;
|
|
52
|
+
/** Port to bind. Defaults to `PORT`, then {@link DEFAULT_START_PORT}. */
|
|
53
|
+
port?: number | undefined;
|
|
54
|
+
/**
|
|
55
|
+
* Address to bind. Defaults to `HOST`, then loopback.
|
|
56
|
+
*
|
|
57
|
+
* Loopback is deliberate: this server has no request authentication of its
|
|
58
|
+
* own, so exposing it is an explicit act, and `HOST=0.0.0.0` is how a
|
|
59
|
+
* container says so. An EMPTY `HOST` means unset, not "every interface".
|
|
60
|
+
*/
|
|
61
|
+
host?: string | undefined;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Load the built agent, or fail saying what to run.
|
|
65
|
+
*
|
|
66
|
+
* A `file:` URL rather than a relative specifier, because on Windows a bare
|
|
67
|
+
* POSIX-looking path is not a valid module specifier.
|
|
68
|
+
*/
|
|
69
|
+
export declare function loadBuiltAgent(cwd: string): Promise<AgentDef>;
|
|
70
|
+
/**
|
|
71
|
+
* Build this project's {@link AgentServer} WITHOUT binding a socket.
|
|
72
|
+
*
|
|
73
|
+
* The seam a custom server is written against, and the one a serverless host
|
|
74
|
+
* needs: Vercel documents `export default <http.Server>` as its Node WebSocket
|
|
75
|
+
* shape and binds the socket itself, so such a host takes `AgentServer.node`
|
|
76
|
+
* and never calls `listen()`. `aai build --target vercel` emits an entry that
|
|
77
|
+
* does exactly this, so nothing host-specific is committed to a project.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```ts no-check
|
|
81
|
+
* // The entry `aai build --target vercel` bundles into
|
|
82
|
+
* // `.vercel/output/functions/index.func/` — `no-check` because this file
|
|
83
|
+
* // lives in the USER's project, where `@alexkroman1/aai-cli/start` resolves;
|
|
84
|
+
* // it cannot resolve from inside this package.
|
|
85
|
+
* import { createProjectServer } from "@alexkroman1/aai-cli/start";
|
|
86
|
+
*
|
|
87
|
+
* const server = (await createProjectServer({ cwd: import.meta.dirname })).node;
|
|
88
|
+
*
|
|
89
|
+
* export default function handler(req, res) {
|
|
90
|
+
* server.emit("request", req, res);
|
|
91
|
+
* }
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
export declare function createProjectServer(options: StartOptions): Promise<AgentServer>;
|
|
95
|
+
/** What {@link executeStart} answers, for `--json` and for tests. */
|
|
96
|
+
export interface StartResult {
|
|
97
|
+
ok: true;
|
|
98
|
+
data: {
|
|
99
|
+
name: string;
|
|
100
|
+
port: number | undefined;
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Serve the built agent and keep serving it: bind, announce, and shut down
|
|
105
|
+
* cleanly on a signal.
|
|
106
|
+
*
|
|
107
|
+
* The signal listeners are SYNCHRONOUS. An `async` one hands its promise to
|
|
108
|
+
* `process`, which discards what a listener returns — so a `close()` that
|
|
109
|
+
* rejected would surface as an unhandled rejection, i.e. a crash with a stack
|
|
110
|
+
* trace on Ctrl-C, instead of the non-zero exit a failed shutdown should be.
|
|
111
|
+
*/
|
|
112
|
+
export declare function executeStart(options: StartOptions): Promise<StartResult>;
|
package/dist/start.mjs
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { n as log } from "./_ui-DCt4qZrk.mjs";
|
|
3
|
+
import { n as WORKER_ARTIFACT_REL, t as CLIENT_ARTIFACT_REL } from "./_artifacts-BJOYGQPp.mjs";
|
|
4
|
+
import { n as resolveServerEnv, t as DEPLOY_ENV_FILES } from "./_server-common-De0haHr9.mjs";
|
|
5
|
+
import { existsSync } from "node:fs";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { pathToFileURL } from "node:url";
|
|
8
|
+
import { createAgentServer, ensureSessionStateSchema, ensureWorkflowJournalSchema, withHostCredentialFallback } from "@alexkroman1/aai-runtime";
|
|
9
|
+
import { defaultClientDir } from "@alexkroman1/aai-ui/client-dir";
|
|
10
|
+
//#region src/start.ts
|
|
11
|
+
/**
|
|
12
|
+
* `aai start` — serve a BUILT agent from a plain Node process.
|
|
13
|
+
*
|
|
14
|
+
* The deployment counterpart of `aai dev`: no file watching, no Vite, no
|
|
15
|
+
* typecheck, no source evaluation. It loads the artifact `aai build` left at
|
|
16
|
+
* {@link WORKER_ARTIFACT_REL} — the same one `aai publish` uploads and the
|
|
17
|
+
* managed platform runs — so a self-hosted agent and a deployed one cannot
|
|
18
|
+
* behave differently.
|
|
19
|
+
*
|
|
20
|
+
* ## Why this is a command rather than a file in every project
|
|
21
|
+
*
|
|
22
|
+
* It used to be `scaffold/server.mjs`, ~300 lines of boot that `aai init`
|
|
23
|
+
* copied into every scaffolded project: worker load, env resolution, schema
|
|
24
|
+
* DDL, client-directory probing, error classification, listen, signal
|
|
25
|
+
* handlers. Shipping that as source made each of those a fact a USER's
|
|
26
|
+
* repository asserted, so improving any of them reached only projects
|
|
27
|
+
* scaffolded afterwards, and an existing project silently kept the old
|
|
28
|
+
* behaviour with nothing to report the drift.
|
|
29
|
+
*
|
|
30
|
+
* Every framework that solved this solved it the same way: the boot belongs to
|
|
31
|
+
* the framework and the project holds none of it. Next has `next start` and
|
|
32
|
+
* generates a `server.js` for `output: "standalone"` rather than asking anyone
|
|
33
|
+
* to write one. Nitro's default production preset is `node-server`, emitted
|
|
34
|
+
* into the build directory. The custom server is a documented opt-out, not the
|
|
35
|
+
* default everyone inherits — and here that opt-out is
|
|
36
|
+
* {@link createProjectServer}, which builds the server and binds nothing.
|
|
37
|
+
*
|
|
38
|
+
* ## Why the CLI rather than the runtime
|
|
39
|
+
*
|
|
40
|
+
* Booting a project needs three things at once: the runtime, the project's
|
|
41
|
+
* `.aai/` layout, and the prebuilt browser client. Only this package depends on
|
|
42
|
+
* all three — `aai-runtime` may not import `@alexkroman1/aai-ui`, and
|
|
43
|
+
* konsistent's `runtime-package-boundary` carries the install-weight argument
|
|
44
|
+
* for why that stays true. `client-dir.ts`'s own module doc records that this
|
|
45
|
+
* composition has always lived in an ENTRY POINT; this is that entry point,
|
|
46
|
+
* owned by the framework instead of copied into each project.
|
|
47
|
+
*
|
|
48
|
+
* The cost, stated because it is real: `npm start` needs `@alexkroman1/aai-cli`
|
|
49
|
+
* installed, so a production image carries a build toolchain it does not run.
|
|
50
|
+
* Next makes the same trade — `next` is a `dependency`, not a `devDependency` —
|
|
51
|
+
* and it is why the scaffold moves this package to `dependencies`.
|
|
52
|
+
*/
|
|
53
|
+
/** The port `aai start` binds when neither an argument nor `PORT` says otherwise. */
|
|
54
|
+
const DEFAULT_START_PORT = 3e3;
|
|
55
|
+
/**
|
|
56
|
+
* Load the built agent, or fail saying what to run.
|
|
57
|
+
*
|
|
58
|
+
* A `file:` URL rather than a relative specifier, because on Windows a bare
|
|
59
|
+
* POSIX-looking path is not a valid module specifier.
|
|
60
|
+
*/
|
|
61
|
+
async function loadBuiltAgent(cwd) {
|
|
62
|
+
const workerPath = path.join(cwd, WORKER_ARTIFACT_REL);
|
|
63
|
+
if (!existsSync(workerPath)) throw new Error(`No built agent at ${WORKER_ARTIFACT_REL}. Run \`aai build\` first — the scaffold's \`prestart\` script normally does it for you.`);
|
|
64
|
+
return (await import(pathToFileURL(workerPath).href)).default;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Static assets to serve at `/`: this project's own built UI when it has one,
|
|
68
|
+
* otherwise the prebuilt default client that ships inside `@alexkroman1/aai-ui`.
|
|
69
|
+
*
|
|
70
|
+
* A `client.tsx` that has not been BUILT is worth saying out loud — the server
|
|
71
|
+
* would otherwise serve the default UI and look like it had ignored the file.
|
|
72
|
+
*/
|
|
73
|
+
function resolveClientDir(cwd) {
|
|
74
|
+
const built = path.join(cwd, CLIENT_ARTIFACT_REL);
|
|
75
|
+
if (existsSync(path.join(built, "index.html"))) return built;
|
|
76
|
+
if (existsSync(path.join(cwd, "client.tsx"))) log.warn("client.tsx is not built — serving the default UI. Run `aai build` first.");
|
|
77
|
+
return defaultClientDir();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Build this project's {@link AgentServer} WITHOUT binding a socket.
|
|
81
|
+
*
|
|
82
|
+
* The seam a custom server is written against, and the one a serverless host
|
|
83
|
+
* needs: Vercel documents `export default <http.Server>` as its Node WebSocket
|
|
84
|
+
* shape and binds the socket itself, so such a host takes `AgentServer.node`
|
|
85
|
+
* and never calls `listen()`. `aai build --target vercel` emits an entry that
|
|
86
|
+
* does exactly this, so nothing host-specific is committed to a project.
|
|
87
|
+
*
|
|
88
|
+
* @example
|
|
89
|
+
* ```ts no-check
|
|
90
|
+
* // The entry `aai build --target vercel` bundles into
|
|
91
|
+
* // `.vercel/output/functions/index.func/` — `no-check` because this file
|
|
92
|
+
* // lives in the USER's project, where `@alexkroman1/aai-cli/start` resolves;
|
|
93
|
+
* // it cannot resolve from inside this package.
|
|
94
|
+
* import { createProjectServer } from "@alexkroman1/aai-cli/start";
|
|
95
|
+
*
|
|
96
|
+
* const server = (await createProjectServer({ cwd: import.meta.dirname })).node;
|
|
97
|
+
*
|
|
98
|
+
* export default function handler(req, res) {
|
|
99
|
+
* server.emit("request", req, res);
|
|
100
|
+
* }
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
async function createProjectServer(options) {
|
|
104
|
+
const { cwd } = options;
|
|
105
|
+
const agent = await loadBuiltAgent(cwd);
|
|
106
|
+
const env = await resolveServerEnv(cwd, void 0, DEPLOY_ENV_FILES);
|
|
107
|
+
if (env.DATABASE_URL) {
|
|
108
|
+
await ensureSessionStateSchema({
|
|
109
|
+
url: env.DATABASE_URL,
|
|
110
|
+
logger: console
|
|
111
|
+
});
|
|
112
|
+
await ensureWorkflowJournalSchema({
|
|
113
|
+
url: env.DATABASE_URL,
|
|
114
|
+
logger: console
|
|
115
|
+
});
|
|
116
|
+
} else log.warn("No DATABASE_URL: session state and durable runs live in THIS process's memory.\nOne replica is fine. Behind a load balancer, enable sticky sessions so a reconnect (the client re-dials with ?sessionId=) reaches the same process — or set DATABASE_URL and let every replica share the state.");
|
|
117
|
+
return createAgentServer({
|
|
118
|
+
agent,
|
|
119
|
+
env,
|
|
120
|
+
providerEnv: withHostCredentialFallback(env),
|
|
121
|
+
clientDir: resolveClientDir(cwd),
|
|
122
|
+
...process.env.PUBLIC_URL?.trim() ? { publicUrl: process.env.PUBLIC_URL.trim() } : {}
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Serve the built agent and keep serving it: bind, announce, and shut down
|
|
127
|
+
* cleanly on a signal.
|
|
128
|
+
*
|
|
129
|
+
* The signal listeners are SYNCHRONOUS. An `async` one hands its promise to
|
|
130
|
+
* `process`, which discards what a listener returns — so a `close()` that
|
|
131
|
+
* rejected would surface as an unhandled rejection, i.e. a crash with a stack
|
|
132
|
+
* trace on Ctrl-C, instead of the non-zero exit a failed shutdown should be.
|
|
133
|
+
*/
|
|
134
|
+
async function executeStart(options) {
|
|
135
|
+
const agent = await loadBuiltAgent(options.cwd);
|
|
136
|
+
const server = await createProjectServer(options);
|
|
137
|
+
const port = options.port ?? Number(process.env.PORT ?? 3e3);
|
|
138
|
+
const host = options.host ?? (process.env.HOST?.trim() || void 0);
|
|
139
|
+
await server.listen(port, host);
|
|
140
|
+
log.info(`${agent.name} listening on http://${host ?? "127.0.0.1"}:${server.port}`);
|
|
141
|
+
for (const signal of ["SIGINT", "SIGTERM"]) process.once(signal, () => {
|
|
142
|
+
server.close().then(() => process.exit(0), (error) => {
|
|
143
|
+
log.error(`shutdown failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
144
|
+
process.exit(1);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
return {
|
|
148
|
+
ok: true,
|
|
149
|
+
data: {
|
|
150
|
+
name: agent.name,
|
|
151
|
+
port: server.port
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
//#endregion
|
|
156
|
+
export { CLIENT_ARTIFACT_REL, DEFAULT_START_PORT, createProjectServer, executeStart, loadBuiltAgent };
|
|
@@ -3,9 +3,9 @@ import { a as ok, t as CliError } from "./_output-DBP9Op_d.mjs";
|
|
|
3
3
|
import { n as log, t as fmtUrl } from "./_ui-DCt4qZrk.mjs";
|
|
4
4
|
import { l as formatCappedList } from "./_utils-D5JGcjiW.mjs";
|
|
5
5
|
import { s as updateProjectConfig } from "./_config-DYzC6WMD.mjs";
|
|
6
|
-
import {
|
|
6
|
+
import { n as resolveServerEnv } from "./_server-common-De0haHr9.mjs";
|
|
7
7
|
import { o as resolveDeployTarget } from "./_agent-BzUeqOdj.mjs";
|
|
8
|
-
import { layerScaffold } from "./_templates-
|
|
8
|
+
import { layerScaffold } from "./_templates-CIlJ3Vay.mjs";
|
|
9
9
|
import { a as publishStudioProject, c as studioProjectUrl, f as checkedResponse, i as projectNameFromDir, n as fetchStudioProject, o as pushStudioSource, r as listStudioProjects, s as studioProjectApiUrl, t as collectSourceFiles, u as apiRequest } from "./_studio-DnR_BqFp.mjs";
|
|
10
10
|
import { existsSync } from "node:fs";
|
|
11
11
|
import path from "node:path";
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// An EVAL: does the desk really DELEGATE, or does it brief you from memory?
|
|
2
|
+
//
|
|
3
|
+
// `agent.test.ts` settles what each tool does once it has been called — it
|
|
4
|
+
// hands `researchAngle` a `stubDelegate` and asserts on the board. What it
|
|
5
|
+
// cannot settle is the two things this template exists to demonstrate: that the
|
|
6
|
+
// MODEL turns "tell me about home battery prices" into ONE `research_topic`
|
|
7
|
+
// call carrying several angles rather than one call per angle, and that nothing
|
|
8
|
+
// the caller hears came from a web tool the DESK holds — because it holds none.
|
|
9
|
+
//
|
|
10
|
+
// Run it with `aai eval`. Without a provider key every case runs against a
|
|
11
|
+
// SCRIPTED model (its `stubReply`), which proves the wiring and nothing about
|
|
12
|
+
// the choice — so the two claims above are `{ live: true }` and the recap case,
|
|
13
|
+
// whose whole point is that it spends no model at all, is not.
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The def a DEPLOYED agent runs: authored, plus what `tools/` declares, plus
|
|
17
|
+
* its PROMPT.
|
|
18
|
+
*
|
|
19
|
+
* Taken from `virtual:aai/agent` rather than a hand-written glob: the plugin
|
|
20
|
+
* expands it against THIS file's own directory, so the spec needs no glob and
|
|
21
|
+
* no shared helper — which matters because this file SHIPS, and a scaffolded
|
|
22
|
+
* project has no repo helper to import. `agent.ts` here is three fields, so an
|
|
23
|
+
* eval driving it alone would measure an agent with no tools and the FRAMEWORK
|
|
24
|
+
* DEFAULT prompt — i.e. a desk that has never heard of a researcher, on which
|
|
25
|
+
* every claim below would pass or fail for the wrong reason. The reasoning is
|
|
26
|
+
* spelled out in `../code-interpreter/agent.eval.test.ts`.
|
|
27
|
+
*/
|
|
28
|
+
import agentDef from "virtual:aai/agent";
|
|
29
|
+
import { describeTurn, toolNames, toolResultIn } from "@alexkroman1/aai-runtime/eval";
|
|
30
|
+
import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
|
|
31
|
+
import { expect } from "vitest";
|
|
32
|
+
import { z } from "zod";
|
|
33
|
+
import { MAX_ANGLES } from "./shared.ts";
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Every tool the desk declares — the two that reach the outside world through a
|
|
37
|
+
* subagent, and the recap that reaches nothing.
|
|
38
|
+
*
|
|
39
|
+
* Named here because the isolation claim is stated as a NEGATIVE — no
|
|
40
|
+
* `web_search`, no `visit_webpage` — and a negative over a hand-typed list is
|
|
41
|
+
* the assertion that goes quietly true when a tool is renamed. Every call the
|
|
42
|
+
* desk makes must be one of these three names.
|
|
43
|
+
*/
|
|
44
|
+
const DESK_TOOLS: readonly string[] = ["research_topic", "verify_claim", "briefing_so_far"];
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* What `research_topic` answers with, as a schema rather than a cast.
|
|
48
|
+
*
|
|
49
|
+
* `toolResultIn` takes one for the reason `night-owl`'s spec records: a result
|
|
50
|
+
* that stopped carrying `findings` FAILS here naming the field, where a cast
|
|
51
|
+
* hands the assertions `undefined` and fails a line later on something
|
|
52
|
+
* unrelated.
|
|
53
|
+
*/
|
|
54
|
+
const BriefingResult = z.object({
|
|
55
|
+
topic: z.string(),
|
|
56
|
+
findings: z.array(z.object({ angle: z.string(), summary: z.string() })),
|
|
57
|
+
failed: z.array(z.object({ angle: z.string() })),
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
/** What `briefing_so_far` answers with on an EMPTY board. */
|
|
61
|
+
const EmptyRecap = z.object({
|
|
62
|
+
topic: z.null(),
|
|
63
|
+
findings: z.array(z.unknown()).length(0),
|
|
64
|
+
message: z.string(),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
/** What `verify_claim` answers with. */
|
|
68
|
+
const Verdict = z.object({ claim: z.string(), verdict: z.string() });
|
|
69
|
+
|
|
70
|
+
describeEval(agentDef, (test) => {
|
|
71
|
+
test(
|
|
72
|
+
"a recap on an empty board costs no researcher",
|
|
73
|
+
async ({ session }) => {
|
|
74
|
+
const turn = await session.say("What have you got for me so far?");
|
|
75
|
+
|
|
76
|
+
// The one tool here that spends no model at all, and the case that can
|
|
77
|
+
// therefore run against a script: the desk answers the recap out of its
|
|
78
|
+
// own slot. A desk that reached for `research_topic` to find out what it
|
|
79
|
+
// already knows is the regression — it bills the caller for four
|
|
80
|
+
// researchers to answer "nothing yet".
|
|
81
|
+
expect(toolNames(turn.toolCalls), describeTurn(turn)).toEqual(["briefing_so_far"]);
|
|
82
|
+
// And the slot really resolved: an empty board reports itself as empty
|
|
83
|
+
// rather than throwing or answering with a half-built shape.
|
|
84
|
+
expect(toolResultIn(turn.toolCalls, "briefing_so_far", EmptyRecap).topic).toBeNull();
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
stubReply: [
|
|
88
|
+
{ tool: "briefing_so_far", args: {} },
|
|
89
|
+
"Nothing yet — tell me a subject and I'll put some researchers on it.",
|
|
90
|
+
],
|
|
91
|
+
},
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
test(
|
|
95
|
+
"fans one subject out across angles in a SINGLE call",
|
|
96
|
+
async ({ session }) => {
|
|
97
|
+
const turn = await session.say("What's going on with home battery prices?");
|
|
98
|
+
|
|
99
|
+
// One call carrying several angles, never one call per angle. That is the
|
|
100
|
+
// whole economic claim of the template — the angles run in parallel, so
|
|
101
|
+
// the caller waits for the slowest rather than the sum — and a desk that
|
|
102
|
+
// called the tool three times in a row would satisfy any assertion that
|
|
103
|
+
// merely counted angles.
|
|
104
|
+
const calls = turn.toolCalls.filter((call) => call.name === "research_topic");
|
|
105
|
+
expect(calls, describeTurn(turn)).toHaveLength(1);
|
|
106
|
+
const angles = z.array(z.string()).parse(calls[0]?.args.angles);
|
|
107
|
+
expect(angles.length).toBeGreaterThan(1);
|
|
108
|
+
expect(angles.length).toBeLessThanOrEqual(MAX_ANGLES);
|
|
109
|
+
// Each angle stands on its own — the prompt's rule, and the one a
|
|
110
|
+
// researcher cannot recover from, having heard none of this call. A bare
|
|
111
|
+
// "the same for Europe" is a handful of characters; a self-contained
|
|
112
|
+
// question is a sentence.
|
|
113
|
+
for (const angle of angles) expect(angle.length).toBeGreaterThan(15);
|
|
114
|
+
|
|
115
|
+
// What came back is what the researchers concluded, not what the desk
|
|
116
|
+
// believes: every finding carries prose, and the board holds the topic.
|
|
117
|
+
const result = toolResultIn(turn.toolCalls, "research_topic", BriefingResult);
|
|
118
|
+
expect(result.findings).not.toEqual([]);
|
|
119
|
+
for (const finding of result.findings) expect(finding.summary.length).toBeGreaterThan(40);
|
|
120
|
+
|
|
121
|
+
// The isolation claim, and the reason this template is not
|
|
122
|
+
// `web-researcher`: the desk has NO web tools, so anything it says about
|
|
123
|
+
// the world crossed back out of a subagent. A `web_search` in this list
|
|
124
|
+
// would mean the builtins had leaked onto the parent.
|
|
125
|
+
expect(toolNames(turn.toolCalls).filter((name) => !DESK_TOOLS.includes(name))).toEqual([]);
|
|
126
|
+
expect(turn.text).not.toBe("");
|
|
127
|
+
},
|
|
128
|
+
// Live only: a script cannot choose the angles, and choosing them is the
|
|
129
|
+
// measurement. It also cannot run a subagent — the researcher resolves a
|
|
130
|
+
// model of its own from `shared.ts`, which the turn's stub never covers.
|
|
131
|
+
{ live: true },
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
test(
|
|
135
|
+
"checks a claim the caller pushes back on instead of defending it",
|
|
136
|
+
async ({ session }) => {
|
|
137
|
+
const turn = await session.say(
|
|
138
|
+
"Someone told me home batteries pay for themselves in two years. " +
|
|
139
|
+
"Is that right? Check it for me.",
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
// `verify_claim` is the cheaper subagent on the narrower surface, and
|
|
143
|
+
// reaching for it rather than answering is the prompt's rule that a live
|
|
144
|
+
// model actually has to keep. The claim it forwards must be the sentence
|
|
145
|
+
// it was given, not a keyword — a fact-checker handed "batteries" answers
|
|
146
|
+
// confidently about nothing.
|
|
147
|
+
const calls = turn.toolCalls.filter((call) => call.name === "verify_claim");
|
|
148
|
+
expect(calls, describeTurn(turn)).toHaveLength(1);
|
|
149
|
+
const verdict = toolResultIn(turn.toolCalls, "verify_claim", Verdict);
|
|
150
|
+
expect(verdict.claim.split(/\s+/).length).toBeGreaterThan(3);
|
|
151
|
+
expect(verdict.verdict).not.toBe("");
|
|
152
|
+
expect(toolNames(turn.toolCalls).filter((name) => !DESK_TOOLS.includes(name))).toEqual([]);
|
|
153
|
+
},
|
|
154
|
+
{ live: true },
|
|
155
|
+
);
|
|
156
|
+
});
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/** The def a DEPLOYED agent runs: authored, plus the `system-prompt.md` beside it. */
|
|
2
|
+
import agentDef from "virtual:aai/agent";
|
|
3
|
+
import { AgentConfigSchema, toAgentConfig } from "@alexkroman1/aai/manifest";
|
|
4
|
+
import { describe, expect, test } from "vitest";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What this starter's spec may assert.
|
|
8
|
+
*
|
|
9
|
+
* Renaming Coda, giving her a voice, swapping a stage or switching the whole
|
|
10
|
+
* thing to speech-to-speech are the first edits this template invites — and
|
|
11
|
+
* `aai build` runs these tests before it bundles, so an assertion that pins the
|
|
12
|
+
* template's own identity turns the first customization into a build failure in
|
|
13
|
+
* a file the author never wrote. Every test here asserts a property that
|
|
14
|
+
* survives those edits, on the RESOLVED config rather than on the def's empty
|
|
15
|
+
* fields.
|
|
16
|
+
*
|
|
17
|
+
* What it may NOT assert is whether `run_code` actually RUNS: the builtin is
|
|
18
|
+
* sandbox-only, so off-platform it declines rather than evaluating
|
|
19
|
+
* model-written JavaScript in the host process. That is right, and it leaves
|
|
20
|
+
* "Coda reached for code, and the code came back with 107823" to
|
|
21
|
+
* `agent.eval.test.ts`, which supplies an executor of its own. What is reachable
|
|
22
|
+
* in memory is the pairing the template is built on — the builtin it asks for
|
|
23
|
+
* and the prompt that commands it — and that is what the last two tests are.
|
|
24
|
+
*/
|
|
25
|
+
describe("code-interpreter template", () => {
|
|
26
|
+
test("config passes manifest validation", () => {
|
|
27
|
+
// Same conversion `aai build`/`aai deploy` run.
|
|
28
|
+
expect(() => toAgentConfig(agentDef)).not.toThrow();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("exports an agent the platform can name", () => {
|
|
32
|
+
// Not the literal: what has to hold is that there IS a name and that the
|
|
33
|
+
// conversion carries it through — `AgentName` refuses a blank one, and the
|
|
34
|
+
// studio lists a deployed agent by exactly this string.
|
|
35
|
+
expect(agentDef.name).toBeTruthy();
|
|
36
|
+
expect(toAgentConfig(agentDef).name).toBe(agentDef.name);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("whatever mode it ends up in, there is a model to call the tool with", () => {
|
|
40
|
+
// A builtin is something a MODEL reaches for, so this template is only
|
|
41
|
+
// itself while some stage can issue a tool call. Asserted per MODE so it
|
|
42
|
+
// survives the swaps above: declare `stt`/`llm`/`tts` and the rest still
|
|
43
|
+
// default to the all-AssemblyAI cascade (which is why the def declares no
|
|
44
|
+
// provider at all and still runs); declare `s2s` and there is no cascade to
|
|
45
|
+
// fill, which is the one thing that must never happen by fallthrough.
|
|
46
|
+
const config = toAgentConfig(agentDef);
|
|
47
|
+
if (config.mode === "s2s") {
|
|
48
|
+
expect(config.s2s?.kind).toBeTruthy();
|
|
49
|
+
expect(config.stt).toBeUndefined();
|
|
50
|
+
expect(config.tts).toBeUndefined();
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
expect(config.llm?.kind).toBeTruthy();
|
|
54
|
+
if (config.mode === "pipeline") {
|
|
55
|
+
expect(config.stt?.kind).toBeTruthy();
|
|
56
|
+
expect(config.tts?.kind).toBeTruthy();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("run_code survives into the config a deploy carries", () => {
|
|
61
|
+
// The template's whole capability, asserted on the CONFIG rather than the
|
|
62
|
+
// def because that is what a deploy ships. `DEFAULT_BUILTIN_TOOLS` is
|
|
63
|
+
// EMPTY — a builtin is something an agent asks for, never something it has
|
|
64
|
+
// to notice and switch off — so a dropped `builtinTools` is not a degraded
|
|
65
|
+
// Coda, it is an agent whose prompt forbids mental arithmetic and leaves it
|
|
66
|
+
// nothing else to do. Adding builtins beside it is fine; losing this one is
|
|
67
|
+
// the regression.
|
|
68
|
+
expect(toAgentConfig(agentDef).builtinTools ?? []).toContain("run_code");
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("every builtin the prompt commands by name is one the agent declares", () => {
|
|
72
|
+
// The pairing that makes this template work, and the failure it catches is
|
|
73
|
+
// silent in both directions: a prompt commanding `fetch_json` at an agent
|
|
74
|
+
// that never declared it produces a model apologizing for a tool it cannot
|
|
75
|
+
// see, and a builtin renamed in `agent.ts` alone leaves the CRITICAL RULES
|
|
76
|
+
// addressed to nothing. Neither shows up in a diff of either file.
|
|
77
|
+
const config = toAgentConfig(agentDef);
|
|
78
|
+
const declared = config.builtinTools ?? [];
|
|
79
|
+
|
|
80
|
+
// Which snake_case tokens in the prose are TOOL NAMES is a question for the
|
|
81
|
+
// SDK's own schema, not for a list restated here: a catalog copied into a
|
|
82
|
+
// spec goes stale, and matching every underscored word would redden on a
|
|
83
|
+
// prompt that names a variable in one of its examples.
|
|
84
|
+
const isBuiltin = (name: string) =>
|
|
85
|
+
AgentConfigSchema.safeParse({ ...config, builtinTools: [name] }).success;
|
|
86
|
+
const commanded = [
|
|
87
|
+
...new Set(config.systemPrompt.match(/\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b/g) ?? []),
|
|
88
|
+
].filter(isBuiltin);
|
|
89
|
+
|
|
90
|
+
// Non-vacuity, and it earns its keep twice: a prompt naming no builtin at
|
|
91
|
+
// all would make the loop below assert nothing, and it is also the state a
|
|
92
|
+
// template gets into when `system-prompt.md` is not applied — the framework
|
|
93
|
+
// default names no builtin, so an agent running on it lands here rather
|
|
94
|
+
// than passing quietly with the CRITICAL RULES nowhere in its context.
|
|
95
|
+
expect(commanded.length).toBeGreaterThan(0);
|
|
96
|
+
for (const name of commanded) {
|
|
97
|
+
expect(declared, `the prompt commands ${name}`).toContain(name);
|
|
98
|
+
}
|
|
99
|
+
// The converse is deliberately NOT asserted: declaring a builtin the prompt
|
|
100
|
+
// never mentions is an ordinary edit, and the model is told about it by its
|
|
101
|
+
// own tool schema.
|
|
102
|
+
});
|
|
103
|
+
});
|
|
@@ -62,9 +62,32 @@
|
|
|
62
62
|
* what narrows it to the newest line, because on a page this small that is the
|
|
63
63
|
* whole of what a status wants; `transcription-workflow` renders the full log,
|
|
64
64
|
* where a fan-out makes the history worth seeing.
|
|
65
|
+
*
|
|
66
|
+
* ## Two things that are only ever true for a MOMENT
|
|
67
|
+
*
|
|
68
|
+
* The Copy button and the reply to "File it now" are both a word that appears
|
|
69
|
+
* and then goes away, and both used to be the sort of thing a page writes with
|
|
70
|
+
* a `useState` and a bare `setTimeout` — which gets two things wrong that only
|
|
71
|
+
* show up on the second click (a second flash has its window cut short by the
|
|
72
|
+
* first one's timer) and on unmount (a `setState` into a torn-down tree). They
|
|
73
|
+
* are `useCopy` and `useFlash` from `@alexkroman1/aai-ui`.
|
|
74
|
+
*
|
|
75
|
+
* Reach for `useCopy` when the moment is a clipboard write — it keys the flash
|
|
76
|
+
* by the copied TEXT, so on a page with several copy buttons only the one
|
|
77
|
+
* clicked lights up, and it reports a REFUSED write as `"Failed"` rather than
|
|
78
|
+
* doing nothing visible (there is no clipboard at all on an insecure origin).
|
|
79
|
+
* Reach for `useFlash` for any other transient word; here it carries what
|
|
80
|
+
* `wake()` answered, which is a number and not a failure at 0.
|
|
65
81
|
*/
|
|
66
82
|
|
|
67
|
-
import {
|
|
83
|
+
import {
|
|
84
|
+
BulletList,
|
|
85
|
+
mountPage,
|
|
86
|
+
useCopy,
|
|
87
|
+
useFlash,
|
|
88
|
+
useWorkflowSubmit,
|
|
89
|
+
WorkflowProgress,
|
|
90
|
+
} from "@alexkroman1/aai-ui";
|
|
68
91
|
import "@alexkroman1/aai-ui/styles.css";
|
|
69
92
|
// ERASED at build time, so naming the agent's own type costs the browser bundle
|
|
70
93
|
// nothing — and it is what stops this file restating a shape `workflows/
|
|
@@ -90,8 +113,22 @@ function pendingNote(startedHere: boolean, found: boolean): string {
|
|
|
90
113
|
return "Still working on the digest this tab started earlier. Reloading is safe.";
|
|
91
114
|
}
|
|
92
115
|
|
|
116
|
+
/** The digest as one pasteable block — a headline and its bullets. */
|
|
117
|
+
function asText(headline: string, points: readonly string[]): string {
|
|
118
|
+
return [headline, ...points.map((point) => `- ${point}`)].join("\n");
|
|
119
|
+
}
|
|
120
|
+
|
|
93
121
|
export function App() {
|
|
94
122
|
const [url, setUrl] = useState("");
|
|
123
|
+
// One copier for the page. It would be one per GROUP of copy buttons on a
|
|
124
|
+
// bigger page — the flash is shared, so clicking a second row clears the
|
|
125
|
+
// first row's "Copied", which is what stops two rows both claiming to be on
|
|
126
|
+
// the clipboard.
|
|
127
|
+
const copier = useCopy();
|
|
128
|
+
// `wake()` resolves with how many sleeps it ended, and 0 is an ANSWER (the
|
|
129
|
+
// run had already moved past its wait) rather than a failure — so the button
|
|
130
|
+
// says which happened, for a moment, and then goes back to being a button.
|
|
131
|
+
const woken = useFlash<string>();
|
|
95
132
|
// The generic is what makes `run.status === "completed"` narrow to a TYPED
|
|
96
133
|
// `run.output` instead of `unknown`. `error` is the agent's own sentence for a
|
|
97
134
|
// rejected input, which is better copy than anything this page could write, and
|
|
@@ -168,10 +205,14 @@ export function App() {
|
|
|
168
205
|
{pending && (
|
|
169
206
|
<button
|
|
170
207
|
type="button"
|
|
171
|
-
onClick={() =>
|
|
208
|
+
onClick={() => {
|
|
209
|
+
void wake().then((count) =>
|
|
210
|
+
woken.flash(count > 0 ? "Filing it now" : "Already past its wait"),
|
|
211
|
+
);
|
|
212
|
+
}}
|
|
172
213
|
className="self-start rounded-md border px-3 py-1 text-sm"
|
|
173
214
|
>
|
|
174
|
-
File it now
|
|
215
|
+
{woken.value ?? "File it now"}
|
|
175
216
|
</button>
|
|
176
217
|
)}
|
|
177
218
|
|
|
@@ -189,6 +230,17 @@ export function App() {
|
|
|
189
230
|
<h2 className="text-xl">{run.output.headline}</h2>
|
|
190
231
|
<BulletList items={run.output.points} />
|
|
191
232
|
<p className="text-sm opacity-70">Filed {run.output.filedAt}</p>
|
|
233
|
+
{/* The whole digest as plain text, which is what somebody pasting it
|
|
234
|
+
into a note wants. `copier.label` is the button's own text: it
|
|
235
|
+
reads "Copy" until it is clicked, then "Copied" — or "Failed",
|
|
236
|
+
which is the case a hand-rolled version silently drops. */}
|
|
237
|
+
<button
|
|
238
|
+
type="button"
|
|
239
|
+
onClick={() => copier.copy(asText(run.output.headline, run.output.points))}
|
|
240
|
+
className="self-start rounded-md border px-3 py-1 text-sm"
|
|
241
|
+
>
|
|
242
|
+
{copier.label(asText(run.output.headline, run.output.points))}
|
|
243
|
+
</button>
|
|
192
244
|
</article>
|
|
193
245
|
)}
|
|
194
246
|
</main>
|