@alexkroman1/aai-cli 13.2.0 → 13.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +1 -1
- package/dist/{_agent-BTwZJu4P.mjs → _agent-Clbq3a4h.mjs} +2 -2
- package/dist/{_bundler-C31rlcYz.mjs → _bundler-DEkjO1d8.mjs} +3 -3
- package/dist/{_config-CmJOFsAP.mjs → _config-ZY2euhdh.mjs} +7 -4
- package/dist/_config.d.ts +25 -1
- package/dist/{_dev-server-DWvkNoIS.mjs → _dev-server-C13xuGB1.mjs} +13 -13
- package/dist/_e2e-test-utils.d.ts +15 -7
- package/dist/{_init-BURbdHNM.mjs → _init-Ci4QGjp8.mjs} +58 -10
- package/dist/{_output-CKkmWs7i.mjs → _output-DbJt9gsl.mjs} +34 -3
- package/dist/_output.d.ts +17 -1
- package/dist/{_preflight-Co1G9Jww.mjs → _preflight-B2laWPdU.mjs} +1 -1
- package/dist/{_server-common-6e0QI4mq.mjs → _server-common-BV4_CrP3.mjs} +3 -3
- package/dist/{_slug-api-CfGQq1Tt.mjs → _slug-api-DlKpj91X.mjs} +3 -3
- package/dist/{_studio-B1waXMuR.mjs → _studio-DTi3EdXQ.mjs} +3 -3
- package/dist/{_templates-0Zr2z6yA.mjs → _templates-V4l5sZJy.mjs} +2 -2
- package/dist/{_typecheck-gate-BXvUNnfs.mjs → _typecheck-gate-sV4N5F93.mjs} +3 -3
- package/dist/{_ui-u7T4YooX.mjs → _ui-DCt4qZrk.mjs} +1 -1
- package/dist/{_utils-B8QmtFhK.mjs → _utils-Dhd058D9.mjs} +1 -1
- package/dist/{_vite-env-D9Qj6KPX.mjs → _vite-env-kejF0p2L.mjs} +1 -1
- package/dist/{_workflow-determinism-f9J-5Zlj.mjs → _workflow-determinism-C9l8uC2R.mjs} +1 -1
- package/dist/{build-BLlLOrng.mjs → build-CWyFGe-W.mjs} +11 -8
- package/dist/cli.d.ts +2 -1
- package/dist/cli.mjs +120 -46
- package/dist/{client-bundler-j5c46x0G.mjs → client-bundler-CT-_26c5.mjs} +4 -4
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-X_CRgCf6.mjs → delete-RqlQZ6Y_.mjs} +6 -6
- package/dist/{deploy-BzZ-r4iM.mjs → deploy-B7f3CK7v.mjs} +13 -13
- package/dist/{dev-BDykFvcc.mjs → dev-DamXieW_.mjs} +5 -5
- package/dist/{eval-0qEOdm0l.mjs → eval-h3DTVfRw.mjs} +27 -11
- package/dist/eval.d.ts +2 -0
- package/dist/{init-BtJI306C.mjs → init-D-vFrV8h.mjs} +7 -7
- package/dist/{login-DeUETobb.mjs → login-CDiemNmC.mjs} +5 -5
- package/dist/{logs-hWnJ66Bl.mjs → logs-DGwPNCzB.mjs} +4 -4
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/.env.example +17 -1
- package/dist/scaffold/package.json +6 -5
- package/dist/secret-V1w7y9-j.mjs +162 -0
- package/dist/secret.d.ts +41 -8
- package/dist/{studio-BxAS_FQQ.mjs → studio-BpMCfUNx.mjs} +9 -9
- package/dist/templates/briefing-desk/agent.test.ts +7 -3
- package/dist/templates/call-audit/agent.test.ts +14 -3
- package/dist/templates/call-audit/client.tsx +14 -17
- package/dist/templates/embedded-assets/agent.test.ts +7 -1
- package/dist/templates/health-assistant/agent.test.ts +8 -4
- package/dist/templates/link-digest/agent.test.ts +7 -1
- package/dist/templates/link-digest/client.tsx +17 -21
- package/dist/templates/night-owl/agent.test.ts +6 -1
- package/dist/templates/pipeline-simple/agent.test.ts +31 -14
- package/dist/templates/podcast-digest/agent.test.ts +13 -3
- package/dist/templates/podcast-digest/client.tsx +12 -15
- package/dist/templates/recap-workflow/agent.test.ts +22 -9
- package/dist/templates/redline/agent.test.ts +7 -1
- package/dist/templates/redline/client.tsx +12 -15
- package/dist/templates/research-workflow/agent.test.ts +7 -1
- package/dist/templates/retail/registry.test.ts +99 -58
- package/dist/templates/simple/agent.test.ts +39 -6
- package/dist/templates/spoken-summary/agent.test.ts +12 -3
- package/dist/templates/spoken-summary/client.tsx +14 -18
- package/dist/templates/transcription-workflow/agent.test.ts +16 -8
- package/dist/templates/transcription-workflow/client.tsx +19 -26
- package/dist/test-guenJUxi.mjs +255 -0
- package/dist/test.d.ts +97 -31
- package/dist/typecheck.mjs +2 -2
- package/dist/worker-bundler.mjs +3 -3
- package/dist/{workflow-BBu5oEDw.mjs → workflow-BZyKLkhV.mjs} +5 -5
- package/package.json +10 -10
- package/dist/bin.d.mts +0 -2
- package/dist/secret-B-3T5tvr.mjs +0 -61
- package/dist/test-CiLab-AA.mjs +0 -182
- package/dist/tsdown.config.d.ts +0 -2
|
@@ -143,15 +143,19 @@ function wavFile(
|
|
|
143
143
|
return head;
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
describe("the agent declares its three workflows
|
|
146
|
+
describe("the agent declares its three workflows", () => {
|
|
147
147
|
test("under the names the REST route resolves them by", () => {
|
|
148
148
|
// The page starts a run by these strings, so a rename is a runtime 400 rather
|
|
149
149
|
// than a compile error — which is what makes pinning them worth a test.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
150
|
+
// `arrayContaining` rather than an exact key list: a fourth desk of your own
|
|
151
|
+
// is an invited edit and must not redden a test you did not write. These
|
|
152
|
+
// three NAMES stay pinned, deliberately — the page starts a run by each
|
|
153
|
+
// string, so renaming one is a runtime 400 rather than a compile error, and
|
|
154
|
+
// this is the only thing that says so. Rename here and in `client.tsx`
|
|
155
|
+
// together.
|
|
156
|
+
expect(Object.keys(agentDef.workflows ?? {})).toEqual(
|
|
157
|
+
expect.arrayContaining(["transcribe", "transcribeStream", "transcribeBatch"]),
|
|
158
|
+
);
|
|
155
159
|
expect(agentDef.workflows?.transcribe).toBe(transcribe);
|
|
156
160
|
expect(agentDef.workflows?.transcribeStream).toBe(transcribeStream);
|
|
157
161
|
expect(agentDef.workflows?.transcribeBatch).toBe(transcribeBatch);
|
|
@@ -168,8 +172,12 @@ describe("the agent declares its three workflows and nothing else", () => {
|
|
|
168
172
|
});
|
|
169
173
|
|
|
170
174
|
test("with no tools, because the interface is the page and the API", () => {
|
|
171
|
-
//
|
|
172
|
-
//
|
|
175
|
+
// A PIN, deliberately, and the one assertion in this file you are expected
|
|
176
|
+
// to edit rather than satisfy. `workflowApp()` declares no session, so a
|
|
177
|
+
// tool has no conversation to be called from — a name appearing here means
|
|
178
|
+
// the voice path crept back in and the tool will never run. If you want a
|
|
179
|
+
// voice agent that STARTS a workflow, `research-workflow` is that template,
|
|
180
|
+
// and this line is what tells you you are on the wrong one.
|
|
173
181
|
expect(Object.keys(agentDef.tools ?? {})).toEqual([]);
|
|
174
182
|
});
|
|
175
183
|
|
|
@@ -153,15 +153,17 @@
|
|
|
153
153
|
* ## A reload keeps two of the three runs, and the third CANNOT be kept
|
|
154
154
|
*
|
|
155
155
|
* The run id lives in React state, so a refresh loses it while the fan-out
|
|
156
|
-
* carries on.
|
|
157
|
-
*
|
|
156
|
+
* carries on. A correlation KEY is the handle that survives that, and
|
|
157
|
+
* `useWorkflowSubmit` keeps one and asks for its newest run on mount — which
|
|
158
|
+
* here lands differently PER MODE:
|
|
158
159
|
*
|
|
159
160
|
* - **"After it uploads"** and **"Let the provider do it"** recover. Their input
|
|
160
161
|
* names a recording that is already stored, so a later load adopting the run
|
|
161
162
|
* is adopting something complete: the transcript arrives, the progress log
|
|
162
163
|
* replays, and nobody is asked to send a 600 MB file a second time.
|
|
163
164
|
* - **"While it uploads" does not, and the hook REFUSES the option rather than
|
|
164
|
-
* ignoring it
|
|
165
|
+
* ignoring it** — `useWorkflowStream` omits `recover` from its options type
|
|
166
|
+
* and mints no key. That run's input names an upload id this page load minted and
|
|
165
167
|
* is still filling, so a later load could only adopt a run waiting for bytes
|
|
166
168
|
* nobody is sending — and it is worse than useless: `workflows/stream.ts`
|
|
167
169
|
* fails a run whose upload stops growing (`MAX_IDLE_POLLS`), so the reload
|
|
@@ -171,10 +173,11 @@
|
|
|
171
173
|
* The MODE is remembered too, and that is not decoration: without it a reload
|
|
172
174
|
* opens on the default flow while the recovered run sits behind a radio nobody
|
|
173
175
|
* pressed, so the reader sees an empty form and starts a second run — the exact
|
|
174
|
-
* thing the key exists to prevent. The KEY is
|
|
175
|
-
* minting, the storage and the argument for
|
|
176
|
-
* `?key=` parameter
|
|
177
|
-
*
|
|
176
|
+
* thing the key exists to prevent. The KEY is the hook's own — `use-run-key.ts`
|
|
177
|
+
* owns the minting, the storage and the argument for it being opaque rather
|
|
178
|
+
* than a `?key=` parameter, and the two submit hooks here share one because the
|
|
179
|
+
* slot is keyed by the PAGE; `recover.ts` owns the mode, which is this page's
|
|
180
|
+
* own concept, and the validation on the way back out of storage that turning a
|
|
178
181
|
* stored string into a workflow name obliges.
|
|
179
182
|
*
|
|
180
183
|
* Two smaller consequences worth knowing. Both recovering hooks look up on
|
|
@@ -193,7 +196,6 @@ import {
|
|
|
193
196
|
page,
|
|
194
197
|
SubmitButton,
|
|
195
198
|
UploadProgressBar,
|
|
196
|
-
useRunKey,
|
|
197
199
|
useWorkflowRuns,
|
|
198
200
|
useWorkflowStream,
|
|
199
201
|
useWorkflowSubmit,
|
|
@@ -268,9 +270,6 @@ function TranscriptionDesk() {
|
|
|
268
270
|
// piece of state for all three hooks, because it describes the UPLOAD and every
|
|
269
271
|
// mode has one — see the module doc.
|
|
270
272
|
const [parallel, setParallel] = useState(true);
|
|
271
|
-
// This tab's handle on its own runs — minted once and remembered, which is
|
|
272
|
-
// what a later load produces to find the run again.
|
|
273
|
-
const key = useRunKey();
|
|
274
273
|
// Did THIS load press Transcribe? A reload cannot have, and it is the only way
|
|
275
274
|
// the page can tell "working on what you just sent" from "picking up where you
|
|
276
275
|
// left off" — the hooks report the run, not who asked for it.
|
|
@@ -279,22 +278,16 @@ function TranscriptionDesk() {
|
|
|
279
278
|
// and that costs nothing here: none of them does anything until its `submit` is
|
|
280
279
|
// called, and `useWorkflowRun` underneath them holds no id until then either.
|
|
281
280
|
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
281
|
+
// The two submit hooks recover unconditionally rather than on `mode === …`:
|
|
282
|
+
// the lookup is a MOUNT-time act, so arming it when a mode is picked would
|
|
283
|
+
// re-adopt a run the reader had just cleared. They share ONE key without
|
|
284
|
+
// saying so — the hook's slot is keyed by the page — and `find` is scoped by
|
|
285
|
+
// workflow as well as by key, so the two modes recover separate runs. The
|
|
286
|
+
// streaming hook has neither half: it refuses `recover` by type, and mints no
|
|
287
|
+
// key it would never read back.
|
|
287
288
|
const streamed = useWorkflowStream<typeof transcribe>(WORKFLOWS.streaming, { parallel });
|
|
288
|
-
const stored = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.classic, {
|
|
289
|
-
|
|
290
|
-
key,
|
|
291
|
-
recover: true,
|
|
292
|
-
});
|
|
293
|
-
const batched = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.batch, {
|
|
294
|
-
parallel,
|
|
295
|
-
key,
|
|
296
|
-
recover: true,
|
|
297
|
-
});
|
|
289
|
+
const stored = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.classic, { parallel });
|
|
290
|
+
const batched = useWorkflowSubmit<typeof transcribe>(WORKFLOWS.batch, { parallel });
|
|
298
291
|
// The batch flow uploads the same way the classic one does — the id comes from the
|
|
299
292
|
// store — so it is the SAME hook against a different workflow. Only the streaming
|
|
300
293
|
// mode needs the other one, because only it needs the id before the bytes.
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { a as ok, n as fail } from "./_output-DbJt9gsl.mjs";
|
|
3
|
+
import { n as log, r as notify } from "./_ui-DCt4qZrk.mjs";
|
|
4
|
+
import { i as errorCode, o as errorMessage$1, r as binFromPackageJson } from "./_utils-Dhd058D9.mjs";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { omitUndefined } from "@alexkroman1/aai/utils";
|
|
9
|
+
import { execaSync } from "execa";
|
|
10
|
+
//#region src/test.ts
|
|
11
|
+
/**
|
|
12
|
+
* `aai test` — run agent tests via vitest.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the agent project's own vitest binary so tests run without the
|
|
16
|
+
* npx resolution overhead (and its potential network fetch of vitest).
|
|
17
|
+
*
|
|
18
|
+
* Resolves `vitest/package.json` from the agent directory, derives the bin
|
|
19
|
+
* script, and runs it with the current Node executable. Falls back to
|
|
20
|
+
* `npx vitest` only when no local install is resolvable.
|
|
21
|
+
*/
|
|
22
|
+
function resolveVitestCommand(cwd, resolve = createRequire(path.join(cwd, "package.json")).resolve) {
|
|
23
|
+
try {
|
|
24
|
+
const bin = binFromPackageJson(resolve("vitest/package.json"), "vitest");
|
|
25
|
+
if (bin) return {
|
|
26
|
+
cmd: process.execPath,
|
|
27
|
+
args: [bin]
|
|
28
|
+
};
|
|
29
|
+
} catch {}
|
|
30
|
+
return {
|
|
31
|
+
cmd: "npx",
|
|
32
|
+
args: ["vitest"]
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** The files `aai test` runs, in preference order. */
|
|
36
|
+
const TEST_FILES = ["agent.test.ts", "agent.test.js"];
|
|
37
|
+
/**
|
|
38
|
+
* Run vitest over `candidates` (or, with `all`, the whole project) in the given
|
|
39
|
+
* project directory.
|
|
40
|
+
*
|
|
41
|
+
* Returns the FILES it ran, or `false` if there was nothing to run. Throws on
|
|
42
|
+
* failure.
|
|
43
|
+
*
|
|
44
|
+
* A vitest FILTER, not an include glob: each argument is matched as a substring
|
|
45
|
+
* against the paths vitest's own include globs already found, which is why the
|
|
46
|
+
* candidates are named `*.test.ts` — `agent.test.ts` cannot match
|
|
47
|
+
* `agent.eval.test.ts` and vice versa, so the two commands stay disjoint
|
|
48
|
+
* without either one having to exclude the other's file. That holds for the
|
|
49
|
+
* `all` list too: it is built from {@link projectSpecFiles}, which drops the
|
|
50
|
+
* eval tier by infix.
|
|
51
|
+
*/
|
|
52
|
+
function runVitest(cwd, opts = { candidates: TEST_FILES }) {
|
|
53
|
+
const files = resolveRunFiles(cwd, opts);
|
|
54
|
+
if (files.length === 0) return false;
|
|
55
|
+
const { cmd, args } = resolveVitestCommand(cwd);
|
|
56
|
+
execaSync(cmd, [
|
|
57
|
+
...args,
|
|
58
|
+
"run",
|
|
59
|
+
"--root",
|
|
60
|
+
".",
|
|
61
|
+
...opts.extraArgs ?? [],
|
|
62
|
+
...files
|
|
63
|
+
], {
|
|
64
|
+
cwd,
|
|
65
|
+
stdio: "inherit",
|
|
66
|
+
...omitUndefined({ env: opts.env ? {
|
|
67
|
+
...process.env,
|
|
68
|
+
...opts.env
|
|
69
|
+
} : void 0 })
|
|
70
|
+
});
|
|
71
|
+
if (opts.announceUnrun !== false) warnUnrunSpecs(cwd, files);
|
|
72
|
+
return files;
|
|
73
|
+
}
|
|
74
|
+
/** The spec files one {@link runVitest} call points vitest at. */
|
|
75
|
+
function resolveRunFiles(cwd, opts) {
|
|
76
|
+
if (opts.all) return projectSpecFiles(cwd);
|
|
77
|
+
const candidate = opts.candidates.find((name) => existsSync(path.join(cwd, name)));
|
|
78
|
+
return candidate ? [candidate] : [];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Classify a {@link runVitest} failure. execaSync throws an ENOENT-coded
|
|
82
|
+
* error when the binary itself couldn't be spawned (infrastructure problem)
|
|
83
|
+
* and an exit-code error when vitest ran and the tests failed.
|
|
84
|
+
*/
|
|
85
|
+
function classifyVitestError(err, label = "Tests") {
|
|
86
|
+
if (errorCode(err) === "ENOENT") return {
|
|
87
|
+
code: "spawn_failed",
|
|
88
|
+
message: `Could not launch the test runner: ${errorMessage$1(err)} — is the binary on your PATH?`
|
|
89
|
+
};
|
|
90
|
+
return {
|
|
91
|
+
code: "test_failed",
|
|
92
|
+
message: `${label} failed: ${errorMessage$1(err)}`
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** Directories a project's own specs never live in. */
|
|
96
|
+
const UNSCANNED_DIRS = /* @__PURE__ */ new Set([
|
|
97
|
+
"node_modules",
|
|
98
|
+
".aai",
|
|
99
|
+
".git",
|
|
100
|
+
"dist",
|
|
101
|
+
".workflow-data"
|
|
102
|
+
]);
|
|
103
|
+
/** What counts as a spec file. */
|
|
104
|
+
const SPEC_FILE_RE = /\.test\.(ts|js|tsx|mts|cts)$/;
|
|
105
|
+
/**
|
|
106
|
+
* Every spec file in the project that belongs to the TEST tier.
|
|
107
|
+
*
|
|
108
|
+
* Eval files are excluded by the `.eval.` INFIX — the tier convention — rather
|
|
109
|
+
* than by a filename list, which is what keeps this module from importing
|
|
110
|
+
* `eval.ts`, which imports this one.
|
|
111
|
+
*/
|
|
112
|
+
function projectSpecFiles(cwd) {
|
|
113
|
+
const found = [];
|
|
114
|
+
collectSpecs(cwd, "", found);
|
|
115
|
+
return found.sort(compareCodeUnits);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Spec files in the project that a run over `ran` did NOT cover.
|
|
119
|
+
*
|
|
120
|
+
* `runVitest` passes the candidate filename as a vitest FILTER, which is what
|
|
121
|
+
* keeps `test` and `eval` disjoint without either excluding the other's file —
|
|
122
|
+
* see its doc. The cost is that every other `*.test.ts` in the project is
|
|
123
|
+
* skipped, and the skip was SILENT: the shipped `retail` template carries seven
|
|
124
|
+
* of them, so `aai test` there ran 1 file / 67 tests, printed "Tests passed",
|
|
125
|
+
* and left 211 of the project's 278 tests unrun with nothing saying so.
|
|
126
|
+
*
|
|
127
|
+
* The narrow default STANDS — which files `aai test` runs is a documented
|
|
128
|
+
* contract (the scaffold guide says "Run agent.test.ts via vitest"), and running
|
|
129
|
+
* a project's other specs by default could reach ones that are slow or want
|
|
130
|
+
* credentials. What does not stand is a GREEN VERDICT over the difference: this
|
|
131
|
+
* set is what {@link executeTest} refuses to call a pass, what it puts in its
|
|
132
|
+
* result for a script to read, and what `--all` opts into running.
|
|
133
|
+
*/
|
|
134
|
+
function unrunSpecFiles(cwd, ran) {
|
|
135
|
+
const covered = new Set(coveredList(ran));
|
|
136
|
+
return projectSpecFiles(cwd).filter((rel) => !covered.has(rel));
|
|
137
|
+
}
|
|
138
|
+
/** {@link RanSpecs} as a list, so nothing downstream re-derives the three cases. */
|
|
139
|
+
function coveredList(ran) {
|
|
140
|
+
if (ran === false) return [];
|
|
141
|
+
return typeof ran === "string" ? [ran] : ran;
|
|
142
|
+
}
|
|
143
|
+
/** Code-unit comparison — see {@link projectSpecFiles} for why not `localeCompare`. */
|
|
144
|
+
function compareCodeUnits(a, b) {
|
|
145
|
+
if (a < b) return -1;
|
|
146
|
+
return a > b ? 1 : 0;
|
|
147
|
+
}
|
|
148
|
+
/** One directory of {@link projectSpecFiles}, recursing into the ones that count. */
|
|
149
|
+
function collectSpecs(dir, prefix, out) {
|
|
150
|
+
let entries;
|
|
151
|
+
try {
|
|
152
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
153
|
+
} catch {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
for (const e of entries) {
|
|
157
|
+
const rel = prefix ? `${prefix}/${e.name}` : e.name;
|
|
158
|
+
if (e.isDirectory()) {
|
|
159
|
+
if (!(UNSCANNED_DIRS.has(e.name) || e.name.startsWith("."))) collectSpecs(path.join(dir, e.name), rel, out);
|
|
160
|
+
} else if (SPEC_FILE_RE.test(e.name) && !e.name.includes(".eval.test.")) out.push(rel);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/** How many unrun spec names a message prints before it starts counting. */
|
|
164
|
+
const MAX_NAMED_SPECS = 10;
|
|
165
|
+
/** The unrun set as one phrase — capped, because a project may hold hundreds. */
|
|
166
|
+
function formatSpecList(files) {
|
|
167
|
+
const named = files.slice(0, MAX_NAMED_SPECS).join(", ");
|
|
168
|
+
const rest = files.length - MAX_NAMED_SPECS;
|
|
169
|
+
return rest > 0 ? `${named}, and ${rest} more` : named;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* The remedy, named the same way wherever the narrowing is reported.
|
|
173
|
+
*
|
|
174
|
+
* The project's own `npm test` comes FIRST because it is the command a
|
|
175
|
+
* scaffolded project already ships (`scaffold/package.json`), so it is the one
|
|
176
|
+
* answer that needs nothing installed or remembered; `--all` is the same thing
|
|
177
|
+
* without leaving the CLI.
|
|
178
|
+
*/
|
|
179
|
+
const WIDEN_HINT = "Run the whole suite with this project's `npm test` (`vitest run --exclude \"**/*.eval.test.*\"`) or `aai test --all`; behaviour evals have their own command (`aai eval`).";
|
|
180
|
+
/**
|
|
181
|
+
* Warn, once, naming the spec files this run did not cover.
|
|
182
|
+
*
|
|
183
|
+
* This is the notice for a caller whose own result says nothing about the
|
|
184
|
+
* narrowing — today that is `aai build`'s pre-build gate, which ran one file
|
|
185
|
+
* out of eight and printed "Build complete". `aai test` does not use it: an
|
|
186
|
+
* incomplete run is a FAILURE there, and the failure's own message is the
|
|
187
|
+
* report.
|
|
188
|
+
*/
|
|
189
|
+
function warnUnrunSpecs(cwd, ran) {
|
|
190
|
+
const skipped = unrunSpecFiles(cwd, ran);
|
|
191
|
+
if (skipped.length === 0) return;
|
|
192
|
+
const ranList = coveredList(ran);
|
|
193
|
+
const preamble = ranList.length === 0 ? `No agent.test.ts, so vitest ran nothing. ${skipped.length} spec file(s) exist and were NOT run:` : `vitest ran ${ranList.join(", ")} only. ${skipped.length} other spec file(s) were NOT run:`;
|
|
194
|
+
notify("warn", `${preamble} ${formatSpecList(skipped)}. ${WIDEN_HINT}`);
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Execute agent tests and return structured result.
|
|
198
|
+
*
|
|
199
|
+
* **An incomplete run is not a pass.** For as long as this command answered
|
|
200
|
+
* `{"ok":true,"data":{"passed":true}}` with exit 0 over specs it had not run,
|
|
201
|
+
* the scaffold's `"test": "aai test"` was what users wired into CI — so a suite
|
|
202
|
+
* of 25 tests could go red in the editor and green in the pipeline, and adding
|
|
203
|
+
* one tool could break `registry.test.ts` in 17 assertions with `pnpm test` and
|
|
204
|
+
* `pnpm build` both staying green throughout. It is the same defect
|
|
205
|
+
* `defineExec`'s `cwd` policy exists for (a green result for a project that is
|
|
206
|
+
* not there), one directory over, and it gets the same answer: the command
|
|
207
|
+
* fails, names the files, and names the flag that runs them.
|
|
208
|
+
*/
|
|
209
|
+
async function executeTest(cwd, opts = {}) {
|
|
210
|
+
log.step(opts.all ? "Running project tests" : "Running agent tests");
|
|
211
|
+
try {
|
|
212
|
+
const ran = runVitest(cwd, {
|
|
213
|
+
candidates: TEST_FILES,
|
|
214
|
+
announceUnrun: false,
|
|
215
|
+
...omitUndefined({ all: opts.all })
|
|
216
|
+
});
|
|
217
|
+
const unrun = unrunSpecFiles(cwd, ran);
|
|
218
|
+
if (unrun.length > 0) return incomplete(ran, unrun);
|
|
219
|
+
if (ran === false) {
|
|
220
|
+
log.info("No test file found. Create agent.test.ts to add tests.");
|
|
221
|
+
return ok({
|
|
222
|
+
passed: true,
|
|
223
|
+
skipped: true,
|
|
224
|
+
ran: [],
|
|
225
|
+
unrun: [],
|
|
226
|
+
complete: true
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
log.success(`Tests passed (${ran.length} spec file(s))`);
|
|
230
|
+
return ok({
|
|
231
|
+
passed: true,
|
|
232
|
+
ran,
|
|
233
|
+
unrun: [],
|
|
234
|
+
complete: true
|
|
235
|
+
});
|
|
236
|
+
} catch (err) {
|
|
237
|
+
const { code, message } = classifyVitestError(err);
|
|
238
|
+
return fail(code, message);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* The verdict for a run that left specs uncovered.
|
|
243
|
+
*
|
|
244
|
+
* Both arms fail, and the `ran === false` arm is the one that had misled
|
|
245
|
+
* longest: `aai test` printed "No test file found" while the project's specs sat
|
|
246
|
+
* right there unrun, which reads as "this project has no tests". Measured on a
|
|
247
|
+
* project whose only spec was `tools/echo_back.test.ts` — `{"passed":true,
|
|
248
|
+
* "skipped":true}`, exit 0, and not a word about it.
|
|
249
|
+
*/
|
|
250
|
+
function incomplete(ran, unrun) {
|
|
251
|
+
const preamble = ran === false ? `\`aai test\` found no agent.test.ts, so it ran nothing, but ${unrun.length} spec file(s) exist` : `\`aai test\` ran ${ran.join(", ")} only — ${unrun.length} other spec file(s) in this project were not run`;
|
|
252
|
+
return fail("incomplete_run", `${preamble}: ${formatSpecList(unrun)}. An unrun spec is not a passing one, so this is not a green result.`, WIDEN_HINT);
|
|
253
|
+
}
|
|
254
|
+
//#endregion
|
|
255
|
+
export { TEST_FILES, classifyVitestError, executeTest, runVitest };
|
package/dist/test.d.ts
CHANGED
|
@@ -2,9 +2,23 @@
|
|
|
2
2
|
* `aai test` — run agent tests via vitest.
|
|
3
3
|
*/
|
|
4
4
|
import { type CommandResult } from "./_output.ts";
|
|
5
|
+
/**
|
|
6
|
+
* What `aai test` measured, not merely whether it exited 0.
|
|
7
|
+
*
|
|
8
|
+
* `passed` alone is what made a narrowed run indistinguishable from a complete
|
|
9
|
+
* one in a script (`jq -e .data.passed` was true either way), so the set it
|
|
10
|
+
* covered rides the result: `ran` is what vitest was pointed at, `unrun` is what
|
|
11
|
+
* it was not, and `complete` is the one field a CI job needs to read.
|
|
12
|
+
*/
|
|
5
13
|
type TestData = {
|
|
6
14
|
passed: boolean;
|
|
7
15
|
skipped?: boolean;
|
|
16
|
+
/** Spec files this run covered, project-relative, code-unit sorted. */
|
|
17
|
+
ran: string[];
|
|
18
|
+
/** Spec files in the project this run did NOT cover — empty when complete. */
|
|
19
|
+
unrun: string[];
|
|
20
|
+
/** Whether the run covered every non-eval spec file in the project. */
|
|
21
|
+
complete: boolean;
|
|
8
22
|
};
|
|
9
23
|
/**
|
|
10
24
|
* Resolve the agent project's own vitest binary so tests run without the
|
|
@@ -24,7 +38,7 @@ export declare const TEST_FILES: readonly ["agent.test.ts", "agent.test.js"];
|
|
|
24
38
|
export type VitestRunOptions = {
|
|
25
39
|
/** File names to look for in the project root, in preference order. */
|
|
26
40
|
readonly candidates: readonly string[];
|
|
27
|
-
/** Extra vitest CLI arguments, inserted before the file
|
|
41
|
+
/** Extra vitest CLI arguments, inserted before the file names. */
|
|
28
42
|
readonly extraArgs?: readonly string[];
|
|
29
43
|
/**
|
|
30
44
|
* Variables to add to the child's environment. Absent leaves the child with
|
|
@@ -33,20 +47,45 @@ export type VitestRunOptions = {
|
|
|
33
47
|
* way a session under `aai dev` does.
|
|
34
48
|
*/
|
|
35
49
|
readonly env?: Record<string, string>;
|
|
50
|
+
/**
|
|
51
|
+
* Run EVERY non-eval spec in the project rather than the first candidate.
|
|
52
|
+
*
|
|
53
|
+
* The opt-in half of the narrowing documented on {@link unrunSpecFiles}: the
|
|
54
|
+
* default stays one file, and this is how a caller says "I want the whole
|
|
55
|
+
* suite" without the command having to guess. Still a filter list rather than
|
|
56
|
+
* an include glob, so the eval tier stays disjoint by construction.
|
|
57
|
+
*/
|
|
58
|
+
readonly all?: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Whether THIS function announces the specs it did not run.
|
|
61
|
+
*
|
|
62
|
+
* Default TRUE, and the default is the point: the caller most in need of the
|
|
63
|
+
* notice is the one that does not know it is narrowing. `aai build` runs this
|
|
64
|
+
* as its pre-build gate and reports nothing of its own, so a build gated on
|
|
65
|
+
* one file out of eight said so nowhere. `aai test` and `aai eval` pass
|
|
66
|
+
* `false` — the first because it reports the same set itself, in its result
|
|
67
|
+
* as well as its output, and the second because "did not run" is a claim
|
|
68
|
+
* about the TEST tier and every unit spec in the project would be named
|
|
69
|
+
* falsely by an eval run.
|
|
70
|
+
*/
|
|
71
|
+
readonly announceUnrun?: boolean;
|
|
36
72
|
};
|
|
37
73
|
/**
|
|
38
|
-
* Run vitest over
|
|
74
|
+
* Run vitest over `candidates` (or, with `all`, the whole project) in the given
|
|
75
|
+
* project directory.
|
|
39
76
|
*
|
|
40
|
-
* Returns the
|
|
41
|
-
*
|
|
77
|
+
* Returns the FILES it ran, or `false` if there was nothing to run. Throws on
|
|
78
|
+
* failure.
|
|
42
79
|
*
|
|
43
|
-
* A vitest FILTER, not an include glob:
|
|
80
|
+
* A vitest FILTER, not an include glob: each argument is matched as a substring
|
|
44
81
|
* against the paths vitest's own include globs already found, which is why the
|
|
45
82
|
* candidates are named `*.test.ts` — `agent.test.ts` cannot match
|
|
46
83
|
* `agent.eval.test.ts` and vice versa, so the two commands stay disjoint
|
|
47
|
-
* without either one having to exclude the other's file.
|
|
84
|
+
* without either one having to exclude the other's file. That holds for the
|
|
85
|
+
* `all` list too: it is built from {@link projectSpecFiles}, which drops the
|
|
86
|
+
* eval tier by infix.
|
|
48
87
|
*/
|
|
49
|
-
export declare function runVitest(cwd: string, opts?: VitestRunOptions): string | false;
|
|
88
|
+
export declare function runVitest(cwd: string, opts?: VitestRunOptions): string[] | false;
|
|
50
89
|
/**
|
|
51
90
|
* Classify a {@link runVitest} failure. execaSync throws an ENOENT-coded
|
|
52
91
|
* error when the binary itself couldn't be spawned (infrastructure problem)
|
|
@@ -59,36 +98,63 @@ label?: string): {
|
|
|
59
98
|
message: string;
|
|
60
99
|
};
|
|
61
100
|
/**
|
|
62
|
-
*
|
|
101
|
+
* Every spec file in the project that belongs to the TEST tier.
|
|
63
102
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
103
|
+
* Eval files are excluded by the `.eval.` INFIX — the tier convention — rather
|
|
104
|
+
* than by a filename list, which is what keeps this module from importing
|
|
105
|
+
* `eval.ts`, which imports this one.
|
|
106
|
+
*/
|
|
107
|
+
export declare function projectSpecFiles(cwd: string): string[];
|
|
108
|
+
/**
|
|
109
|
+
* Spec files in the project that a run over `ran` did NOT cover.
|
|
70
110
|
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
111
|
+
* `runVitest` passes the candidate filename as a vitest FILTER, which is what
|
|
112
|
+
* keeps `test` and `eval` disjoint without either excluding the other's file —
|
|
113
|
+
* see its doc. The cost is that every other `*.test.ts` in the project is
|
|
114
|
+
* skipped, and the skip was SILENT: the shipped `retail` template carries seven
|
|
115
|
+
* of them, so `aai test` there ran 1 file / 67 tests, printed "Tests passed",
|
|
116
|
+
* and left 211 of the project's 278 tests unrun with nothing saying so.
|
|
76
117
|
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
118
|
+
* The narrow default STANDS — which files `aai test` runs is a documented
|
|
119
|
+
* contract (the scaffold guide says "Run agent.test.ts via vitest"), and running
|
|
120
|
+
* a project's other specs by default could reach ones that are slow or want
|
|
121
|
+
* credentials. What does not stand is a GREEN VERDICT over the difference: this
|
|
122
|
+
* set is what {@link executeTest} refuses to call a pass, what it puts in its
|
|
123
|
+
* result for a script to read, and what `--all` opts into running.
|
|
124
|
+
*/
|
|
125
|
+
export declare function unrunSpecFiles(cwd: string, ran: RanSpecs): string[];
|
|
126
|
+
/**
|
|
127
|
+
* What a caller reports as covered: one filename, a list of them, or `false`
|
|
128
|
+
* for a run that found nothing to point vitest at.
|
|
79
129
|
*/
|
|
80
|
-
export
|
|
130
|
+
export type RanSpecs = string | readonly string[] | false;
|
|
81
131
|
/**
|
|
82
132
|
* Warn, once, naming the spec files this run did not cover.
|
|
83
133
|
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
|
|
134
|
+
* This is the notice for a caller whose own result says nothing about the
|
|
135
|
+
* narrowing — today that is `aai build`'s pre-build gate, which ran one file
|
|
136
|
+
* out of eight and printed "Build complete". `aai test` does not use it: an
|
|
137
|
+
* incomplete run is a FAILURE there, and the failure's own message is the
|
|
138
|
+
* report.
|
|
139
|
+
*/
|
|
140
|
+
export declare function warnUnrunSpecs(cwd: string, ran: RanSpecs): void;
|
|
141
|
+
/** What `aai test` was asked to cover. */
|
|
142
|
+
export type TestOptions = {
|
|
143
|
+
/** Run every non-eval spec in the project rather than `agent.test.ts` alone. */
|
|
144
|
+
readonly all?: boolean | undefined;
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Execute agent tests and return structured result.
|
|
148
|
+
*
|
|
149
|
+
* **An incomplete run is not a pass.** For as long as this command answered
|
|
150
|
+
* `{"ok":true,"data":{"passed":true}}` with exit 0 over specs it had not run,
|
|
151
|
+
* the scaffold's `"test": "aai test"` was what users wired into CI — so a suite
|
|
152
|
+
* of 25 tests could go red in the editor and green in the pipeline, and adding
|
|
153
|
+
* one tool could break `registry.test.ts` in 17 assertions with `pnpm test` and
|
|
154
|
+
* `pnpm build` both staying green throughout. It is the same defect
|
|
155
|
+
* `defineExec`'s `cwd` policy exists for (a green result for a project that is
|
|
156
|
+
* not there), one directory over, and it gets the same answer: the command
|
|
157
|
+
* fails, names the files, and names the flag that runs them.
|
|
90
158
|
*/
|
|
91
|
-
export declare function
|
|
92
|
-
/** Execute agent tests and return structured result. */
|
|
93
|
-
export declare function executeTest(cwd: string): Promise<CommandResult<TestData>>;
|
|
159
|
+
export declare function executeTest(cwd: string, opts?: TestOptions): Promise<CommandResult<TestData>>;
|
|
94
160
|
export {};
|
package/dist/typecheck.mjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { n as binFromManifest, o as errorMessage, u as readPackageJson } from "./_utils-
|
|
2
|
+
import { n as binFromManifest, o as errorMessage, u as readPackageJson } from "./_utils-Dhd058D9.mjs";
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
|
-
//#region typecheck.ts
|
|
6
|
+
//#region src/typecheck.ts
|
|
7
7
|
/**
|
|
8
8
|
* Project typechecking — `tsc --noEmit` over the project's own tsconfig.
|
|
9
9
|
*
|
package/dist/worker-bundler.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { i as errorCode } from "./_utils-
|
|
3
|
-
import { n as withPreservedNodeEnv } from "./_vite-env-
|
|
2
|
+
import { i as errorCode } from "./_utils-Dhd058D9.mjs";
|
|
3
|
+
import { n as withPreservedNodeEnv } from "./_vite-env-kejF0p2L.mjs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import fs from "node:fs/promises";
|
|
6
6
|
import { invariant } from "@alexkroman1/aai/internal";
|
|
7
7
|
import { build } from "vite";
|
|
8
|
-
//#region worker-bundler.ts
|
|
8
|
+
//#region src/worker-bundler.ts
|
|
9
9
|
/**
|
|
10
10
|
* Generated wrapper entry, written under `.aai/` for the duration of the
|
|
11
11
|
* build (the CLI's own scratch dir — dot-paths are ignored by the dev
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { a as ok, n as fail, t as CliError } from "./_output-
|
|
3
|
-
import { n as log } from "./_ui-
|
|
4
|
-
import { i as readProjectConfig } from "./_config-
|
|
5
|
-
import { n as getServerInfo } from "./_agent-
|
|
2
|
+
import { a as ok, n as fail, t as CliError } from "./_output-DbJt9gsl.mjs";
|
|
3
|
+
import { n as log } from "./_ui-DCt4qZrk.mjs";
|
|
4
|
+
import { i as readProjectConfig } from "./_config-ZY2euhdh.mjs";
|
|
5
|
+
import { n as getServerInfo } from "./_agent-Clbq3a4h.mjs";
|
|
6
6
|
import { errorMessage, omitUndefined } from "@alexkroman1/aai/utils";
|
|
7
7
|
import { createWorkflowApiClient } from "@alexkroman1/aai/workflow-api";
|
|
8
|
-
//#region workflow.ts
|
|
8
|
+
//#region src/workflow.ts
|
|
9
9
|
/**
|
|
10
10
|
* `aai workflow` — reading and steering a deployed agent's durable runs from a
|
|
11
11
|
* terminal.
|