@alexkroman1/aai-cli 13.1.0 → 13.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +1 -1
- package/dist/{_agent-BTwZJu4P.mjs → _agent-Clbq3a4h.mjs} +2 -2
- package/dist/{_bundler-C31rlcYz.mjs → _bundler-DEkjO1d8.mjs} +3 -3
- package/dist/{_config-CmJOFsAP.mjs → _config-ZY2euhdh.mjs} +7 -4
- package/dist/_config.d.ts +25 -1
- package/dist/{_dev-server-DWvkNoIS.mjs → _dev-server-C13xuGB1.mjs} +13 -13
- package/dist/_e2e-test-utils.d.ts +15 -7
- package/dist/{_init-BURbdHNM.mjs → _init-Ci4QGjp8.mjs} +58 -10
- package/dist/{_output-CKkmWs7i.mjs → _output-DbJt9gsl.mjs} +34 -3
- package/dist/_output.d.ts +17 -1
- package/dist/{_preflight-Co1G9Jww.mjs → _preflight-B2laWPdU.mjs} +1 -1
- package/dist/{_server-common-6e0QI4mq.mjs → _server-common-BV4_CrP3.mjs} +3 -3
- package/dist/{_slug-api-CfGQq1Tt.mjs → _slug-api-DlKpj91X.mjs} +3 -3
- package/dist/{_studio-B1waXMuR.mjs → _studio-DTi3EdXQ.mjs} +3 -3
- package/dist/{_templates-0Zr2z6yA.mjs → _templates-V4l5sZJy.mjs} +2 -2
- package/dist/{_typecheck-gate-BXvUNnfs.mjs → _typecheck-gate-sV4N5F93.mjs} +3 -3
- package/dist/{_ui-u7T4YooX.mjs → _ui-DCt4qZrk.mjs} +1 -1
- package/dist/{_utils-B8QmtFhK.mjs → _utils-Dhd058D9.mjs} +1 -1
- package/dist/{_vite-env-D9Qj6KPX.mjs → _vite-env-kejF0p2L.mjs} +1 -1
- package/dist/{_workflow-determinism-f9J-5Zlj.mjs → _workflow-determinism-C9l8uC2R.mjs} +1 -1
- package/dist/{build-BLlLOrng.mjs → build-CWyFGe-W.mjs} +11 -8
- package/dist/cli.d.ts +2 -1
- package/dist/cli.mjs +120 -46
- package/dist/{client-bundler-j5c46x0G.mjs → client-bundler-CT-_26c5.mjs} +4 -4
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-X_CRgCf6.mjs → delete-RqlQZ6Y_.mjs} +6 -6
- package/dist/{deploy-BzZ-r4iM.mjs → deploy-B7f3CK7v.mjs} +13 -13
- package/dist/{dev-BDykFvcc.mjs → dev-DamXieW_.mjs} +5 -5
- package/dist/{eval-0qEOdm0l.mjs → eval-h3DTVfRw.mjs} +27 -11
- package/dist/eval.d.ts +2 -0
- package/dist/{init-BtJI306C.mjs → init-D-vFrV8h.mjs} +7 -7
- package/dist/{login-DeUETobb.mjs → login-CDiemNmC.mjs} +5 -5
- package/dist/{logs-hWnJ66Bl.mjs → logs-DGwPNCzB.mjs} +4 -4
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/.env.example +17 -1
- package/dist/scaffold/package.json +6 -5
- package/dist/secret-V1w7y9-j.mjs +162 -0
- package/dist/secret.d.ts +41 -8
- package/dist/{studio-BxAS_FQQ.mjs → studio-BpMCfUNx.mjs} +9 -9
- package/dist/templates/briefing-desk/agent.test.ts +7 -3
- package/dist/templates/call-audit/agent.eval.test.ts +14 -11
- package/dist/templates/call-audit/agent.test.ts +34 -7
- package/dist/templates/call-audit/client.tsx +14 -17
- package/dist/templates/call-audit/workflows/ingest.ts +10 -1
- package/dist/templates/code-interpreter/agent.eval.test.ts +27 -17
- package/dist/templates/dispatch-center/agent.eval.test.ts +18 -24
- package/dist/templates/embedded-assets/agent.eval.test.ts +3 -3
- package/dist/templates/embedded-assets/agent.test.ts +7 -1
- package/dist/templates/health-assistant/agent.eval.test.ts +38 -15
- package/dist/templates/health-assistant/agent.test.ts +8 -4
- package/dist/templates/link-digest/agent.eval.test.ts +24 -15
- package/dist/templates/link-digest/agent.test.ts +7 -1
- package/dist/templates/link-digest/client.tsx +17 -21
- package/dist/templates/math-buddy/agent.eval.test.ts +28 -17
- package/dist/templates/night-owl/agent.eval.test.ts +30 -15
- package/dist/templates/night-owl/agent.test.ts +6 -1
- package/dist/templates/personal-finance/agent.eval.test.ts +27 -17
- package/dist/templates/pipeline-simple/agent.test.ts +31 -14
- package/dist/templates/pizza-ordering/agent.eval.test.ts +11 -6
- package/dist/templates/plan-and-execute/agent.eval.test.ts +14 -7
- package/dist/templates/podcast-digest/agent.test.ts +13 -3
- package/dist/templates/podcast-digest/client.tsx +12 -15
- package/dist/templates/recap-workflow/agent.eval.test.ts +49 -20
- package/dist/templates/recap-workflow/agent.test.ts +22 -9
- package/dist/templates/redline/agent.eval.test.ts +32 -24
- package/dist/templates/redline/agent.test.ts +7 -1
- package/dist/templates/redline/client.tsx +12 -15
- package/dist/templates/research-workflow/agent.eval.test.ts +32 -22
- package/dist/templates/research-workflow/agent.test.ts +7 -1
- package/dist/templates/retail/agent.eval.test.ts +18 -34
- package/dist/templates/retail/registry.test.ts +99 -58
- package/dist/templates/simple/agent.test.ts +39 -6
- package/dist/templates/spoken-summary/agent.eval.test.ts +25 -16
- package/dist/templates/spoken-summary/agent.test.ts +21 -7
- package/dist/templates/spoken-summary/client.tsx +14 -18
- package/dist/templates/support-line/agent.eval.test.ts +23 -26
- package/dist/templates/transcription-workflow/agent.test.ts +26 -8
- package/dist/templates/transcription-workflow/client.tsx +19 -26
- package/dist/templates/transcription-workflow/workflows/normalize.ts +10 -1
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +5 -2
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +11 -4
- package/dist/templates/travel-concierge/agent.eval.test.ts +37 -56
- package/dist/test-guenJUxi.mjs +255 -0
- package/dist/test.d.ts +97 -31
- package/dist/typecheck.mjs +2 -2
- package/dist/worker-bundler.mjs +3 -3
- package/dist/{workflow-BBu5oEDw.mjs → workflow-BZyKLkhV.mjs} +5 -5
- package/package.json +10 -10
- package/dist/bin.d.mts +0 -2
- package/dist/secret-B-3T5tvr.mjs +0 -61
- package/dist/test-CiLab-AA.mjs +0 -182
- package/dist/tsdown.config.d.ts +0 -2
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { a as ok, n as fail } from "./_output-DbJt9gsl.mjs";
|
|
3
|
+
import { n as log, r as notify } from "./_ui-DCt4qZrk.mjs";
|
|
4
|
+
import { i as errorCode, o as errorMessage$1, r as binFromPackageJson } from "./_utils-Dhd058D9.mjs";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { omitUndefined } from "@alexkroman1/aai/utils";
|
|
9
|
+
import { execaSync } from "execa";
|
|
10
|
+
//#region src/test.ts
|
|
11
|
+
/**
|
|
12
|
+
* `aai test` — run agent tests via vitest.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Resolve the agent project's own vitest binary so tests run without the
|
|
16
|
+
* npx resolution overhead (and its potential network fetch of vitest).
|
|
17
|
+
*
|
|
18
|
+
* Resolves `vitest/package.json` from the agent directory, derives the bin
|
|
19
|
+
* script, and runs it with the current Node executable. Falls back to
|
|
20
|
+
* `npx vitest` only when no local install is resolvable.
|
|
21
|
+
*/
|
|
22
|
+
function resolveVitestCommand(cwd, resolve = createRequire(path.join(cwd, "package.json")).resolve) {
|
|
23
|
+
try {
|
|
24
|
+
const bin = binFromPackageJson(resolve("vitest/package.json"), "vitest");
|
|
25
|
+
if (bin) return {
|
|
26
|
+
cmd: process.execPath,
|
|
27
|
+
args: [bin]
|
|
28
|
+
};
|
|
29
|
+
} catch {}
|
|
30
|
+
return {
|
|
31
|
+
cmd: "npx",
|
|
32
|
+
args: ["vitest"]
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** The files `aai test` runs, in preference order. */
|
|
36
|
+
const TEST_FILES = ["agent.test.ts", "agent.test.js"];
|
|
37
|
+
/**
|
|
38
|
+
* Run vitest over `candidates` (or, with `all`, the whole project) in the given
|
|
39
|
+
* project directory.
|
|
40
|
+
*
|
|
41
|
+
* Returns the FILES it ran, or `false` if there was nothing to run. Throws on
|
|
42
|
+
* failure.
|
|
43
|
+
*
|
|
44
|
+
* A vitest FILTER, not an include glob: each argument is matched as a substring
|
|
45
|
+
* against the paths vitest's own include globs already found, which is why the
|
|
46
|
+
* candidates are named `*.test.ts` — `agent.test.ts` cannot match
|
|
47
|
+
* `agent.eval.test.ts` and vice versa, so the two commands stay disjoint
|
|
48
|
+
* without either one having to exclude the other's file. That holds for the
|
|
49
|
+
* `all` list too: it is built from {@link projectSpecFiles}, which drops the
|
|
50
|
+
* eval tier by infix.
|
|
51
|
+
*/
|
|
52
|
+
function runVitest(cwd, opts = { candidates: TEST_FILES }) {
|
|
53
|
+
const files = resolveRunFiles(cwd, opts);
|
|
54
|
+
if (files.length === 0) return false;
|
|
55
|
+
const { cmd, args } = resolveVitestCommand(cwd);
|
|
56
|
+
execaSync(cmd, [
|
|
57
|
+
...args,
|
|
58
|
+
"run",
|
|
59
|
+
"--root",
|
|
60
|
+
".",
|
|
61
|
+
...opts.extraArgs ?? [],
|
|
62
|
+
...files
|
|
63
|
+
], {
|
|
64
|
+
cwd,
|
|
65
|
+
stdio: "inherit",
|
|
66
|
+
...omitUndefined({ env: opts.env ? {
|
|
67
|
+
...process.env,
|
|
68
|
+
...opts.env
|
|
69
|
+
} : void 0 })
|
|
70
|
+
});
|
|
71
|
+
if (opts.announceUnrun !== false) warnUnrunSpecs(cwd, files);
|
|
72
|
+
return files;
|
|
73
|
+
}
|
|
74
|
+
/** The spec files one {@link runVitest} call points vitest at. */
|
|
75
|
+
function resolveRunFiles(cwd, opts) {
|
|
76
|
+
if (opts.all) return projectSpecFiles(cwd);
|
|
77
|
+
const candidate = opts.candidates.find((name) => existsSync(path.join(cwd, name)));
|
|
78
|
+
return candidate ? [candidate] : [];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Classify a {@link runVitest} failure. execaSync throws an ENOENT-coded
|
|
82
|
+
* error when the binary itself couldn't be spawned (infrastructure problem)
|
|
83
|
+
* and an exit-code error when vitest ran and the tests failed.
|
|
84
|
+
*/
|
|
85
|
+
function classifyVitestError(err, label = "Tests") {
|
|
86
|
+
if (errorCode(err) === "ENOENT") return {
|
|
87
|
+
code: "spawn_failed",
|
|
88
|
+
message: `Could not launch the test runner: ${errorMessage$1(err)} — is the binary on your PATH?`
|
|
89
|
+
};
|
|
90
|
+
return {
|
|
91
|
+
code: "test_failed",
|
|
92
|
+
message: `${label} failed: ${errorMessage$1(err)}`
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** Directories a project's own specs never live in. */
|
|
96
|
+
const UNSCANNED_DIRS = /* @__PURE__ */ new Set([
|
|
97
|
+
"node_modules",
|
|
98
|
+
".aai",
|
|
99
|
+
".git",
|
|
100
|
+
"dist",
|
|
101
|
+
".workflow-data"
|
|
102
|
+
]);
|
|
103
|
+
/** What counts as a spec file. */
|
|
104
|
+
const SPEC_FILE_RE = /\.test\.(ts|js|tsx|mts|cts)$/;
|
|
105
|
+
/**
|
|
106
|
+
* Every spec file in the project that belongs to the TEST tier.
|
|
107
|
+
*
|
|
108
|
+
* Eval files are excluded by the `.eval.` INFIX — the tier convention — rather
|
|
109
|
+
* than by a filename list, which is what keeps this module from importing
|
|
110
|
+
* `eval.ts`, which imports this one.
|
|
111
|
+
*/
|
|
112
|
+
function projectSpecFiles(cwd) {
|
|
113
|
+
const found = [];
|
|
114
|
+
collectSpecs(cwd, "", found);
|
|
115
|
+
return found.sort(compareCodeUnits);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Spec files in the project that a run over `ran` did NOT cover.
|
|
119
|
+
*
|
|
120
|
+
* `runVitest` passes the candidate filename as a vitest FILTER, which is what
|
|
121
|
+
* keeps `test` and `eval` disjoint without either excluding the other's file —
|
|
122
|
+
* see its doc. The cost is that every other `*.test.ts` in the project is
|
|
123
|
+
* skipped, and the skip was SILENT: the shipped `retail` template carries seven
|
|
124
|
+
* of them, so `aai test` there ran 1 file / 67 tests, printed "Tests passed",
|
|
125
|
+
* and left 211 of the project's 278 tests unrun with nothing saying so.
|
|
126
|
+
*
|
|
127
|
+
* The narrow default STANDS — which files `aai test` runs is a documented
|
|
128
|
+
* contract (the scaffold guide says "Run agent.test.ts via vitest"), and running
|
|
129
|
+
* a project's other specs by default could reach ones that are slow or want
|
|
130
|
+
* credentials. What does not stand is a GREEN VERDICT over the difference: this
|
|
131
|
+
* set is what {@link executeTest} refuses to call a pass, what it puts in its
|
|
132
|
+
* result for a script to read, and what `--all` opts into running.
|
|
133
|
+
*/
|
|
134
|
+
function unrunSpecFiles(cwd, ran) {
|
|
135
|
+
const covered = new Set(coveredList(ran));
|
|
136
|
+
return projectSpecFiles(cwd).filter((rel) => !covered.has(rel));
|
|
137
|
+
}
|
|
138
|
+
/** {@link RanSpecs} as a list, so nothing downstream re-derives the three cases. */
|
|
139
|
+
function coveredList(ran) {
|
|
140
|
+
if (ran === false) return [];
|
|
141
|
+
return typeof ran === "string" ? [ran] : ran;
|
|
142
|
+
}
|
|
143
|
+
/** Code-unit comparison — see {@link projectSpecFiles} for why not `localeCompare`. */
|
|
144
|
+
function compareCodeUnits(a, b) {
|
|
145
|
+
if (a < b) return -1;
|
|
146
|
+
return a > b ? 1 : 0;
|
|
147
|
+
}
|
|
148
|
+
/** One directory of {@link projectSpecFiles}, recursing into the ones that count. */
|
|
149
|
+
function collectSpecs(dir, prefix, out) {
|
|
150
|
+
let entries;
|
|
151
|
+
try {
|
|
152
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
153
|
+
} catch {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
for (const e of entries) {
|
|
157
|
+
const rel = prefix ? `${prefix}/${e.name}` : e.name;
|
|
158
|
+
if (e.isDirectory()) {
|
|
159
|
+
if (!(UNSCANNED_DIRS.has(e.name) || e.name.startsWith("."))) collectSpecs(path.join(dir, e.name), rel, out);
|
|
160
|
+
} else if (SPEC_FILE_RE.test(e.name) && !e.name.includes(".eval.test.")) out.push(rel);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/** How many unrun spec names a message prints before it starts counting. */
|
|
164
|
+
const MAX_NAMED_SPECS = 10;
|
|
165
|
+
/** The unrun set as one phrase — capped, because a project may hold hundreds. */
|
|
166
|
+
function formatSpecList(files) {
|
|
167
|
+
const named = files.slice(0, MAX_NAMED_SPECS).join(", ");
|
|
168
|
+
const rest = files.length - MAX_NAMED_SPECS;
|
|
169
|
+
return rest > 0 ? `${named}, and ${rest} more` : named;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* The remedy, named the same way wherever the narrowing is reported.
|
|
173
|
+
*
|
|
174
|
+
* The project's own `npm test` comes FIRST because it is the command a
|
|
175
|
+
* scaffolded project already ships (`scaffold/package.json`), so it is the one
|
|
176
|
+
* answer that needs nothing installed or remembered; `--all` is the same thing
|
|
177
|
+
* without leaving the CLI.
|
|
178
|
+
*/
|
|
179
|
+
const WIDEN_HINT = "Run the whole suite with this project's `npm test` (`vitest run --exclude \"**/*.eval.test.*\"`) or `aai test --all`; behaviour evals have their own command (`aai eval`).";
|
|
180
|
+
/**
|
|
181
|
+
* Warn, once, naming the spec files this run did not cover.
|
|
182
|
+
*
|
|
183
|
+
* This is the notice for a caller whose own result says nothing about the
|
|
184
|
+
* narrowing — today that is `aai build`'s pre-build gate, which ran one file
|
|
185
|
+
* out of eight and printed "Build complete". `aai test` does not use it: an
|
|
186
|
+
* incomplete run is a FAILURE there, and the failure's own message is the
|
|
187
|
+
* report.
|
|
188
|
+
*/
|
|
189
|
+
function warnUnrunSpecs(cwd, ran) {
|
|
190
|
+
const skipped = unrunSpecFiles(cwd, ran);
|
|
191
|
+
if (skipped.length === 0) return;
|
|
192
|
+
const ranList = coveredList(ran);
|
|
193
|
+
const preamble = ranList.length === 0 ? `No agent.test.ts, so vitest ran nothing. ${skipped.length} spec file(s) exist and were NOT run:` : `vitest ran ${ranList.join(", ")} only. ${skipped.length} other spec file(s) were NOT run:`;
|
|
194
|
+
notify("warn", `${preamble} ${formatSpecList(skipped)}. ${WIDEN_HINT}`);
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Execute agent tests and return structured result.
|
|
198
|
+
*
|
|
199
|
+
* **An incomplete run is not a pass.** For as long as this command answered
|
|
200
|
+
* `{"ok":true,"data":{"passed":true}}` with exit 0 over specs it had not run,
|
|
201
|
+
* the scaffold's `"test": "aai test"` was what users wired into CI — so a suite
|
|
202
|
+
* of 25 tests could go red in the editor and green in the pipeline, and adding
|
|
203
|
+
* one tool could break `registry.test.ts` in 17 assertions with `pnpm test` and
|
|
204
|
+
* `pnpm build` both staying green throughout. It is the same defect
|
|
205
|
+
* `defineExec`'s `cwd` policy exists for (a green result for a project that is
|
|
206
|
+
* not there), one directory over, and it gets the same answer: the command
|
|
207
|
+
* fails, names the files, and names the flag that runs them.
|
|
208
|
+
*/
|
|
209
|
+
async function executeTest(cwd, opts = {}) {
|
|
210
|
+
log.step(opts.all ? "Running project tests" : "Running agent tests");
|
|
211
|
+
try {
|
|
212
|
+
const ran = runVitest(cwd, {
|
|
213
|
+
candidates: TEST_FILES,
|
|
214
|
+
announceUnrun: false,
|
|
215
|
+
...omitUndefined({ all: opts.all })
|
|
216
|
+
});
|
|
217
|
+
const unrun = unrunSpecFiles(cwd, ran);
|
|
218
|
+
if (unrun.length > 0) return incomplete(ran, unrun);
|
|
219
|
+
if (ran === false) {
|
|
220
|
+
log.info("No test file found. Create agent.test.ts to add tests.");
|
|
221
|
+
return ok({
|
|
222
|
+
passed: true,
|
|
223
|
+
skipped: true,
|
|
224
|
+
ran: [],
|
|
225
|
+
unrun: [],
|
|
226
|
+
complete: true
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
log.success(`Tests passed (${ran.length} spec file(s))`);
|
|
230
|
+
return ok({
|
|
231
|
+
passed: true,
|
|
232
|
+
ran,
|
|
233
|
+
unrun: [],
|
|
234
|
+
complete: true
|
|
235
|
+
});
|
|
236
|
+
} catch (err) {
|
|
237
|
+
const { code, message } = classifyVitestError(err);
|
|
238
|
+
return fail(code, message);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* The verdict for a run that left specs uncovered.
|
|
243
|
+
*
|
|
244
|
+
* Both arms fail, and the `ran === false` arm is the one that had misled
|
|
245
|
+
* longest: `aai test` printed "No test file found" while the project's specs sat
|
|
246
|
+
* right there unrun, which reads as "this project has no tests". Measured on a
|
|
247
|
+
* project whose only spec was `tools/echo_back.test.ts` — `{"passed":true,
|
|
248
|
+
* "skipped":true}`, exit 0, and not a word about it.
|
|
249
|
+
*/
|
|
250
|
+
function incomplete(ran, unrun) {
|
|
251
|
+
const preamble = ran === false ? `\`aai test\` found no agent.test.ts, so it ran nothing, but ${unrun.length} spec file(s) exist` : `\`aai test\` ran ${ran.join(", ")} only — ${unrun.length} other spec file(s) in this project were not run`;
|
|
252
|
+
return fail("incomplete_run", `${preamble}: ${formatSpecList(unrun)}. An unrun spec is not a passing one, so this is not a green result.`, WIDEN_HINT);
|
|
253
|
+
}
|
|
254
|
+
//#endregion
|
|
255
|
+
export { TEST_FILES, classifyVitestError, executeTest, runVitest };
|
package/dist/test.d.ts
CHANGED
|
@@ -2,9 +2,23 @@
|
|
|
2
2
|
* `aai test` — run agent tests via vitest.
|
|
3
3
|
*/
|
|
4
4
|
import { type CommandResult } from "./_output.ts";
|
|
5
|
+
/**
|
|
6
|
+
* What `aai test` measured, not merely whether it exited 0.
|
|
7
|
+
*
|
|
8
|
+
* `passed` alone is what made a narrowed run indistinguishable from a complete
|
|
9
|
+
* one in a script (`jq -e .data.passed` was true either way), so the set it
|
|
10
|
+
* covered rides the result: `ran` is what vitest was pointed at, `unrun` is what
|
|
11
|
+
* it was not, and `complete` is the one field a CI job needs to read.
|
|
12
|
+
*/
|
|
5
13
|
type TestData = {
|
|
6
14
|
passed: boolean;
|
|
7
15
|
skipped?: boolean;
|
|
16
|
+
/** Spec files this run covered, project-relative, code-unit sorted. */
|
|
17
|
+
ran: string[];
|
|
18
|
+
/** Spec files in the project this run did NOT cover — empty when complete. */
|
|
19
|
+
unrun: string[];
|
|
20
|
+
/** Whether the run covered every non-eval spec file in the project. */
|
|
21
|
+
complete: boolean;
|
|
8
22
|
};
|
|
9
23
|
/**
|
|
10
24
|
* Resolve the agent project's own vitest binary so tests run without the
|
|
@@ -24,7 +38,7 @@ export declare const TEST_FILES: readonly ["agent.test.ts", "agent.test.js"];
|
|
|
24
38
|
export type VitestRunOptions = {
|
|
25
39
|
/** File names to look for in the project root, in preference order. */
|
|
26
40
|
readonly candidates: readonly string[];
|
|
27
|
-
/** Extra vitest CLI arguments, inserted before the file
|
|
41
|
+
/** Extra vitest CLI arguments, inserted before the file names. */
|
|
28
42
|
readonly extraArgs?: readonly string[];
|
|
29
43
|
/**
|
|
30
44
|
* Variables to add to the child's environment. Absent leaves the child with
|
|
@@ -33,20 +47,45 @@ export type VitestRunOptions = {
|
|
|
33
47
|
* way a session under `aai dev` does.
|
|
34
48
|
*/
|
|
35
49
|
readonly env?: Record<string, string>;
|
|
50
|
+
/**
|
|
51
|
+
* Run EVERY non-eval spec in the project rather than the first candidate.
|
|
52
|
+
*
|
|
53
|
+
* The opt-in half of the narrowing documented on {@link unrunSpecFiles}: the
|
|
54
|
+
* default stays one file, and this is how a caller says "I want the whole
|
|
55
|
+
* suite" without the command having to guess. Still a filter list rather than
|
|
56
|
+
* an include glob, so the eval tier stays disjoint by construction.
|
|
57
|
+
*/
|
|
58
|
+
readonly all?: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Whether THIS function announces the specs it did not run.
|
|
61
|
+
*
|
|
62
|
+
* Default TRUE, and the default is the point: the caller most in need of the
|
|
63
|
+
* notice is the one that does not know it is narrowing. `aai build` runs this
|
|
64
|
+
* as its pre-build gate and reports nothing of its own, so a build gated on
|
|
65
|
+
* one file out of eight said so nowhere. `aai test` and `aai eval` pass
|
|
66
|
+
* `false` — the first because it reports the same set itself, in its result
|
|
67
|
+
* as well as its output, and the second because "did not run" is a claim
|
|
68
|
+
* about the TEST tier and every unit spec in the project would be named
|
|
69
|
+
* falsely by an eval run.
|
|
70
|
+
*/
|
|
71
|
+
readonly announceUnrun?: boolean;
|
|
36
72
|
};
|
|
37
73
|
/**
|
|
38
|
-
* Run vitest over
|
|
74
|
+
* Run vitest over `candidates` (or, with `all`, the whole project) in the given
|
|
75
|
+
* project directory.
|
|
39
76
|
*
|
|
40
|
-
* Returns the
|
|
41
|
-
*
|
|
77
|
+
* Returns the FILES it ran, or `false` if there was nothing to run. Throws on
|
|
78
|
+
* failure.
|
|
42
79
|
*
|
|
43
|
-
* A vitest FILTER, not an include glob:
|
|
80
|
+
* A vitest FILTER, not an include glob: each argument is matched as a substring
|
|
44
81
|
* against the paths vitest's own include globs already found, which is why the
|
|
45
82
|
* candidates are named `*.test.ts` — `agent.test.ts` cannot match
|
|
46
83
|
* `agent.eval.test.ts` and vice versa, so the two commands stay disjoint
|
|
47
|
-
* without either one having to exclude the other's file.
|
|
84
|
+
* without either one having to exclude the other's file. That holds for the
|
|
85
|
+
* `all` list too: it is built from {@link projectSpecFiles}, which drops the
|
|
86
|
+
* eval tier by infix.
|
|
48
87
|
*/
|
|
49
|
-
export declare function runVitest(cwd: string, opts?: VitestRunOptions): string | false;
|
|
88
|
+
export declare function runVitest(cwd: string, opts?: VitestRunOptions): string[] | false;
|
|
50
89
|
/**
|
|
51
90
|
* Classify a {@link runVitest} failure. execaSync throws an ENOENT-coded
|
|
52
91
|
* error when the binary itself couldn't be spawned (infrastructure problem)
|
|
@@ -59,36 +98,63 @@ label?: string): {
|
|
|
59
98
|
message: string;
|
|
60
99
|
};
|
|
61
100
|
/**
|
|
62
|
-
*
|
|
101
|
+
* Every spec file in the project that belongs to the TEST tier.
|
|
63
102
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
103
|
+
* Eval files are excluded by the `.eval.` INFIX — the tier convention — rather
|
|
104
|
+
* than by a filename list, which is what keeps this module from importing
|
|
105
|
+
* `eval.ts`, which imports this one.
|
|
106
|
+
*/
|
|
107
|
+
export declare function projectSpecFiles(cwd: string): string[];
|
|
108
|
+
/**
|
|
109
|
+
* Spec files in the project that a run over `ran` did NOT cover.
|
|
70
110
|
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
111
|
+
* `runVitest` passes the candidate filename as a vitest FILTER, which is what
|
|
112
|
+
* keeps `test` and `eval` disjoint without either excluding the other's file —
|
|
113
|
+
* see its doc. The cost is that every other `*.test.ts` in the project is
|
|
114
|
+
* skipped, and the skip was SILENT: the shipped `retail` template carries seven
|
|
115
|
+
* of them, so `aai test` there ran 1 file / 67 tests, printed "Tests passed",
|
|
116
|
+
* and left 211 of the project's 278 tests unrun with nothing saying so.
|
|
76
117
|
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
118
|
+
* The narrow default STANDS — which files `aai test` runs is a documented
|
|
119
|
+
* contract (the scaffold guide says "Run agent.test.ts via vitest"), and running
|
|
120
|
+
* a project's other specs by default could reach ones that are slow or want
|
|
121
|
+
* credentials. What does not stand is a GREEN VERDICT over the difference: this
|
|
122
|
+
* set is what {@link executeTest} refuses to call a pass, what it puts in its
|
|
123
|
+
* result for a script to read, and what `--all` opts into running.
|
|
124
|
+
*/
|
|
125
|
+
export declare function unrunSpecFiles(cwd: string, ran: RanSpecs): string[];
|
|
126
|
+
/**
|
|
127
|
+
* What a caller reports as covered: one filename, a list of them, or `false`
|
|
128
|
+
* for a run that found nothing to point vitest at.
|
|
79
129
|
*/
|
|
80
|
-
export
|
|
130
|
+
export type RanSpecs = string | readonly string[] | false;
|
|
81
131
|
/**
|
|
82
132
|
* Warn, once, naming the spec files this run did not cover.
|
|
83
133
|
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
|
|
134
|
+
* This is the notice for a caller whose own result says nothing about the
|
|
135
|
+
* narrowing — today that is `aai build`'s pre-build gate, which ran one file
|
|
136
|
+
* out of eight and printed "Build complete". `aai test` does not use it: an
|
|
137
|
+
* incomplete run is a FAILURE there, and the failure's own message is the
|
|
138
|
+
* report.
|
|
139
|
+
*/
|
|
140
|
+
export declare function warnUnrunSpecs(cwd: string, ran: RanSpecs): void;
|
|
141
|
+
/** What `aai test` was asked to cover. */
|
|
142
|
+
export type TestOptions = {
|
|
143
|
+
/** Run every non-eval spec in the project rather than `agent.test.ts` alone. */
|
|
144
|
+
readonly all?: boolean | undefined;
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Execute agent tests and return structured result.
|
|
148
|
+
*
|
|
149
|
+
* **An incomplete run is not a pass.** For as long as this command answered
|
|
150
|
+
* `{"ok":true,"data":{"passed":true}}` with exit 0 over specs it had not run,
|
|
151
|
+
* the scaffold's `"test": "aai test"` was what users wired into CI — so a suite
|
|
152
|
+
* of 25 tests could go red in the editor and green in the pipeline, and adding
|
|
153
|
+
* one tool could break `registry.test.ts` in 17 assertions with `pnpm test` and
|
|
154
|
+
* `pnpm build` both staying green throughout. It is the same defect
|
|
155
|
+
* `defineExec`'s `cwd` policy exists for (a green result for a project that is
|
|
156
|
+
* not there), one directory over, and it gets the same answer: the command
|
|
157
|
+
* fails, names the files, and names the flag that runs them.
|
|
90
158
|
*/
|
|
91
|
-
export declare function
|
|
92
|
-
/** Execute agent tests and return structured result. */
|
|
93
|
-
export declare function executeTest(cwd: string): Promise<CommandResult<TestData>>;
|
|
159
|
+
export declare function executeTest(cwd: string, opts?: TestOptions): Promise<CommandResult<TestData>>;
|
|
94
160
|
export {};
|
package/dist/typecheck.mjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { n as binFromManifest, o as errorMessage, u as readPackageJson } from "./_utils-
|
|
2
|
+
import { n as binFromManifest, o as errorMessage, u as readPackageJson } from "./_utils-Dhd058D9.mjs";
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
|
-
//#region typecheck.ts
|
|
6
|
+
//#region src/typecheck.ts
|
|
7
7
|
/**
|
|
8
8
|
* Project typechecking — `tsc --noEmit` over the project's own tsconfig.
|
|
9
9
|
*
|
package/dist/worker-bundler.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { i as errorCode } from "./_utils-
|
|
3
|
-
import { n as withPreservedNodeEnv } from "./_vite-env-
|
|
2
|
+
import { i as errorCode } from "./_utils-Dhd058D9.mjs";
|
|
3
|
+
import { n as withPreservedNodeEnv } from "./_vite-env-kejF0p2L.mjs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import fs from "node:fs/promises";
|
|
6
6
|
import { invariant } from "@alexkroman1/aai/internal";
|
|
7
7
|
import { build } from "vite";
|
|
8
|
-
//#region worker-bundler.ts
|
|
8
|
+
//#region src/worker-bundler.ts
|
|
9
9
|
/**
|
|
10
10
|
* Generated wrapper entry, written under `.aai/` for the duration of the
|
|
11
11
|
* build (the CLI's own scratch dir — dot-paths are ignored by the dev
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { a as ok, n as fail, t as CliError } from "./_output-
|
|
3
|
-
import { n as log } from "./_ui-
|
|
4
|
-
import { i as readProjectConfig } from "./_config-
|
|
5
|
-
import { n as getServerInfo } from "./_agent-
|
|
2
|
+
import { a as ok, n as fail, t as CliError } from "./_output-DbJt9gsl.mjs";
|
|
3
|
+
import { n as log } from "./_ui-DCt4qZrk.mjs";
|
|
4
|
+
import { i as readProjectConfig } from "./_config-ZY2euhdh.mjs";
|
|
5
|
+
import { n as getServerInfo } from "./_agent-Clbq3a4h.mjs";
|
|
6
6
|
import { errorMessage, omitUndefined } from "@alexkroman1/aai/utils";
|
|
7
7
|
import { createWorkflowApiClient } from "@alexkroman1/aai/workflow-api";
|
|
8
|
-
//#region workflow.ts
|
|
8
|
+
//#region src/workflow.ts
|
|
9
9
|
/**
|
|
10
10
|
* `aai workflow` — reading and steering a deployed agent's durable runs from a
|
|
11
11
|
* terminal.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alexkroman1/aai-cli",
|
|
3
|
-
"version": "13.
|
|
3
|
+
"version": "13.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -20,22 +20,22 @@
|
|
|
20
20
|
},
|
|
21
21
|
"exports": {
|
|
22
22
|
"./client-bundler": {
|
|
23
|
-
"@dev/source": "./client-bundler.ts",
|
|
23
|
+
"@dev/source": "./src/client-bundler.ts",
|
|
24
24
|
"types": "./dist/client-bundler.d.ts",
|
|
25
25
|
"import": "./dist/client-bundler.mjs"
|
|
26
26
|
},
|
|
27
27
|
"./worker-bundler": {
|
|
28
|
-
"@dev/source": "./worker-bundler.ts",
|
|
28
|
+
"@dev/source": "./src/worker-bundler.ts",
|
|
29
29
|
"types": "./dist/worker-bundler.d.ts",
|
|
30
30
|
"import": "./dist/worker-bundler.mjs"
|
|
31
31
|
},
|
|
32
32
|
"./typecheck": {
|
|
33
|
-
"@dev/source": "./typecheck.ts",
|
|
33
|
+
"@dev/source": "./src/typecheck.ts",
|
|
34
34
|
"types": "./dist/typecheck.d.ts",
|
|
35
35
|
"import": "./dist/typecheck.mjs"
|
|
36
36
|
},
|
|
37
37
|
"./project-config": {
|
|
38
|
-
"@dev/source": "./project-config.ts",
|
|
38
|
+
"@dev/source": "./src/project-config.ts",
|
|
39
39
|
"types": "./dist/project-config.d.ts",
|
|
40
40
|
"import": "./dist/project-config.mjs"
|
|
41
41
|
}
|
|
@@ -52,9 +52,9 @@
|
|
|
52
52
|
"p-timeout": "^7.0.1",
|
|
53
53
|
"vite": "^8.2.2",
|
|
54
54
|
"zod": "^4.5.4",
|
|
55
|
-
"@alexkroman1/aai": "13.
|
|
56
|
-
"@alexkroman1/aai-runtime": "13.
|
|
57
|
-
"@alexkroman1/aai-ui": "13.
|
|
55
|
+
"@alexkroman1/aai": "13.3.0",
|
|
56
|
+
"@alexkroman1/aai-runtime": "13.3.0",
|
|
57
|
+
"@alexkroman1/aai-ui": "13.3.0"
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"playwright": "^1.62.1",
|
|
@@ -80,8 +80,8 @@
|
|
|
80
80
|
"lint": "biome check .",
|
|
81
81
|
"test": "vitest run",
|
|
82
82
|
"test:coverage": "vitest run --coverage",
|
|
83
|
-
"test:e2e": "VITEST_PROFILE=e2e VITEST_SETUP=./_test-setup.ts VITEST_INCLUDE=e2e*.test.ts vitest run -c ../../vitest.slow.config.ts",
|
|
84
|
-
"test:scenario": "VITEST_PROFILE=scenario VITEST_SETUP=./_test-setup.ts VITEST_INCLUDE=**/*.scenario.test.ts vitest run -c ../../vitest.slow.config.ts",
|
|
83
|
+
"test:e2e": "VITEST_PROFILE=e2e VITEST_SETUP=./src/_test-setup.ts VITEST_INCLUDE=src/e2e*.test.ts vitest run -c ../../vitest.slow.config.ts",
|
|
84
|
+
"test:scenario": "VITEST_PROFILE=scenario VITEST_SETUP=./src/_test-setup.ts VITEST_INCLUDE=**/*.scenario.test.ts vitest run -c ../../vitest.slow.config.ts",
|
|
85
85
|
"typecheck": "tsc --noEmit"
|
|
86
86
|
}
|
|
87
87
|
}
|
package/dist/bin.d.mts
DELETED
package/dist/secret-B-3T5tvr.mjs
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { a as ok, n as fail } from "./_output-CKkmWs7i.mjs";
|
|
3
|
-
import { n as log, s as unwrapCancel } from "./_ui-u7T4YooX.mjs";
|
|
4
|
-
import { f as checkedResponse, p as isStringArray } from "./_studio-B1waXMuR.mjs";
|
|
5
|
-
import { t as secretRequest } from "./_slug-api-CfGQq1Tt.mjs";
|
|
6
|
-
import { isRecord } from "@alexkroman1/aai/utils";
|
|
7
|
-
import * as p from "@clack/prompts";
|
|
8
|
-
import { text } from "node:stream/consumers";
|
|
9
|
-
//#region secret.ts
|
|
10
|
-
/**
|
|
11
|
-
* The one `no_input` failure for `secret put`, shared by the JSON-mode stdin
|
|
12
|
-
* path (cli.ts) and the TTY prompt path below so the two can't drift.
|
|
13
|
-
*/
|
|
14
|
-
const NO_INPUT = [
|
|
15
|
-
"no_input",
|
|
16
|
-
"No value provided",
|
|
17
|
-
"Pipe secret value to stdin"
|
|
18
|
-
];
|
|
19
|
-
/** Read secret value from stdin (for non-TTY / piped input). */
|
|
20
|
-
async function readStdin() {
|
|
21
|
-
return (await text(process.stdin)).trim();
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Execute secret put. If `value` is provided, use it directly (non-TTY path).
|
|
25
|
-
* If not provided, prompt interactively (TTY path).
|
|
26
|
-
*/
|
|
27
|
-
async function executeSecretPut(cwd, name, value, server) {
|
|
28
|
-
let secretValue = value;
|
|
29
|
-
if (!secretValue) {
|
|
30
|
-
const result = unwrapCancel(await p.password({ message: `Enter value for ${name}` }));
|
|
31
|
-
if (!result) return fail(...NO_INPUT);
|
|
32
|
-
secretValue = result;
|
|
33
|
-
}
|
|
34
|
-
const { target } = await secretRequest(cwd, "", {
|
|
35
|
-
method: "PUT",
|
|
36
|
-
body: { [name]: secretValue },
|
|
37
|
-
action: "secret"
|
|
38
|
-
}, server);
|
|
39
|
-
log.success(`Set ${name} for ${target}`);
|
|
40
|
-
return ok({ name });
|
|
41
|
-
}
|
|
42
|
-
async function executeSecretDelete(cwd, name, server) {
|
|
43
|
-
const { target } = await secretRequest(cwd, `/${encodeURIComponent(name)}`, {
|
|
44
|
-
method: "DELETE",
|
|
45
|
-
action: "secret"
|
|
46
|
-
}, server);
|
|
47
|
-
log.success(`Deleted ${name} from ${target}`);
|
|
48
|
-
return ok({ name });
|
|
49
|
-
}
|
|
50
|
-
async function executeSecretList(cwd, server) {
|
|
51
|
-
const { data, target } = await secretRequest(cwd, "", { action: "secret" }, server);
|
|
52
|
-
const { vars } = checkedResponse(data, (value) => isRecord(value) && isStringArray(value.vars), `the secret list for ${target}`);
|
|
53
|
-
if (vars.length === 0) log.info("No secrets set. Use `aai secret put <name>` to add one.");
|
|
54
|
-
else {
|
|
55
|
-
log.message(`${vars.length} secret${vars.length === 1 ? "" : "s"}:`);
|
|
56
|
-
for (const v of vars) log.message(` ${v}`);
|
|
57
|
-
}
|
|
58
|
-
return ok({ secrets: vars });
|
|
59
|
-
}
|
|
60
|
-
//#endregion
|
|
61
|
-
export { NO_INPUT, executeSecretDelete, executeSecretList, executeSecretPut, readStdin };
|