@post-print/agent-test 1.1.9 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/README.md +18 -66
  2. package/dist/cli.d.ts +2 -64
  3. package/dist/cli.d.ts.map +1 -1
  4. package/dist/cli.js +26 -691
  5. package/dist/cli.js.map +1 -1
  6. package/dist/compare-scenario.d.ts +1 -1
  7. package/dist/compare-scenario.d.ts.map +1 -1
  8. package/dist/compare-scenario.js +72 -75
  9. package/dist/compare-scenario.js.map +1 -1
  10. package/dist/failures.d.ts +5 -1
  11. package/dist/failures.d.ts.map +1 -1
  12. package/dist/failures.js +1 -1
  13. package/dist/failures.js.map +1 -1
  14. package/dist/host-login.d.ts +1 -26
  15. package/dist/host-login.d.ts.map +1 -1
  16. package/dist/host-login.js +1 -92
  17. package/dist/host-login.js.map +1 -1
  18. package/dist/index.d.ts +6 -28
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +4 -26
  21. package/dist/index.js.map +1 -1
  22. package/dist/scenario-story.d.ts.map +1 -1
  23. package/dist/scenario-story.js +57 -39
  24. package/dist/scenario-story.js.map +1 -1
  25. package/dist/sdk/cli.d.ts +3 -0
  26. package/dist/sdk/cli.d.ts.map +1 -0
  27. package/dist/sdk/cli.js +20 -0
  28. package/dist/sdk/cli.js.map +1 -0
  29. package/dist/sdk/definitions.d.ts +22 -0
  30. package/dist/sdk/definitions.d.ts.map +1 -0
  31. package/dist/sdk/definitions.js +58 -0
  32. package/dist/sdk/definitions.js.map +1 -0
  33. package/dist/sdk/expect.d.ts +31 -0
  34. package/dist/sdk/expect.d.ts.map +1 -0
  35. package/dist/sdk/expect.js +68 -0
  36. package/dist/sdk/expect.js.map +1 -0
  37. package/dist/sdk/judge.d.ts +16 -0
  38. package/dist/sdk/judge.d.ts.map +1 -0
  39. package/dist/sdk/judge.js +82 -0
  40. package/dist/sdk/judge.js.map +1 -0
  41. package/dist/sdk/metrics.d.ts +3 -0
  42. package/dist/sdk/metrics.d.ts.map +1 -0
  43. package/dist/sdk/metrics.js +24 -0
  44. package/dist/sdk/metrics.js.map +1 -0
  45. package/dist/sdk/reporter.d.ts +12 -0
  46. package/dist/sdk/reporter.d.ts.map +1 -0
  47. package/dist/sdk/reporter.js +58 -0
  48. package/dist/sdk/reporter.js.map +1 -0
  49. package/dist/sdk/runtime.d.ts +33 -0
  50. package/dist/sdk/runtime.d.ts.map +1 -0
  51. package/dist/sdk/runtime.js +188 -0
  52. package/dist/sdk/runtime.js.map +1 -0
  53. package/dist/sdk/test.d.ts +27 -0
  54. package/dist/sdk/test.d.ts.map +1 -0
  55. package/dist/sdk/test.js +92 -0
  56. package/dist/sdk/test.js.map +1 -0
  57. package/dist/sdk/types.d.ts +75 -0
  58. package/dist/sdk/types.d.ts.map +1 -0
  59. package/dist/sdk/types.js +2 -0
  60. package/dist/sdk/types.js.map +1 -0
  61. package/dist/sdk/usage.d.ts +10 -0
  62. package/dist/sdk/usage.d.ts.map +1 -0
  63. package/dist/sdk/usage.js +13 -0
  64. package/dist/sdk/usage.js.map +1 -0
  65. package/dist/sdk/viewer-results.d.ts +43 -0
  66. package/dist/sdk/viewer-results.d.ts.map +1 -0
  67. package/dist/sdk/viewer-results.js +126 -0
  68. package/dist/sdk/viewer-results.js.map +1 -0
  69. package/dist/sdk/viewer.d.ts +11 -0
  70. package/dist/sdk/viewer.d.ts.map +1 -0
  71. package/dist/sdk/viewer.js +160 -0
  72. package/dist/sdk/viewer.js.map +1 -0
  73. package/dist/sdk/workspace.d.ts +33 -0
  74. package/dist/sdk/workspace.d.ts.map +1 -0
  75. package/dist/sdk/workspace.js +100 -0
  76. package/dist/sdk/workspace.js.map +1 -0
  77. package/dist/types.d.ts +4 -40
  78. package/dist/types.d.ts.map +1 -1
  79. package/dist/viewer/catalog.d.ts +2 -8
  80. package/dist/viewer/catalog.d.ts.map +1 -1
  81. package/dist/viewer/catalog.js +23 -128
  82. package/dist/viewer/catalog.js.map +1 -1
  83. package/dist/viewer/client.d.ts.map +1 -1
  84. package/dist/viewer/client.js +12 -12
  85. package/dist/viewer/client.js.map +1 -1
  86. package/dist/viewer/run-controller.d.ts +4 -2
  87. package/dist/viewer/run-controller.d.ts.map +1 -1
  88. package/dist/viewer/run-controller.js +297 -248
  89. package/dist/viewer/run-controller.js.map +1 -1
  90. package/dist/viewer/server.d.ts +4 -3
  91. package/dist/viewer/server.d.ts.map +1 -1
  92. package/dist/viewer/server.js +149 -112
  93. package/dist/viewer/server.js.map +1 -1
  94. package/dist/viewer/styles.d.ts +2 -0
  95. package/dist/viewer/styles.d.ts.map +1 -0
  96. package/dist/viewer/styles.js +444 -0
  97. package/dist/viewer/styles.js.map +1 -0
  98. package/dist/worker-pool.d.ts +5 -1
  99. package/dist/worker-pool.d.ts.map +1 -1
  100. package/dist/worker-pool.js +1 -1
  101. package/dist/worker-pool.js.map +1 -1
  102. package/package.json +7 -7
  103. package/dist/check.d.ts +0 -38
  104. package/dist/check.d.ts.map +0 -1
  105. package/dist/check.js +0 -169
  106. package/dist/check.js.map +0 -1
  107. package/dist/command-allowlist.d.ts +0 -9
  108. package/dist/command-allowlist.d.ts.map +0 -1
  109. package/dist/command-allowlist.js +0 -29
  110. package/dist/command-allowlist.js.map +0 -1
  111. package/dist/debug-bundle.d.ts +0 -81
  112. package/dist/debug-bundle.d.ts.map +0 -1
  113. package/dist/debug-bundle.js +0 -392
  114. package/dist/debug-bundle.js.map +0 -1
  115. package/dist/discover-suites.d.ts +0 -2
  116. package/dist/discover-suites.d.ts.map +0 -1
  117. package/dist/discover-suites.js +0 -19
  118. package/dist/discover-suites.js.map +0 -1
  119. package/dist/doctor.d.ts +0 -24
  120. package/dist/doctor.d.ts.map +0 -1
  121. package/dist/doctor.js +0 -272
  122. package/dist/doctor.js.map +0 -1
  123. package/dist/expect.d.ts +0 -59
  124. package/dist/expect.d.ts.map +0 -1
  125. package/dist/expect.js +0 -518
  126. package/dist/expect.js.map +0 -1
  127. package/dist/host-log.d.ts +0 -17
  128. package/dist/host-log.d.ts.map +0 -1
  129. package/dist/host-log.js +0 -154
  130. package/dist/host-log.js.map +0 -1
  131. package/dist/hosts.d.ts +0 -22
  132. package/dist/hosts.d.ts.map +0 -1
  133. package/dist/hosts.js +0 -84
  134. package/dist/hosts.js.map +0 -1
  135. package/dist/html-report.d.ts +0 -18
  136. package/dist/html-report.d.ts.map +0 -1
  137. package/dist/html-report.js +0 -1276
  138. package/dist/html-report.js.map +0 -1
  139. package/dist/live-isolation.d.ts +0 -82
  140. package/dist/live-isolation.d.ts.map +0 -1
  141. package/dist/live-isolation.js +0 -399
  142. package/dist/live-isolation.js.map +0 -1
  143. package/dist/live-timeout.d.ts +0 -11
  144. package/dist/live-timeout.d.ts.map +0 -1
  145. package/dist/live-timeout.js +0 -29
  146. package/dist/live-timeout.js.map +0 -1
  147. package/dist/load-adapters.d.ts +0 -13
  148. package/dist/load-adapters.d.ts.map +0 -1
  149. package/dist/load-adapters.js +0 -76
  150. package/dist/load-adapters.js.map +0 -1
  151. package/dist/load-suite.d.ts +0 -22
  152. package/dist/load-suite.d.ts.map +0 -1
  153. package/dist/load-suite.js +0 -154
  154. package/dist/load-suite.js.map +0 -1
  155. package/dist/mcp-config.d.ts +0 -13
  156. package/dist/mcp-config.d.ts.map +0 -1
  157. package/dist/mcp-config.js +0 -121
  158. package/dist/mcp-config.js.map +0 -1
  159. package/dist/preflight.d.ts +0 -3
  160. package/dist/preflight.d.ts.map +0 -1
  161. package/dist/preflight.js +0 -17
  162. package/dist/preflight.js.map +0 -1
  163. package/dist/progress.d.ts +0 -24
  164. package/dist/progress.d.ts.map +0 -1
  165. package/dist/progress.js +0 -111
  166. package/dist/progress.js.map +0 -1
  167. package/dist/record-trace.d.ts +0 -64
  168. package/dist/record-trace.d.ts.map +0 -1
  169. package/dist/record-trace.js +0 -124
  170. package/dist/record-trace.js.map +0 -1
  171. package/dist/report-preview-server.d.ts +0 -2
  172. package/dist/report-preview-server.d.ts.map +0 -1
  173. package/dist/report-preview-server.js +0 -24
  174. package/dist/report-preview-server.js.map +0 -1
  175. package/dist/report-preview.d.ts +0 -35
  176. package/dist/report-preview.d.ts.map +0 -1
  177. package/dist/report-preview.js +0 -205
  178. package/dist/report-preview.js.map +0 -1
  179. package/dist/run-suite.d.ts +0 -139
  180. package/dist/run-suite.d.ts.map +0 -1
  181. package/dist/run-suite.js +0 -1688
  182. package/dist/run-suite.js.map +0 -1
  183. package/dist/scenario-finalizer.d.ts +0 -27
  184. package/dist/scenario-finalizer.d.ts.map +0 -1
  185. package/dist/scenario-finalizer.js +0 -39
  186. package/dist/scenario-finalizer.js.map +0 -1
  187. package/dist/scenario-retry.d.ts +0 -15
  188. package/dist/scenario-retry.d.ts.map +0 -1
  189. package/dist/scenario-retry.js +0 -52
  190. package/dist/scenario-retry.js.map +0 -1
  191. package/dist/scenario-seed.d.ts +0 -13
  192. package/dist/scenario-seed.d.ts.map +0 -1
  193. package/dist/scenario-seed.js +0 -72
  194. package/dist/scenario-seed.js.map +0 -1
  195. package/dist/suite-summary.d.ts +0 -16
  196. package/dist/suite-summary.d.ts.map +0 -1
  197. package/dist/suite-summary.js +0 -218
  198. package/dist/suite-summary.js.map +0 -1
  199. package/dist/theme.d.ts +0 -96
  200. package/dist/theme.d.ts.map +0 -1
  201. package/dist/theme.js +0 -446
  202. package/dist/theme.js.map +0 -1
  203. package/dist/validate-seeds.d.ts +0 -20
  204. package/dist/validate-seeds.d.ts.map +0 -1
  205. package/dist/validate-seeds.js +0 -72
  206. package/dist/validate-seeds.js.map +0 -1
  207. package/dist/validate-suite.d.ts +0 -24
  208. package/dist/validate-suite.d.ts.map +0 -1
  209. package/dist/validate-suite.js +0 -542
  210. package/dist/validate-suite.js.map +0 -1
  211. package/dist/viewer/cli.d.ts +0 -17
  212. package/dist/viewer/cli.d.ts.map +0 -1
  213. package/dist/viewer/cli.js +0 -38
  214. package/dist/viewer/cli.js.map +0 -1
  215. package/dist/viewer/compare-metrics.d.ts +0 -30
  216. package/dist/viewer/compare-metrics.d.ts.map +0 -1
  217. package/dist/viewer/compare-metrics.js +0 -101
  218. package/dist/viewer/compare-metrics.js.map +0 -1
  219. package/dist/viewer/context-files.d.ts +0 -8
  220. package/dist/viewer/context-files.d.ts.map +0 -1
  221. package/dist/viewer/context-files.js +0 -57
  222. package/dist/viewer/context-files.js.map +0 -1
  223. package/dist/viewer/emit.d.ts +0 -7
  224. package/dist/viewer/emit.d.ts.map +0 -1
  225. package/dist/viewer/emit.js +0 -25
  226. package/dist/viewer/emit.js.map +0 -1
  227. package/dist/viewer/live-runner.d.ts +0 -27
  228. package/dist/viewer/live-runner.d.ts.map +0 -1
  229. package/dist/viewer/live-runner.js +0 -166
  230. package/dist/viewer/live-runner.js.map +0 -1
  231. package/dist/viewer/session-server.d.ts +0 -2
  232. package/dist/viewer/session-server.d.ts.map +0 -1
  233. package/dist/viewer/session-server.js +0 -71
  234. package/dist/viewer/session-server.js.map +0 -1
  235. package/dist/warnings.d.ts +0 -3
  236. package/dist/warnings.d.ts.map +0 -1
  237. package/dist/warnings.js +0 -13
  238. package/dist/warnings.js.map +0 -1
package/dist/cli.js CHANGED
@@ -1,702 +1,37 @@
1
- #!/usr/bin/env -S node --disable-warning=ExperimentalWarning
2
- import { join, resolve } from "node:path";
1
+ #!/usr/bin/env node
3
2
  import { fileURLToPath } from "node:url";
4
- import { cleanupStaleScenarioWorktrees, isPathUnderRoot, knownAgentHosts, missingClassifierAuth, parseHostAuthModeFlag, setProcessAuthMode, } from "@post-print/agent-harness";
5
- import { collectSuiteHosts, formatCheckReport, formatCheckSummary, missingHostsAuth, runCheck, } from "./check.js";
3
+ import { parseArgs } from "node:util";
6
4
  import { isCliMain } from "./cli-entry.js";
7
- import { parseCompareArmId } from "./compare-scenario.js";
8
- import { missingAgentAuth } from "./doctor.js";
9
- import { installHostLogFilter } from "./host-log.js";
10
- import { ensureCursorSdkLoginForLive, offerCursorSdkReloginIfNeeded, runHostLogin, } from "./host-login.js";
11
- import { parseHostList, uniqueHosts } from "./hosts.js";
12
- import { writeHtmlReport } from "./html-report.js";
13
- import { loadHostAdapters } from "./load-adapters.js";
14
- import { assertDirectAgentPreflight } from "./preflight.js";
15
- import { logProgress } from "./progress.js";
16
- import { cleanupStagingSession, createLiveStagingSessionId, getLiveStagingSessionRoot, setLiveStagingRootOverride, } from "./record-trace.js";
17
- import { shouldStartReportPreview, startDetachedViewer } from "./report-preview.js";
18
- import { registerLiveRunHandlers, runAllSuites, selectedRunNeedsJudge } from "./run-suite.js";
19
- import { formatRunSummary, shouldFailScenario, summarizeReports, } from "./suite-summary.js";
20
- import { configureCliColor, theme } from "./theme.js";
21
- import { loadViewerCatalog } from "./viewer/catalog.js";
22
- import { startViewerCli } from "./viewer/cli.js";
23
- import { suppressNoisyRuntimeWarnings } from "./warnings.js";
24
- import { parseWorkerCount } from "./worker-pool.js";
25
- suppressNoisyRuntimeWarnings();
26
- configureCliColor();
27
- function splitArgvFlag(token) {
28
- if (!token.startsWith("-") || token === "-") {
29
- return undefined;
30
- }
31
- if (token.startsWith("--")) {
32
- const eq = token.indexOf("=");
33
- if (eq === -1) {
34
- return { flag: token };
35
- }
36
- return { flag: token.slice(0, eq), inline: token.slice(eq + 1) };
37
- }
38
- return { flag: token };
39
- }
40
- function readFlagValue(flag, inline, argv, index) {
41
- if (inline !== undefined) {
42
- if (inline.length === 0) {
43
- throw new Error(`${flag} requires a value`);
44
- }
45
- return { value: inline, nextIndex: index };
46
- }
47
- const next = argv[index + 1];
48
- if (!next || next.startsWith("--") || /^-[A-Za-z]/.test(next)) {
49
- throw new Error(`${flag} requires a value`);
50
- }
51
- return { value: next, nextIndex: index + 1 };
52
- }
53
- function enableCheckMode(target) {
54
- target.check = true;
55
- target.validateOnly = true;
56
- target.validatePaths = true;
57
- target.validateSeeds = true;
58
- }
59
- /** Usage text for `--help`. */
60
- export function formatHelp() {
61
- return [
62
- "agent-test [options] [suite]",
63
- "agent-test login [--host cursor|claude|openai]",
64
- "agent-test viewer [--suites-dir agent-suites]",
65
- "",
66
- "Launch a host agent and score the transcript.",
67
- "A run checks the suite and host first.",
68
- "",
69
- " login Store a Cursor SDK login. Codex uses `codex login`.",
70
- " viewer Open the unified localhost catalog and session run history.",
71
- " --check Check the suite and host. Do not launch an agent.",
72
- " --help, -h Print this help.",
73
- "",
74
- " --host cursor|claude|openai|all Host adapter (default: suite or cursor).",
75
- " Repeat or comma-separate for a matrix.",
76
- " --auth-mode subscription|api-key",
77
- " Host auth (default: subscription).",
78
- " --auth-method is the same flag.",
79
- " --adapter <module> Load a consumer host adapter module.",
80
- " --suites-dir <path> Suite root (default: agent-suites)",
81
- " --suite <name> Run one suite",
82
- " --scenario <name> Run one scenario",
83
- " --compare-arm <id> Isolated child: run one compare arm",
84
- " --port <n> Viewer listen port. Default is a free port.",
85
- " --no-judge Skip the judge",
86
- " --fail-on all|behavior|infra-only",
87
- " --allow-user-input Let a user agent answer AskQuestion tools",
88
- " --timeout-ms <n> Agent deadline",
89
- " --no-timeout Disable the deadline",
90
- " --scenario-retries <n> Announce-stop retries",
91
- " --workers <n> Parallel live agents (1-32). Viewer default 4. CLI default 1.",
92
- " --debug Keep recordings and write a debug bundle",
93
- " --debug-dir <path> Debug session parent directory",
94
- " --report-out <path> Self-contained read-only HTML report file or directory",
95
- " --no-html-report Skip the HTML report",
96
- " --no-worktree Run in the caller checkout (needs AGENT_TEST_ALLOW_IN_PLACE=1)",
97
- "",
98
- "--doctor, --validate-only, --validate-paths, and --validate-seeds run --check.",
99
- ].join("\n");
100
- }
101
- /** Parse agent-test CLI argv (exported for unit tests). */
102
- export function parseCliArgs(argv) {
103
- const cwd = process.cwd();
104
- let suitesDir = "agent-suites";
105
- let rubricsDir;
106
- const parsedHosts = [];
107
- let hostAll = false;
108
- const adapterModules = [];
109
- let filter;
110
- let scenarioFilter;
111
- let stagingSessionId;
112
- let judge;
113
- let worktree;
114
- let keepRecordings = false;
115
- let timeoutMs;
116
- let noTimeout = false;
117
- let allowUserInput = false;
118
- let check = false;
119
- let help = false;
120
- let doctor = false;
121
- let htmlReport = true;
122
- let reportOut;
123
- let debug = process.env.AGENT_TEST_DEBUG === "1" || process.env.AGENT_TEST_DEBUG === "true";
124
- let debugDir;
125
- let validateOnly = false;
126
- let validateSeeds = false;
127
- let validatePaths = false;
128
- let failOn = "all";
129
- let scenarioRetries;
130
- let authMode;
131
- let compareArm;
132
- let viewerPort;
133
- let workers;
134
- if (argv[2] === "compare") {
135
- throw new Error("The compare subcommand is removed. Add compare.a and compare.b, or compare.arms, on one scenario. Add compare.gates for required outcomes or metrics.");
136
- }
137
- const login = argv[2] === "login";
138
- const viewer = argv[2] === "viewer";
139
- const startIndex = login || viewer ? 3 : 2;
140
- const checkMode = {
141
- check: false,
142
- validateOnly: false,
143
- validatePaths: false,
144
- validateSeeds: false,
145
- };
146
- for (let i = startIndex; i < argv.length; i++) {
147
- const token = argv[i];
148
- if (!token) {
149
- continue;
150
- }
151
- const parsed = splitArgvFlag(token);
152
- if (!parsed) {
153
- if (login) {
154
- throw new Error("login does not take a suite name");
155
- }
156
- filter = token;
157
- continue;
158
- }
159
- const { flag, inline } = parsed;
160
- const read = () => {
161
- const result = readFlagValue(flag, inline, argv, i);
162
- i = result.nextIndex;
163
- return result.value;
164
- };
165
- switch (flag) {
166
- case "--help":
167
- case "-h":
168
- help = true;
169
- break;
170
- case "--check":
171
- enableCheckMode(checkMode);
172
- break;
173
- case "--doctor":
174
- doctor = true;
175
- enableCheckMode(checkMode);
176
- break;
177
- case "--validate-only":
178
- enableCheckMode(checkMode);
179
- break;
180
- case "--validate-seeds":
181
- enableCheckMode(checkMode);
182
- break;
183
- case "--validate-paths":
184
- enableCheckMode(checkMode);
185
- break;
186
- case "--host": {
187
- const value = read();
188
- if (value
189
- .split(",")
190
- .map((token) => token.trim().toLowerCase())
191
- .includes("all")) {
192
- hostAll = true;
193
- }
194
- parsedHosts.push(...parseHostList(value));
195
- break;
196
- }
197
- case "--auth-mode":
198
- case "--auth-method":
199
- authMode = parseHostAuthModeFlag(read(), flag);
200
- break;
201
- case "--adapter":
202
- adapterModules.push(resolve(cwd, read()));
203
- break;
204
- case "--suites-dir":
205
- suitesDir = read();
206
- break;
207
- case "--rubrics-dir":
208
- rubricsDir = read();
209
- break;
210
- case "--suite":
211
- filter = read();
212
- break;
213
- case "--scenario":
214
- scenarioFilter = read();
215
- break;
216
- case "--compare-arm": {
217
- const value = parseCompareArmId(read());
218
- if (!value) {
219
- throw new Error("--compare-arm must be a lowercase slug");
220
- }
221
- compareArm = value;
222
- break;
223
- }
224
- case "--port": {
225
- const parsedPort = Number(read());
226
- if (!Number.isInteger(parsedPort) || parsedPort < 0 || parsedPort > 65535) {
227
- throw new Error("--port must be an integer 0-65535");
228
- }
229
- viewerPort = parsedPort;
230
- break;
231
- }
232
- case "--staging-session-id":
233
- stagingSessionId = read();
234
- break;
235
- case "--live":
236
- throw new Error("--live was removed because agent-test now always runs a real agent");
237
- case "--record":
238
- throw new Error("--record was removed; direct runs capture transient traces automatically (use --keep-recordings to retain them)");
239
- case "--record-fixtures":
240
- throw new Error("--record-fixtures was removed because replay-based testing is deprecated and no longer supported");
241
- case "--keep-recordings":
242
- keepRecordings = true;
243
- break;
244
- case "--judge":
245
- judge = true;
246
- break;
247
- case "--no-judge":
248
- judge = false;
249
- break;
250
- case "--no-worktree":
251
- worktree = false;
252
- break;
253
- case "--timeout-ms": {
254
- const parsedTimeout = Number(read());
255
- if (Number.isFinite(parsedTimeout) && parsedTimeout > 0) {
256
- timeoutMs = parsedTimeout;
257
- }
258
- break;
259
- }
260
- case "--no-timeout":
261
- noTimeout = true;
262
- break;
263
- case "--allow-user-input":
264
- allowUserInput = true;
265
- break;
266
- case "--fail-on": {
267
- const mode = read();
268
- if (mode !== "all" && mode !== "behavior" && mode !== "infra-only") {
269
- throw new Error("--fail-on must be all|behavior|infra-only");
270
- }
271
- failOn = mode;
272
- break;
273
- }
274
- case "--scenario-retries": {
275
- const parsedRetries = Number(read());
276
- if (!Number.isInteger(parsedRetries) || parsedRetries < 0) {
277
- throw new Error("--scenario-retries must be an integer >= 0");
278
- }
279
- scenarioRetries = parsedRetries;
280
- break;
281
- }
282
- case "--workers":
283
- workers = parseWorkerCount(read());
284
- break;
285
- case "--compare-pairs":
286
- case "--a":
287
- case "--b":
288
- case "--compare-a":
289
- case "--compare-b":
290
- case "--out-dir":
291
- case "--compare-out":
292
- throw new Error(`${flag} is removed. Add compare.a and compare.b, or compare.arms, on one scenario. Add compare.gates for required outcomes or metrics.`);
293
- case "--report-out":
294
- reportOut = read();
295
- break;
296
- case "--no-html-report":
297
- htmlReport = false;
298
- break;
299
- case "--debug":
300
- debug = true;
301
- break;
302
- case "--debug-dir": {
303
- const value = read();
304
- debugDir = value;
305
- debug = true;
306
- break;
307
- }
308
- default:
309
- throw new Error(`Unknown flag: ${flag}`);
310
- }
311
- }
312
- check = checkMode.check;
313
- validateOnly = checkMode.validateOnly;
314
- validatePaths = checkMode.validatePaths;
315
- validateSeeds = checkMode.validateSeeds;
316
- const hosts = uniqueHosts(parsedHosts);
317
- const host = hosts?.[0];
318
- judge = judge ?? true;
319
- worktree = worktree ?? true;
320
- if (workers === undefined) {
321
- const fromEnv = process.env.AGENT_TEST_WORKERS?.trim();
322
- if (fromEnv) {
323
- workers = parseWorkerCount(fromEnv, "AGENT_TEST_WORKERS");
324
- }
325
- }
326
- if (debug) {
327
- keepRecordings = true;
328
- process.env.AGENT_TEST_DEBUG = "1";
329
- process.env.AGENT_TEST_VERBOSE = process.env.AGENT_TEST_VERBOSE ?? "1";
330
- process.env.AGENT_TEST_VERBOSE_PATHS = process.env.AGENT_TEST_VERBOSE_PATHS ?? "1";
331
- }
332
- return {
333
- cwd,
334
- suitesDir,
335
- rubricsDir,
336
- host,
337
- hosts,
338
- hostAll,
339
- adapterModules: adapterModules.length > 0 ? adapterModules : undefined,
340
- filter,
341
- scenarioFilter,
342
- stagingSessionId,
343
- judge,
344
- worktree,
345
- keepRecordings,
346
- timeoutMs: noTimeout ? 0 : timeoutMs,
347
- noTimeout,
348
- allowUserInput,
349
- check,
350
- help,
351
- doctor,
352
- htmlReport,
353
- reportOut: reportOut ? resolve(cwd, reportOut) : undefined,
354
- debug,
355
- debugDir: debugDir ? resolve(cwd, debugDir) : undefined,
356
- validateOnly,
357
- validateSeeds,
358
- validatePaths,
359
- failOn,
360
- scenarioRetries,
361
- authMode,
362
- login,
363
- viewer,
364
- compareArm,
365
- viewerPort,
366
- workers,
367
- };
368
- }
369
- /** Resolve an explicit report target into the HTML path and optional artifact directory. */
370
- export function resolveReportOutput(reportOut) {
371
- if (!reportOut) {
372
- return {};
373
- }
374
- if (reportOut.toLowerCase().endsWith(".html")) {
375
- return { htmlPath: reportOut };
376
- }
377
- return { htmlPath: join(reportOut, "report.html"), outDir: reportOut };
378
- }
379
- /** Path passed to `writeHtmlReport`. `undefined` keeps the temp default. */
380
- export function resolveHtmlReportWritePath(htmlReport, reportOut) {
381
- if (!htmlReport) {
382
- return undefined;
383
- }
384
- return resolveReportOutput(reportOut).htmlPath;
385
- }
386
- async function htmlReportTip(label, reportPath, reports, args) {
387
- if (shouldStartReportPreview()) {
388
- const url = await startDetachedViewer({
389
- cliPath: process.argv[1],
390
- cwd: args.cwd,
391
- suitesDir: args.suitesDir,
392
- rubricsDir: args.rubricsDir,
393
- judge: args.judge,
394
- timeoutMs: args.timeoutMs,
395
- authMode: args.authMode,
396
- adapterModules: args.adapterModules,
397
- workers: args.workers,
398
- worktree: args.worktree,
399
- keepRecordings: args.keepRecordings,
400
- allowUserInput: args.allowUserInput,
401
- debug: args.debug,
402
- debugDir: args.debugDir,
403
- request: {
404
- ...(args.filter ? { suite: args.filter } : {}),
405
- ...(args.scenarioFilter ? { scenario: args.scenarioFilter } : {}),
406
- ...(args.hosts?.length ? { hosts: args.hosts } : args.host ? { hosts: [args.host] } : {}),
407
- },
408
- reports,
5
+ import { runHostLogin } from "./host-login.js";
6
+ import { runSdkCli } from "./sdk/cli.js";
7
+ import { runSdkViewerCli } from "./sdk/viewer.js";
8
+ export async function runCli(args) {
9
+ const [command, ...rest] = args;
10
+ if (command === "test")
11
+ return runSdkCli(rest);
12
+ if (command === "viewer")
13
+ return runSdkViewerCli(rest);
14
+ if (command === "login") {
15
+ const { values } = parseArgs({
16
+ args: rest,
17
+ options: { host: { type: "string", default: "cursor" } },
409
18
  });
410
- if (url) {
411
- return theme.fileTip(label, url);
412
- }
19
+ return runHostLogin(values.host);
413
20
  }
414
- return theme.fileTip(label, reportPath);
415
- }
416
- async function cleanupRunArtifacts(stagingSessionRoot, keepRecordings) {
417
- if (keepRecordings) {
418
- return;
419
- }
420
- if (stagingSessionRoot) {
421
- try {
422
- await cleanupStagingSession(stagingSessionRoot);
423
- }
424
- catch {
425
- // best-effort
426
- }
427
- }
428
- }
429
- async function main() {
430
- let args;
431
- try {
432
- args = parseCliArgs(process.argv);
433
- if (args.authMode) {
434
- setProcessAuthMode(args.authMode);
435
- }
436
- installHostLogFilter();
437
- }
438
- catch (error) {
439
- console.error(error instanceof Error ? error.message : error);
440
- return 1;
441
- }
442
- if (args.help) {
443
- console.log(formatHelp());
21
+ if (!command || command === "--help" || command === "-h") {
22
+ console.log("agent-test test [Playwright options]\nagent-test viewer [--config path] [--port number]\nagent-test login [--host cursor|openai|claude]\n\nConfigure agents in agent-test.config.ts. JSON suites and legacy scoring flags were removed.");
444
23
  return 0;
445
24
  }
446
- if (args.login) {
447
- try {
448
- return await runHostLogin(args.host ?? "cursor");
449
- }
450
- catch (error) {
451
- console.error(error instanceof Error ? error.message : error);
452
- return 1;
453
- }
454
- }
455
- try {
456
- await loadHostAdapters({ cwd: args.cwd, adapterModules: args.adapterModules });
457
- if (args.hostAll) {
458
- args.hosts = uniqueHosts(knownAgentHosts());
459
- args.host = args.hosts?.[0];
460
- }
461
- }
462
- catch (error) {
463
- console.error(error instanceof Error ? error.message : error);
464
- return 1;
465
- }
466
- if (args.viewer) {
467
- try {
468
- return await startViewerCli(args);
469
- }
470
- catch (error) {
471
- console.error(error instanceof Error ? error.message : error);
472
- return 1;
473
- }
474
- }
475
- if (args.check || args.doctor || args.validateOnly || args.validateSeeds) {
476
- const report = await runCheck({
477
- cwd: args.cwd,
478
- suitesDir: args.suitesDir,
479
- filter: args.filter,
480
- rubricsDir: args.rubricsDir,
481
- host: args.host,
482
- hosts: args.hosts,
483
- });
484
- console.log(formatCheckReport(report));
485
- return report.ok ? 0 : 1;
486
- }
487
- if (args.debugDir) {
488
- setLiveStagingRootOverride(args.debugDir);
489
- }
490
- const isChild = process.env.AGENT_TEST_CHILD === "1";
491
- if (args.debugDir && isPathUnderRoot(args.debugDir, args.cwd) && !isChild) {
492
- console.warn(theme.warn(`--debug-dir is inside the repo (${args.debugDir}). Default is $TMPDIR/agent-spec — prefer that for direct runs so debug output stays out of git status.`));
493
- }
494
- const verbose = args.debug || process.env.AGENT_TEST_VERBOSE === "1" || process.env.AGENT_TEST_DEBUG === "1";
495
- const stagingSessionId = args.stagingSessionId?.trim() ||
496
- process.env.AGENT_TEST_STAGING_SESSION_ID?.trim() ||
497
- createLiveStagingSessionId();
498
- const stagingSessionRoot = stagingSessionId
499
- ? getLiveStagingSessionRoot(stagingSessionId)
500
- : undefined;
501
- try {
502
- let checkHosts = args.hosts ?? (args.host ? [args.host] : ["cursor"]);
503
- if (!isChild) {
504
- checkHosts = await collectSuiteHosts({
505
- cwd: args.cwd,
506
- suitesDir: args.suitesDir,
507
- filter: args.filter,
508
- rubricsDir: args.rubricsDir,
509
- host: args.host,
510
- hosts: args.hosts,
511
- });
512
- await ensureCursorSdkLoginForLive({
513
- hosts: checkHosts,
514
- authMode: args.authMode,
515
- });
516
- const report = await runCheck({
517
- cwd: args.cwd,
518
- suitesDir: args.suitesDir,
519
- filter: args.filter,
520
- rubricsDir: args.rubricsDir,
521
- host: args.host,
522
- hosts: args.hosts,
523
- });
524
- console.log(theme.tip(formatCheckSummary(report)));
525
- if (!report.ok) {
526
- console.error(formatCheckReport(report));
527
- return 1;
528
- }
529
- checkHosts = report.hosts;
530
- const missingHost = missingHostsAuth(report.hosts);
531
- if (missingHost) {
532
- console.error(missingHost);
533
- return 1;
534
- }
535
- }
536
- else {
537
- const missingHost = missingAgentAuth(args.host ?? "cursor");
538
- if (missingHost) {
539
- console.error(missingHost);
540
- return 1;
541
- }
542
- }
543
- if (args.judge !== false) {
544
- let needsJudge = false;
545
- try {
546
- needsJudge = await selectedRunNeedsJudge({
547
- cwd: args.cwd,
548
- suitesDir: args.suitesDir,
549
- filter: args.filter,
550
- scenarioFilter: args.scenarioFilter,
551
- rubricsDir: args.rubricsDir,
552
- judge: args.judge,
553
- });
554
- }
555
- catch {
556
- needsJudge = false;
557
- }
558
- if (needsJudge) {
559
- for (const host of checkHosts) {
560
- const missingJudge = missingClassifierAuth(host);
561
- if (missingJudge) {
562
- console.error(`${missingJudge} (use --no-judge to skip)`);
563
- return 1;
564
- }
565
- }
566
- }
567
- }
568
- {
569
- try {
570
- await assertDirectAgentPreflight(args.cwd, args.suitesDir);
571
- }
572
- catch (error) {
573
- console.error(error instanceof Error ? error.message : error);
574
- return 1;
575
- }
576
- const inPlaceAllowed = process.env.AGENT_TEST_ALLOW_IN_PLACE === "1";
577
- const worktreeDisabled = args.worktree === false ||
578
- process.env.AGENT_TEST_NO_WORKTREE === "1" ||
579
- process.env.AGENT_TEST_NO_WORKTREE === "true";
580
- if (worktreeDisabled && !inPlaceAllowed) {
581
- console.error("Direct agent tests require a sealed temp workspace. Set AGENT_TEST_ALLOW_IN_PLACE=1 to run in repo cwd (--no-worktree leaks agent edits into your working tree).");
582
- return 1;
583
- }
584
- registerLiveRunHandlers();
585
- if (!isChild) {
586
- const removed = await cleanupStaleScenarioWorktrees(args.cwd);
587
- if (removed.length > 0) {
588
- console.log(theme.warn(`Cleaned ${removed.length} stale agent-test worktree(s) from a prior crash`));
589
- }
590
- console.log(theme.banner(args.debug ? "direct debug" : "direct"));
591
- if (stagingSessionRoot) {
592
- console.log(theme.bannerSession(stagingSessionRoot));
593
- }
594
- const hints = ["Ctrl+C cancels"];
595
- if (args.debug || process.env.AGENT_TEST_VERBOSE === "1") {
596
- hints.push(args.keepRecordings ? "recordings kept" : "traces removed on exit");
597
- hints.push("exit 137 = macOS OOM");
598
- }
599
- else if (!args.keepRecordings) {
600
- hints.push("traces removed on exit");
601
- }
602
- console.log(theme.bannerHints(hints));
603
- if (worktreeDisabled) {
604
- console.warn(theme.warn("running in repo cwd — agent file edits will persist in your working tree"));
605
- }
606
- }
607
- }
608
- const reports = await runAllSuites({
609
- ...args,
610
- stagingSessionId,
611
- timeoutMs: args.timeoutMs,
612
- allowUserInput: args.allowUserInput,
613
- debug: args.debug,
614
- debugDir: args.debugDir,
615
- });
616
- let exitCode = 0;
617
- if (!isChild) {
618
- for (const report of reports) {
619
- const failed = report.results.filter((result) => !result.passed && !result.skipped);
620
- if (failed.length > 0) {
621
- const behaviorFailures = failed.filter((result) => shouldFailScenario(result.failures, args.failOn));
622
- if (behaviorFailures.length > 0) {
623
- exitCode = 1;
624
- }
625
- console.log(`\n${theme.failedScenariosHeader()}`);
626
- for (const result of failed) {
627
- console.log(theme.failedScenarioName(result.scenario));
628
- if (verbose) {
629
- for (const failure of result.failures) {
630
- console.log(theme.verboseFailure(failure.matcher, failure.message, failure.evidence, failure.category));
631
- }
632
- }
633
- if (args.debug && result.debugBundleDir) {
634
- console.log(` ${theme.tip(result.debugBundleDir)}`);
635
- }
636
- }
637
- }
638
- console.log(theme.summary(report.suite, report.passed, report.failed, report.skipped));
639
- }
640
- const runSummary = summarizeReports(reports);
641
- const runSummaryText = formatRunSummary(runSummary);
642
- if (runSummaryText) {
643
- console.log(`\n${theme.runSummary(runSummaryText)}`);
644
- }
645
- if (args.failOn === "behavior" && runSummary.infraFailures > 0 && exitCode === 0) {
646
- console.log(theme.tip(`${runSummary.infraFailures} infra failure(s) ignored (--fail-on=behavior)`));
647
- }
648
- if (args.debug && stagingSessionRoot) {
649
- console.log(`\n${theme.tip(`debug session: ${stagingSessionRoot}`)}`);
650
- }
651
- if (args.htmlReport && reports.length > 0) {
652
- try {
653
- const catalog = await loadViewerCatalog({
654
- cwd: args.cwd,
655
- suitesDir: args.suitesDir,
656
- rubricsDir: args.rubricsDir,
657
- }).catch(() => undefined);
658
- const reportPath = await writeHtmlReport(reports, {
659
- host: args.host,
660
- suitesDir: args.suitesDir,
661
- ...(catalog ? { catalog } : {}),
662
- }, resolveHtmlReportWritePath(args.htmlReport, args.reportOut));
663
- console.log(`\n${await htmlReportTip("View report", reportPath, reports, args)}`);
664
- }
665
- catch (error) {
666
- console.warn(theme.warn(`HTML report failed: ${error instanceof Error ? error.message : String(error)}`));
667
- }
668
- }
669
- }
670
- else {
671
- for (const report of reports) {
672
- const failed = report.results.filter((result) => !result.passed && !result.skipped);
673
- if (failed.some((result) => shouldFailScenario(result.failures, args.failOn))) {
674
- exitCode = 1;
675
- }
676
- }
677
- }
678
- if (reports.length === 0) {
679
- logProgress(`No suites found under ${args.suitesDir}`);
680
- return 1;
681
- }
682
- if (!isChild) {
683
- await offerCursorSdkReloginIfNeeded({ reports });
684
- }
685
- return exitCode;
686
- }
687
- finally {
688
- if (stagingSessionId && !isChild) {
689
- await cleanupRunArtifacts(stagingSessionRoot, args.keepRecordings);
690
- }
691
- }
25
+ throw new Error(`Unsupported command: ${command}. Use agent-test test with a TypeScript suite. Run agent-test --help for commands.`);
692
26
  }
693
- const entry = fileURLToPath(import.meta.url);
694
- if (isCliMain(process.argv[1], entry)) {
695
- void main()
696
- .then((code) => process.exit(code))
27
+ if (isCliMain(process.argv[1], fileURLToPath(import.meta.url))) {
28
+ runCli(process.argv.slice(2))
29
+ .then((code) => {
30
+ process.exitCode = code;
31
+ })
697
32
  .catch((error) => {
698
- console.error(error instanceof Error ? error.message : error);
699
- process.exit(1);
33
+ console.error(error instanceof Error ? error.message : String(error));
34
+ process.exitCode = 1;
700
35
  });
701
36
  }
702
37
  //# sourceMappingURL=cli.js.map