@post-print/agent-test 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -173
- package/dist/cli.d.ts +2 -8
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +53 -86
- package/dist/cli.js.map +1 -1
- package/dist/debug-bundle.d.ts +0 -1
- package/dist/debug-bundle.d.ts.map +1 -1
- package/dist/debug-bundle.js +0 -7
- package/dist/debug-bundle.js.map +1 -1
- package/dist/doctor.js +4 -4
- package/dist/doctor.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +0 -1
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +1 -4
- package/dist/live-isolation.js.map +1 -1
- package/dist/load-suite.d.ts.map +1 -1
- package/dist/load-suite.js +12 -0
- package/dist/load-suite.js.map +1 -1
- package/dist/preflight.d.ts +2 -2
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js +4 -4
- package/dist/preflight.js.map +1 -1
- package/dist/record-trace.d.ts +3 -14
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js +7 -43
- package/dist/record-trace.js.map +1 -1
- package/dist/run-suite.d.ts +28 -10
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +157 -135
- package/dist/run-suite.js.map +1 -1
- package/dist/types.d.ts +11 -14
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +11 -12
- package/dist/validate-suite.js.map +1 -1
- package/package.json +3 -3
package/dist/record-trace.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { mkdir,
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
|
-
import { dirname, join
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
5
|
import { enrichTrace } from "@post-print/agent-harness";
|
|
6
6
|
export const LIVE_STAGING_DIR_NAME = "agent-spec";
|
|
7
7
|
let liveStagingRootOverride;
|
|
@@ -80,26 +80,14 @@ export function getStagingResultPath(stagingSessionId, suiteName, scenarioName)
|
|
|
80
80
|
export function createLiveStagingSessionId() {
|
|
81
81
|
return `${process.pid}-${Date.now()}`;
|
|
82
82
|
}
|
|
83
|
-
/**
|
|
84
|
-
export function resolveRecordingPath(suiteName, scenarioName,
|
|
85
|
-
if (
|
|
86
|
-
if (!replayTrace) {
|
|
87
|
-
return undefined;
|
|
88
|
-
}
|
|
89
|
-
return {
|
|
90
|
-
path: resolve(options.repoRoot, replayTrace),
|
|
91
|
-
kind: "fixture",
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
if (!options.stagingSessionId) {
|
|
83
|
+
/** Resolve the transient trace used by isolated direct-agent runs. */
|
|
84
|
+
export function resolveRecordingPath(suiteName, scenarioName, stagingSessionId) {
|
|
85
|
+
if (!stagingSessionId) {
|
|
95
86
|
return undefined;
|
|
96
87
|
}
|
|
97
|
-
return
|
|
98
|
-
path: getStagingTracePath(options.stagingSessionId, suiteName, scenarioName),
|
|
99
|
-
kind: "staging",
|
|
100
|
-
};
|
|
88
|
+
return getStagingTracePath(stagingSessionId, suiteName, scenarioName);
|
|
101
89
|
}
|
|
102
|
-
/** Persist a
|
|
90
|
+
/** Persist a direct-run trace for parent-process judging and diagnostics. */
|
|
103
91
|
export async function recordTrace(outputPath, trace) {
|
|
104
92
|
const enriched = enrichTrace(trace);
|
|
105
93
|
const payload = {
|
|
@@ -142,28 +130,4 @@ export async function loadStagingResult(path) {
|
|
|
142
130
|
export async function cleanupStagingSession(sessionRoot) {
|
|
143
131
|
await rm(sessionRoot, { recursive: true, force: true });
|
|
144
132
|
}
|
|
145
|
-
/** Remove legacy in-repo staging dirs from before tmpdir staging (best-effort). */
|
|
146
|
-
export async function cleanupLegacyRepoRecordings(repoRoot) {
|
|
147
|
-
const suitesRoot = join(repoRoot, "agent-suites");
|
|
148
|
-
const removed = [];
|
|
149
|
-
let suiteNames;
|
|
150
|
-
try {
|
|
151
|
-
suiteNames = await readdir(suitesRoot);
|
|
152
|
-
}
|
|
153
|
-
catch {
|
|
154
|
-
return removed;
|
|
155
|
-
}
|
|
156
|
-
for (const suiteName of suiteNames) {
|
|
157
|
-
const recordingsDir = join(suitesRoot, suiteName, "fixtures", "recordings");
|
|
158
|
-
try {
|
|
159
|
-
await stat(recordingsDir);
|
|
160
|
-
await rm(recordingsDir, { recursive: true, force: true });
|
|
161
|
-
removed.push(recordingsDir);
|
|
162
|
-
}
|
|
163
|
-
catch {
|
|
164
|
-
// absent
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
return removed;
|
|
168
|
-
}
|
|
169
133
|
//# sourceMappingURL=record-trace.js.map
|
package/dist/record-trace.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"record-trace.js","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"record-trace.js","sourceRoot":"","sources":["../src/record-trace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAIxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;AAElD,IAAI,uBAA2C,CAAC;AAEhD,2FAA2F;AAC3F,MAAM,UAAU,0BAA0B,CAAC,IAAwB;IAClE,uBAAuB,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,0BAA0B;IACzC,OAAO,uBAAuB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAChD,MAAM,IAAI,GACT,IAAI;SACF,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC;IAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzE,OAAO,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,OAAO,IAAI;SACT,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC;SAC7B,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;SAClB,IAAI,EAAE;SACN,WAAW,EAAE,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,IAAI,uBAAuB,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,uBAAuB,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,qBAAqB,EAAE,UAAU,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,SAAiB;IAC1D,OAAO,IAAI,CAAC,kBAAkB,EAAE,EAAE,SAAS,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAoB;IACpD,OAAO,oBAAoB,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,mBAAmB,CAClC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,OAAO,CAC/C,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,wBAAwB,CACvC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,cAAc,CACtD,CAAC;AACH,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY;IACvD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACtD,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,oBAAoB,CACnC,gBAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,OAAO,IAAI,CACV,yBAAyB,CAAC,gBAAgB,CAAC,EAC3C,SAAS,EACT,GAAG,uBAAuB,CAAC,YAAY,CAAC,cAAc,CACtD,CAAC;AACH,CAAC;AAQD,MAAM,UAAU,0BAA0B;IACzC,OAAO,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AACvC,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,oBAAoB,CACnC,SAAiB,EACjB,YAAoB,EACpB,gBAAoC;IAEpC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO,mBAAmB,CAAC,gBAAgB,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AACvE,CAAC;AAED,6EAA6E;AAC7E,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB,EAAE,KAAiB;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG;QACf,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,wBAAwB,EAAE,QAAQ,CAAC,wBAAwB;QAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;KACrC,CAAC;IAEF,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7E,OAAO,UAAU,CAAC;AACnB,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAe,CAAC;IACnE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,IAAY,EACZ,MAAiC;IAEjC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACvE,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,IAAY;IAEZ,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAA8B,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,WAAmB;IAC9D,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC"}
|
package/dist/run-suite.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AgentHost, RoutingContract } from "@post-print/agent-harness";
|
|
2
|
-
import type { ScenarioRubric, SuiteRunReport } from "./types.js";
|
|
2
|
+
import type { AgentScenario, AgentSuiteDefaults, ScenarioResult, ScenarioRubric, SuiteRunReport } from "./types.js";
|
|
3
3
|
/** Parent process prints suite headers and final verdicts; children only print phases. */
|
|
4
4
|
export declare function shouldPrintSuiteChrome(): boolean;
|
|
5
5
|
/** Best-effort cancel + worktree cleanup when live runs are interrupted (Ctrl+C / SIGINT/SIGTERM). */
|
|
@@ -10,15 +10,11 @@ export interface RunSuiteOptions {
|
|
|
10
10
|
host?: AgentHost;
|
|
11
11
|
/** Run only this scenario name (used by live subprocess isolation). */
|
|
12
12
|
scenarioFilter?: string;
|
|
13
|
-
/**
|
|
14
|
-
record?: boolean;
|
|
15
|
-
/** Overwrite committed replayTrace paths (default: write staging traces under $TMPDIR). */
|
|
16
|
-
recordFixtures?: boolean;
|
|
17
|
-
/** Run harness LLM judge for rubric.judge criteria (live hosts only). */
|
|
13
|
+
/** Run the harness LLM judge for rubric.judge criteria. */
|
|
18
14
|
judge?: boolean;
|
|
19
|
-
/** Isolate each
|
|
15
|
+
/** Isolate each scenario in a detached git worktree. */
|
|
20
16
|
worktree?: boolean;
|
|
21
|
-
/** Session id for
|
|
17
|
+
/** Session id for transient staging traces under $TMPDIR (see record-trace.ts). */
|
|
22
18
|
stagingSessionId?: string;
|
|
23
19
|
keepRecordings?: boolean;
|
|
24
20
|
suitesDir?: string;
|
|
@@ -36,17 +32,39 @@ export interface RunSuiteOptions {
|
|
|
36
32
|
/** Prefer `<rubricsDir>/<suite>/rubrics.json` over sibling rubrics (harness-only). */
|
|
37
33
|
rubricsDir?: string;
|
|
38
34
|
}
|
|
35
|
+
export interface RunAgentTestOptions {
|
|
36
|
+
cwd: string;
|
|
37
|
+
scenario: AgentScenario;
|
|
38
|
+
suiteName?: string;
|
|
39
|
+
defaults?: AgentSuiteDefaults;
|
|
40
|
+
/** Overrides the suite default; a scenario host still wins. */
|
|
41
|
+
host?: AgentHost;
|
|
42
|
+
judge?: boolean;
|
|
43
|
+
worktree?: boolean;
|
|
44
|
+
timeoutMs?: number;
|
|
45
|
+
allowUserInput?: boolean;
|
|
46
|
+
debug?: boolean;
|
|
47
|
+
debugDir?: string;
|
|
48
|
+
keepRecordings?: boolean;
|
|
49
|
+
scenarioRetries?: number;
|
|
50
|
+
/** Internal suite orchestration metadata; safe for programmatic callers to omit. */
|
|
51
|
+
stagingSessionId?: string;
|
|
52
|
+
suitesDir?: string;
|
|
53
|
+
rubricsDir?: string;
|
|
54
|
+
scenarioIndex?: number;
|
|
55
|
+
scenarioTotal?: number;
|
|
56
|
+
}
|
|
39
57
|
/** Live-only mode hint from rubric — not part of the user scenario prompt. */
|
|
40
58
|
export declare function outputContractForRubric(rubric: ScenarioRubric): RoutingContract | undefined;
|
|
41
59
|
export declare function runSuite(options: RunSuiteOptions): Promise<SuiteRunReport>;
|
|
60
|
+
/** Run one real agent scenario. JSON suites delegate to this same execution boundary. */
|
|
61
|
+
export declare function runAgentTest(options: RunAgentTestOptions): Promise<ScenarioResult>;
|
|
42
62
|
export declare function runAllSuites(options: {
|
|
43
63
|
cwd: string;
|
|
44
64
|
suitesDir: string;
|
|
45
65
|
host?: AgentHost;
|
|
46
66
|
filter?: string;
|
|
47
67
|
scenarioFilter?: string;
|
|
48
|
-
record?: boolean;
|
|
49
|
-
recordFixtures?: boolean;
|
|
50
68
|
judge?: boolean;
|
|
51
69
|
worktree?: boolean;
|
|
52
70
|
stagingSessionId?: string;
|
package/dist/run-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACX,SAAS,EAIT,eAAe,EAEf,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACX,SAAS,EAIT,eAAe,EAEf,MAAM,2BAA2B,CAAC;AA+DnC,OAAO,KAAK,EACX,aAAa,EACb,kBAAkB,EAIlB,cAAc,EACd,cAAc,EACd,cAAc,EACd,MAAM,YAAY,CAAC;AA+BpB,0FAA0F;AAC1F,wBAAgB,sBAAsB,IAAI,OAAO,CAEhD;AAED,sGAAsG;AACtG,wBAAgB,uBAAuB,IAAI,IAAI,CAgC9C;AASD,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wDAAwD;IACxD,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,sDAAsD;IACtD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sFAAsF;IACtF,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,aAAa,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,+DAA+D;IAC/D,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oFAAoF;IACpF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,8EAA8E;AAC9E,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,GAAG,SAAS,CAQ3F;AAmMD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAoBhF;AAgQD,yFAAyF;AACzF,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,CAAC,CAexF;AAsfD,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAiC5B;AAED,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/run-suite.js
CHANGED
|
@@ -9,12 +9,13 @@ import { failuresForLiveSubprocessExit, killActiveLiveChildren, liveScenarioIsol
|
|
|
9
9
|
import { resolveLiveTimeoutMs } from "./live-timeout.js";
|
|
10
10
|
import { loadSuiteFile } from "./load-suite.js";
|
|
11
11
|
import { formatDuration, logPhase, logProgress, logVerdict, withHeartbeat } from "./progress.js";
|
|
12
|
-
import { getLiveStagingRootOverride, getLiveStagingSessionRoot, getStagingAgentStartPath, getStagingResultPath, getStagingTracePath, loadStagingResult, loadStagingTrace, recordTrace, resolveRecordingPath, setLiveStagingRootOverride, writeAgentStartMarker, writeStagingResult, } from "./record-trace.js";
|
|
12
|
+
import { cleanupStagingSession, createLiveStagingSessionId, getLiveStagingRootOverride, getLiveStagingSessionRoot, getStagingAgentStartPath, getStagingResultPath, getStagingTracePath, loadStagingResult, loadStagingTrace, recordTrace, resolveRecordingPath, setLiveStagingRootOverride, writeAgentStartMarker, writeStagingResult, } from "./record-trace.js";
|
|
13
13
|
import { resolveScenarioRetryMaxAttempts, shouldRetryAnnounceStopFlake } from "./scenario-retry.js";
|
|
14
14
|
import { captureCallerHead, restoreCallerHeadIfSeedCommit, seedScenarioWorktree, } from "./scenario-seed.js";
|
|
15
15
|
import { buildScenarioResultUsage, totalTokensFromScenarioUsage } from "./scenario-usage.js";
|
|
16
16
|
import { summarizeReportResults } from "./suite-summary.js";
|
|
17
17
|
import { theme } from "./theme.js";
|
|
18
|
+
import { validateSuiteFile } from "./validate-suite.js";
|
|
18
19
|
const require = createRequire(import.meta.url);
|
|
19
20
|
const packageVersion = require("../package.json").version;
|
|
20
21
|
let activeWorktreeCleanup;
|
|
@@ -170,7 +171,7 @@ async function maybeWriteDebugBundle(options) {
|
|
|
170
171
|
host: options.host,
|
|
171
172
|
timeoutMs: options.timeoutMs,
|
|
172
173
|
worktree: options.worktree,
|
|
173
|
-
isolateLive:
|
|
174
|
+
isolateLive: liveScenarioIsolationEnabled(),
|
|
174
175
|
}),
|
|
175
176
|
rerun: {
|
|
176
177
|
cliPath,
|
|
@@ -179,7 +180,6 @@ async function maybeWriteDebugBundle(options) {
|
|
|
179
180
|
rubricsDir: options.rubricsDir,
|
|
180
181
|
suite: options.suiteName,
|
|
181
182
|
scenario: options.scenario.name,
|
|
182
|
-
live: options.live,
|
|
183
183
|
host: options.host,
|
|
184
184
|
judge: options.judge,
|
|
185
185
|
worktree: options.worktree,
|
|
@@ -210,10 +210,15 @@ export async function runSuite(options) {
|
|
|
210
210
|
if (options.debugDir !== undefined) {
|
|
211
211
|
setLiveStagingRootOverride(options.debugDir);
|
|
212
212
|
}
|
|
213
|
+
const ownsStagingSession = options.stagingSessionId === undefined;
|
|
214
|
+
const stagingSessionId = options.stagingSessionId ?? createLiveStagingSessionId();
|
|
213
215
|
try {
|
|
214
|
-
return await runSuiteBody(options);
|
|
216
|
+
return await runSuiteBody({ ...options, stagingSessionId });
|
|
215
217
|
}
|
|
216
218
|
finally {
|
|
219
|
+
if (ownsStagingSession && !options.keepRecordings && !options.debug) {
|
|
220
|
+
await cleanupStagingSession(getLiveStagingSessionRoot(stagingSessionId)).catch(() => undefined);
|
|
221
|
+
}
|
|
217
222
|
if (options.debugDir !== undefined) {
|
|
218
223
|
setLiveStagingRootOverride(previousStagingRoot);
|
|
219
224
|
}
|
|
@@ -221,7 +226,14 @@ export async function runSuite(options) {
|
|
|
221
226
|
}
|
|
222
227
|
async function runSuiteBody(options) {
|
|
223
228
|
const suite = await loadSuiteFile(options.suitePath, { rubricsDir: options.rubricsDir });
|
|
224
|
-
const
|
|
229
|
+
const validationIssues = validateSuiteFile(options.suitePath, suite);
|
|
230
|
+
if (validationIssues.length > 0) {
|
|
231
|
+
const details = validationIssues
|
|
232
|
+
.map((issue) => `${issue.scenario ? `${issue.scenario} · ` : ""}${issue.field}: ${issue.message}`)
|
|
233
|
+
.join("\n");
|
|
234
|
+
throw new Error(`Invalid suite file ${options.suitePath}:\n${details}`);
|
|
235
|
+
}
|
|
236
|
+
const defaultHost = options.host ?? suite.defaults?.host ?? "cursor";
|
|
225
237
|
const results = [];
|
|
226
238
|
const scenarios = options.scenarioFilter
|
|
227
239
|
? suite.scenarios.filter((scenario) => scenario.name === options.scenarioFilter)
|
|
@@ -232,8 +244,7 @@ async function runSuiteBody(options) {
|
|
|
232
244
|
const filteredTotal = scenarios.length;
|
|
233
245
|
const parentCounters = parentScenarioCounters();
|
|
234
246
|
const displayTotal = parentCounters?.total ?? filteredTotal;
|
|
235
|
-
const
|
|
236
|
-
const isolateLive = isLiveSuite && liveScenarioIsolationEnabled() && !options.scenarioFilter && filteredTotal > 1;
|
|
247
|
+
const isolateLive = liveScenarioIsolationEnabled() && !options.scenarioFilter && filteredTotal > 1;
|
|
237
248
|
let previousIsolatedExitCode;
|
|
238
249
|
if (shouldPrintSuiteChrome()) {
|
|
239
250
|
logProgress(`\n${theme.suiteHeader(suite.name, defaultHost, displayTotal)}`);
|
|
@@ -264,7 +275,7 @@ async function runSuiteBody(options) {
|
|
|
264
275
|
}
|
|
265
276
|
const started = performance.now();
|
|
266
277
|
const debug = isDebugEnabled(options);
|
|
267
|
-
const maxAttempts = !isChildProcess()
|
|
278
|
+
const maxAttempts = !isChildProcess()
|
|
268
279
|
? resolveScenarioRetryMaxAttempts(options.scenarioRetries)
|
|
269
280
|
: 1;
|
|
270
281
|
let attempts = 0;
|
|
@@ -282,7 +293,6 @@ async function runSuiteBody(options) {
|
|
|
282
293
|
suiteFilter: options.suiteFilter ?? suite.name,
|
|
283
294
|
stagingSessionId: options.stagingSessionId,
|
|
284
295
|
keepRecordings: options.keepRecordings,
|
|
285
|
-
recordFixtures: options.recordFixtures,
|
|
286
296
|
worktree: options.worktree,
|
|
287
297
|
judge: options.judge,
|
|
288
298
|
host: defaultHost,
|
|
@@ -373,7 +383,6 @@ async function runSuiteBody(options) {
|
|
|
373
383
|
timeoutMs: options.timeoutMs,
|
|
374
384
|
worktree: options.worktree,
|
|
375
385
|
judge: options.judge,
|
|
376
|
-
live: true,
|
|
377
386
|
allowUserInput: options.allowUserInput,
|
|
378
387
|
keepRecordings: options.keepRecordings,
|
|
379
388
|
});
|
|
@@ -394,67 +403,27 @@ async function runSuiteBody(options) {
|
|
|
394
403
|
releaseLiveMemory();
|
|
395
404
|
continue;
|
|
396
405
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
:
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
const debugBundleDir = await maybeWriteDebugBundle({
|
|
419
|
-
debug,
|
|
420
|
-
cwd: options.cwd,
|
|
421
|
-
suitesDir: options.suitesDir ?? "agent-suites",
|
|
422
|
-
rubricsDir: options.rubricsDir,
|
|
423
|
-
stagingSessionId: options.stagingSessionId,
|
|
424
|
-
debugDir: options.debugDir,
|
|
425
|
-
suiteName: suite.name,
|
|
426
|
-
scenario,
|
|
427
|
-
host: scenario.host ?? defaultHost,
|
|
428
|
-
result: scenarioResult,
|
|
429
|
-
trace: scenarioResult.trace,
|
|
430
|
-
timeoutMs: options.timeoutMs,
|
|
431
|
-
worktree: options.worktree,
|
|
432
|
-
judge: options.judge,
|
|
433
|
-
live: true,
|
|
434
|
-
allowUserInput: options.allowUserInput,
|
|
435
|
-
keepRecordings: options.keepRecordings,
|
|
436
|
-
});
|
|
437
|
-
scenarioResult.debugBundleDir = debugBundleDir;
|
|
438
|
-
emitScenarioVerdict({
|
|
439
|
-
passed: scenarioResult.passed,
|
|
440
|
-
index: scenarioIndex,
|
|
441
|
-
total: scenarioTotal,
|
|
442
|
-
name: scenario.name,
|
|
443
|
-
durationMs: scenarioResult.durationMs,
|
|
444
|
-
totalTokens: totalTokensFromScenarioUsage(scenarioResult.usage, scenarioResult.trace?.usage),
|
|
445
|
-
judgeVerdicts: scenarioResult.judgeVerdicts,
|
|
446
|
-
failures: scenarioResult.failures,
|
|
447
|
-
debug,
|
|
448
|
-
debugBundleDir,
|
|
449
|
-
});
|
|
450
|
-
results.push(scenarioResult);
|
|
451
|
-
}
|
|
452
|
-
else {
|
|
453
|
-
results.push(await runScenario(options.cwd, suite.name, scenario, defaultHost, suite.defaults?.profile, suite.defaults?.skills, suite.defaults?.contextSources, suite.defaults?.mcpServers, options.record, options.recordFixtures, options.judge, options.worktree, options.stagingSessionId, scenarioIndex, scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir));
|
|
454
|
-
}
|
|
455
|
-
if (isLiveSuite) {
|
|
456
|
-
releaseLiveMemory();
|
|
457
|
-
}
|
|
406
|
+
results.push(await runAgentTest({
|
|
407
|
+
cwd: options.cwd,
|
|
408
|
+
suiteName: suite.name,
|
|
409
|
+
scenario,
|
|
410
|
+
defaults: suite.defaults,
|
|
411
|
+
host: options.host,
|
|
412
|
+
judge: options.judge,
|
|
413
|
+
worktree: options.worktree,
|
|
414
|
+
stagingSessionId: options.stagingSessionId,
|
|
415
|
+
scenarioIndex,
|
|
416
|
+
scenarioTotal,
|
|
417
|
+
timeoutMs: options.timeoutMs,
|
|
418
|
+
allowUserInput: options.allowUserInput,
|
|
419
|
+
debug: options.debug,
|
|
420
|
+
debugDir: options.debugDir,
|
|
421
|
+
suitesDir: options.suitesDir,
|
|
422
|
+
keepRecordings: options.keepRecordings,
|
|
423
|
+
rubricsDir: options.rubricsDir,
|
|
424
|
+
scenarioRetries: isChildProcess() ? 0 : options.scenarioRetries,
|
|
425
|
+
}));
|
|
426
|
+
releaseLiveMemory();
|
|
458
427
|
}
|
|
459
428
|
return {
|
|
460
429
|
suite: suite.name,
|
|
@@ -470,7 +439,86 @@ function mergeContextSources(defaults, scenarioSources) {
|
|
|
470
439
|
const merged = [...(defaults ?? []), ...(scenarioSources ?? [])].filter((value) => typeof value === "string" && value.trim().length > 0);
|
|
471
440
|
return merged.length > 0 ? merged : undefined;
|
|
472
441
|
}
|
|
473
|
-
|
|
442
|
+
/** Run one real agent scenario. JSON suites delegate to this same execution boundary. */
|
|
443
|
+
export async function runAgentTest(options) {
|
|
444
|
+
const previousStagingRoot = getLiveStagingRootOverride();
|
|
445
|
+
if (options.debugDir !== undefined) {
|
|
446
|
+
setLiveStagingRootOverride(options.debugDir);
|
|
447
|
+
}
|
|
448
|
+
const stagingSessionId = options.stagingSessionId ??
|
|
449
|
+
(options.debug || options.keepRecordings ? createLiveStagingSessionId() : undefined);
|
|
450
|
+
try {
|
|
451
|
+
return await runAgentTestBody({ ...options, stagingSessionId });
|
|
452
|
+
}
|
|
453
|
+
finally {
|
|
454
|
+
if (options.debugDir !== undefined) {
|
|
455
|
+
setLiveStagingRootOverride(previousStagingRoot);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
async function runAgentTestBody(options) {
|
|
460
|
+
const legacyScenario = options.scenario;
|
|
461
|
+
const legacyDefaults = options.defaults;
|
|
462
|
+
if (legacyScenario.host === "replay" ||
|
|
463
|
+
legacyDefaults?.host === "replay" ||
|
|
464
|
+
"replayTrace" in legacyScenario) {
|
|
465
|
+
throw new Error("Replay-based testing is deprecated and no longer supported; use Cursor or Claude.");
|
|
466
|
+
}
|
|
467
|
+
const suiteName = options.suiteName ?? "direct";
|
|
468
|
+
const defaultHost = options.host ?? options.defaults?.host ?? "cursor";
|
|
469
|
+
const maxAttempts = resolveScenarioRetryMaxAttempts(options.scenarioRetries);
|
|
470
|
+
let attempts = 0;
|
|
471
|
+
let result;
|
|
472
|
+
while (true) {
|
|
473
|
+
attempts++;
|
|
474
|
+
result = await runAgentTestOnce(options.cwd, suiteName, options.scenario, defaultHost, options.defaults?.profile, options.defaults?.skills, options.defaults?.contextSources, options.defaults?.mcpServers, options.judge ?? true, options.worktree ?? true, options.stagingSessionId, options.scenarioIndex, options.scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir, { suppressEmit: maxAttempts > 1 });
|
|
475
|
+
const canRetry = !result.skipped &&
|
|
476
|
+
!result.passed &&
|
|
477
|
+
attempts < maxAttempts &&
|
|
478
|
+
shouldRetryAnnounceStopFlake(result.failures, result.trace);
|
|
479
|
+
if (!canRetry) {
|
|
480
|
+
break;
|
|
481
|
+
}
|
|
482
|
+
logPhase(theme.phase("retry", `${attempts}/${maxAttempts - 1}`));
|
|
483
|
+
}
|
|
484
|
+
result.attempts = attempts;
|
|
485
|
+
if (maxAttempts > 1) {
|
|
486
|
+
const debug = isDebugEnabled(options);
|
|
487
|
+
const debugBundleDir = await maybeWriteDebugBundle({
|
|
488
|
+
debug,
|
|
489
|
+
cwd: options.cwd,
|
|
490
|
+
suitesDir: options.suitesDir ?? "agent-suites",
|
|
491
|
+
rubricsDir: options.rubricsDir,
|
|
492
|
+
stagingSessionId: options.stagingSessionId,
|
|
493
|
+
debugDir: options.debugDir,
|
|
494
|
+
suiteName,
|
|
495
|
+
scenario: options.scenario,
|
|
496
|
+
host: options.scenario.host ?? defaultHost,
|
|
497
|
+
result,
|
|
498
|
+
trace: result.trace,
|
|
499
|
+
timeoutMs: options.timeoutMs,
|
|
500
|
+
worktree: options.worktree,
|
|
501
|
+
judge: options.judge ?? true,
|
|
502
|
+
allowUserInput: options.allowUserInput,
|
|
503
|
+
keepRecordings: options.keepRecordings,
|
|
504
|
+
});
|
|
505
|
+
result.debugBundleDir = debugBundleDir;
|
|
506
|
+
emitScenarioVerdict({
|
|
507
|
+
passed: result.passed,
|
|
508
|
+
index: options.scenarioIndex,
|
|
509
|
+
total: options.scenarioTotal,
|
|
510
|
+
name: options.scenario.name,
|
|
511
|
+
durationMs: result.durationMs,
|
|
512
|
+
totalTokens: totalTokensFromScenarioUsage(result.usage, result.trace?.usage),
|
|
513
|
+
judgeVerdicts: result.judgeVerdicts,
|
|
514
|
+
failures: result.failures,
|
|
515
|
+
debug,
|
|
516
|
+
debugBundleDir,
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
return result;
|
|
520
|
+
}
|
|
521
|
+
async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir = "agent-suites", keepRecordings, rubricsDir, runOptions) {
|
|
474
522
|
const started = performance.now();
|
|
475
523
|
const debug = isDebugEnabled({ debug: debugFlag });
|
|
476
524
|
const suppressEmit = runOptions?.suppressEmit === true;
|
|
@@ -493,8 +541,7 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
493
541
|
const skills = scenario.skills ?? defaultSkills;
|
|
494
542
|
const contextSources = mergeContextSources(defaultContextSources, scenario.contextSources);
|
|
495
543
|
const mcpServers = mergeMcpServers(defaultMcpServers, scenario.mcpServers);
|
|
496
|
-
const
|
|
497
|
-
const liveTimeoutMs = isLive ? resolveLiveTimeoutMs(timeoutMs) : undefined;
|
|
544
|
+
const liveTimeoutMs = resolveLiveTimeoutMs(timeoutMs);
|
|
498
545
|
const failOnUserInput = !allowUserInput;
|
|
499
546
|
if (scenarioIndex !== undefined && scenarioTotal !== undefined) {
|
|
500
547
|
logProgress(theme.scenarioTitle(scenarioIndex, scenarioTotal, scenario.name, host));
|
|
@@ -502,79 +549,61 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
502
549
|
else {
|
|
503
550
|
logProgress(theme.scenarioLabel(scenario.name, host));
|
|
504
551
|
}
|
|
505
|
-
const useWorktree =
|
|
552
|
+
const useWorktree = worktree !== false && !process.env.AGENT_TEST_NO_WORKTREE;
|
|
506
553
|
let worktreeHandle;
|
|
507
554
|
let callerHeadBefore;
|
|
508
555
|
const callerTreeBefore = useWorktree ? await captureWorkingTreeStatus(cwd) : undefined;
|
|
509
556
|
if (useWorktree) {
|
|
510
|
-
if (
|
|
557
|
+
if (scenario.seedPatch) {
|
|
511
558
|
callerHeadBefore = await captureCallerHead(cwd);
|
|
512
559
|
setCallerHeadRestore(cwd, callerHeadBefore);
|
|
513
560
|
}
|
|
514
561
|
worktreeHandle = await createScenarioWorktree(cwd, `${suiteName}-${scenario.name}`);
|
|
515
562
|
activeWorktreeCleanup = worktreeHandle.cleanup;
|
|
516
563
|
logPhase(theme.phase("worktree", theme.path(worktreeHandle.path)));
|
|
517
|
-
if (
|
|
564
|
+
if (scenario.seedPatch) {
|
|
518
565
|
logPhase(theme.phase("seed", theme.basename(scenario.seedPatch)));
|
|
519
566
|
await seedScenarioWorktree(cwd, worktreeHandle.path, scenario.seedPatch, {
|
|
520
567
|
stageOnly: scenario.seedStageOnly === true,
|
|
521
568
|
});
|
|
522
569
|
}
|
|
523
570
|
}
|
|
524
|
-
else
|
|
571
|
+
else {
|
|
525
572
|
logPhase(theme.phase("worktree", theme.phaseDim("disabled (AGENT_TEST_ALLOW_IN_PLACE=1)")));
|
|
526
573
|
}
|
|
527
574
|
const runCwd = worktreeHandle?.path ?? cwd;
|
|
528
575
|
try {
|
|
529
576
|
logPhase(theme.phase("context"));
|
|
530
|
-
//
|
|
577
|
+
// Worktree runs code in an isolated checkout; load rules/AGENTS from caller cwd
|
|
531
578
|
// so uncommitted .cursor/rules and AGENTS.md edits apply during dogfood.
|
|
532
|
-
const contextRoot =
|
|
579
|
+
const contextRoot = useWorktree ? cwd : runCwd;
|
|
533
580
|
const context = await loadContext({
|
|
534
581
|
cwd: contextRoot,
|
|
535
582
|
profile,
|
|
536
583
|
skills,
|
|
537
584
|
contextSources,
|
|
538
585
|
});
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
}
|
|
543
|
-
else {
|
|
544
|
-
logPhase(theme.phase("agent"));
|
|
545
|
-
}
|
|
546
|
-
const outputContract = isLive ? outputContractForRubric(scenario.rubric) : undefined;
|
|
547
|
-
const agentStartMarkerPath = isChildProcess() && isLive && stagingSessionId
|
|
586
|
+
logPhase(theme.phase("agent"));
|
|
587
|
+
const outputContract = outputContractForRubric(scenario.rubric);
|
|
588
|
+
const agentStartMarkerPath = isChildProcess() && stagingSessionId
|
|
548
589
|
? getStagingAgentStartPath(stagingSessionId, suiteName, scenario.name)
|
|
549
590
|
: undefined;
|
|
550
591
|
const agentStarted = performance.now();
|
|
551
|
-
const session = await (
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
: runAgent({
|
|
567
|
-
host,
|
|
568
|
-
cwd: runCwd,
|
|
569
|
-
context,
|
|
570
|
-
profile,
|
|
571
|
-
prompt: scenario.prompt,
|
|
572
|
-
replayTracePath: useReplay ? scenario.replayTrace : undefined,
|
|
573
|
-
mcpServers,
|
|
574
|
-
}));
|
|
575
|
-
if (isLive) {
|
|
576
|
-
logPhase(theme.phase("agent", `${theme.statusCompleted(session.status)} ${theme.duration(formatDuration(performance.now() - agentStarted))}`));
|
|
577
|
-
}
|
|
592
|
+
const session = await withHeartbeat(runAgent({
|
|
593
|
+
host,
|
|
594
|
+
cwd: runCwd,
|
|
595
|
+
context,
|
|
596
|
+
profile,
|
|
597
|
+
prompt: scenario.prompt,
|
|
598
|
+
outputContract,
|
|
599
|
+
mcpServers,
|
|
600
|
+
timeoutMs: liveTimeoutMs,
|
|
601
|
+
failOnUserInput,
|
|
602
|
+
onDeadlineStart: agentStartMarkerPath
|
|
603
|
+
? () => writeAgentStartMarker(agentStartMarkerPath)
|
|
604
|
+
: undefined,
|
|
605
|
+
}), { started: agentStarted });
|
|
606
|
+
logPhase(theme.phase("agent", `${theme.statusCompleted(session.status)} ${theme.duration(formatDuration(performance.now() - agentStarted))}`));
|
|
578
607
|
let trace = enrichTrace(session.trace);
|
|
579
608
|
const failures = [];
|
|
580
609
|
if (session.status !== "completed") {
|
|
@@ -597,7 +626,7 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
597
626
|
.join("\n");
|
|
598
627
|
failures.push(assertionFailure("runAgent", session.error ?? `agent session ${session.status}`, "agent_runtime", runtimeEvidence));
|
|
599
628
|
}
|
|
600
|
-
else if (
|
|
629
|
+
else if (failOnUserInput && traceHasUserInputTool(trace.toolCalls)) {
|
|
601
630
|
failures.push(assertionFailure("runAgent", "agent trace contains AskQuestion-style user-input tool in headless mode", "agent_runtime", `toolCalls=${trace.toolCalls.map((call) => call.name).join(", ")}`));
|
|
602
631
|
}
|
|
603
632
|
logPhase(theme.phase("rubric"));
|
|
@@ -608,7 +637,7 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
608
637
|
const callerTreeAfter = await captureWorkingTreeStatus(cwd);
|
|
609
638
|
const ignoreRoots = resolveHarnessArtifactIgnoreRoots(cwd, getLiveStagingRootOverride());
|
|
610
639
|
const leaked = filterWorkingTreeLeaks(findWorkingTreeLeak(callerTreeBefore, callerTreeAfter), ignoreRoots, cwd);
|
|
611
|
-
const seedPaths =
|
|
640
|
+
const seedPaths = scenario.seedPatch
|
|
612
641
|
? await loadUnifiedDiffPaths(resolve(cwd, scenario.seedPatch)).catch(() => [])
|
|
613
642
|
: [];
|
|
614
643
|
const outsideEdits = worktreeHandle !== undefined
|
|
@@ -634,22 +663,19 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
634
663
|
].join("\n")));
|
|
635
664
|
}
|
|
636
665
|
}
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
catch (error) {
|
|
646
|
-
failures.push(assertionFailure("recordTrace", error instanceof Error ? error.message : "failed to record trace", "recording_error"));
|
|
647
|
-
}
|
|
666
|
+
const stagingTracePath = resolveRecordingPath(suiteName, scenario.name, stagingSessionId);
|
|
667
|
+
if (stagingTracePath) {
|
|
668
|
+
try {
|
|
669
|
+
const path = await recordTrace(stagingTracePath, trace);
|
|
670
|
+
logPhase(theme.phase("trace", theme.path(path)));
|
|
671
|
+
}
|
|
672
|
+
catch (error) {
|
|
673
|
+
failures.push(assertionFailure("recordTrace", error instanceof Error ? error.message : "failed to record trace", "recording_error"));
|
|
648
674
|
}
|
|
649
675
|
}
|
|
650
676
|
const deferJudgeToParent = isChildProcess();
|
|
651
677
|
let judgeVerdicts;
|
|
652
|
-
if (judge &&
|
|
678
|
+
if (judge && !deferJudgeToParent) {
|
|
653
679
|
const criteria = normalizeJudgeCriteria(scenario.rubric.judge);
|
|
654
680
|
if (criteria.length > 0) {
|
|
655
681
|
logPhase(theme.judgePhase(criteria.length), { last: true });
|
|
@@ -660,7 +686,7 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
660
686
|
judgeVerdicts = toJudgeVerdictResults(judged.trace, criteria, judged.verdicts);
|
|
661
687
|
}
|
|
662
688
|
const durationMs = Math.round(performance.now() - started);
|
|
663
|
-
if (isChildProcess() &&
|
|
689
|
+
if (isChildProcess() && stagingSessionId) {
|
|
664
690
|
await writeStagingResult(getStagingResultPath(stagingSessionId, suiteName, scenario.name), {
|
|
665
691
|
passed: failures.length === 0,
|
|
666
692
|
failures,
|
|
@@ -669,7 +695,6 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
669
695
|
}
|
|
670
696
|
if (worktreeHandle) {
|
|
671
697
|
const willJudge = Boolean(judge) &&
|
|
672
|
-
isLive &&
|
|
673
698
|
!isChildProcess() &&
|
|
674
699
|
normalizeJudgeCriteria(scenario.rubric.judge).length > 0;
|
|
675
700
|
logPhase(theme.phase("cleanup"), { last: !willJudge });
|
|
@@ -712,7 +737,6 @@ async function runScenario(cwd, suiteName, scenario, defaultHost, defaultProfile
|
|
|
712
737
|
timeoutMs,
|
|
713
738
|
worktree,
|
|
714
739
|
judge,
|
|
715
|
-
live: isLive,
|
|
716
740
|
allowUserInput,
|
|
717
741
|
keepRecordings,
|
|
718
742
|
});
|
|
@@ -803,8 +827,6 @@ export async function runAllSuites(options) {
|
|
|
803
827
|
suitePath,
|
|
804
828
|
host: options.host,
|
|
805
829
|
scenarioFilter: options.scenarioFilter,
|
|
806
|
-
record: options.record,
|
|
807
|
-
recordFixtures: options.recordFixtures,
|
|
808
830
|
judge: options.judge,
|
|
809
831
|
worktree: options.worktree,
|
|
810
832
|
stagingSessionId: options.stagingSessionId,
|