@post-print/agent-test 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -157
- package/dist/check.d.ts +38 -0
- package/dist/check.d.ts.map +1 -0
- package/dist/check.js +169 -0
- package/dist/check.js.map +1 -0
- package/dist/cli.d.ts +12 -14
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +342 -366
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +33 -0
- package/dist/compare-scenario.d.ts.map +1 -0
- package/dist/compare-scenario.js +128 -0
- package/dist/compare-scenario.js.map +1 -0
- package/dist/debug-bundle.d.ts +2 -2
- package/dist/debug-bundle.d.ts.map +1 -1
- package/dist/debug-bundle.js +12 -12
- package/dist/debug-bundle.js.map +1 -1
- package/dist/doctor.d.ts +6 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +136 -14
- package/dist/doctor.js.map +1 -1
- package/dist/expect.d.ts +6 -5
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +67 -57
- package/dist/expect.js.map +1 -1
- package/dist/host-log.d.ts +17 -0
- package/dist/host-log.d.ts.map +1 -0
- package/dist/host-log.js +154 -0
- package/dist/host-log.js.map +1 -0
- package/dist/hosts.d.ts +22 -0
- package/dist/hosts.d.ts.map +1 -0
- package/dist/hosts.js +84 -0
- package/dist/hosts.js.map +1 -0
- package/dist/html-report.d.ts +0 -13
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +381 -318
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +15 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -7
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +11 -5
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +32 -14
- package/dist/live-isolation.js.map +1 -1
- package/dist/load-adapters.d.ts +13 -0
- package/dist/load-adapters.d.ts.map +1 -0
- package/dist/load-adapters.js +76 -0
- package/dist/load-adapters.js.map +1 -0
- package/dist/load-suite.d.ts.map +1 -1
- package/dist/load-suite.js +16 -0
- package/dist/load-suite.js.map +1 -1
- package/dist/mcp-config.d.ts +8 -0
- package/dist/mcp-config.d.ts.map +1 -1
- package/dist/mcp-config.js +46 -0
- package/dist/mcp-config.js.map +1 -1
- package/dist/preflight.d.ts +2 -2
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js +4 -4
- package/dist/preflight.js.map +1 -1
- package/dist/progress.d.ts +7 -0
- package/dist/progress.d.ts.map +1 -1
- package/dist/progress.js +39 -9
- package/dist/progress.js.map +1 -1
- package/dist/record-trace.d.ts +11 -20
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js +10 -57
- package/dist/record-trace.js.map +1 -1
- package/dist/report-preview-server.d.ts +2 -0
- package/dist/report-preview-server.d.ts.map +1 -0
- package/dist/report-preview-server.js +24 -0
- package/dist/report-preview-server.js.map +1 -0
- package/dist/report-preview.d.ts +11 -0
- package/dist/report-preview.d.ts.map +1 -0
- package/dist/report-preview.js +161 -0
- package/dist/report-preview.js.map +1 -0
- package/dist/run-suite.d.ts +62 -11
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +697 -206
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-story.d.ts +41 -0
- package/dist/scenario-story.d.ts.map +1 -0
- package/dist/scenario-story.js +198 -0
- package/dist/scenario-story.js.map +1 -0
- package/dist/suite-summary.d.ts +2 -0
- package/dist/suite-summary.d.ts.map +1 -1
- package/dist/suite-summary.js +70 -22
- package/dist/suite-summary.js.map +1 -1
- package/dist/theme.d.ts +29 -5
- package/dist/theme.d.ts.map +1 -1
- package/dist/theme.js +204 -45
- package/dist/theme.js.map +1 -1
- package/dist/types.d.ts +107 -24
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-seeds.d.ts.map +1 -1
- package/dist/validate-seeds.js +21 -6
- package/dist/validate-seeds.js.map +1 -1
- package/dist/validate-suite.d.ts +3 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +256 -22
- package/dist/validate-suite.js.map +1 -1
- package/package.json +5 -5
- package/dist/compare.d.ts +0 -79
- package/dist/compare.d.ts.map +0 -1
- package/dist/compare.js +0 -232
- package/dist/compare.js.map +0 -1
package/dist/types.d.ts
CHANGED
|
@@ -8,48 +8,106 @@ export interface ScenarioRubric {
|
|
|
8
8
|
tier?: "low" | "medium" | "high";
|
|
9
9
|
/** Chat-style tier announce — infer one-line routing from transcript when routing.tier absent. */
|
|
10
10
|
handsOnRouting?: boolean;
|
|
11
|
+
/** Substring in assistant text, commands, artifacts, or tool args and results. */
|
|
11
12
|
must?: string[];
|
|
13
|
+
/** Substring that must not appear in assistant text, commands, artifacts, or tool args. Tool results are ignored. */
|
|
12
14
|
mustNot?: string[];
|
|
13
15
|
mustRun?: string[];
|
|
14
16
|
/**
|
|
15
|
-
* Tool name substring, or `name:
|
|
16
|
-
* Matches built-in and MCP tool calls
|
|
17
|
+
* Tool name substring, or `name:fragment` where fragment must appear in
|
|
18
|
+
* JSON args or the tool result. Matches built-in and MCP tool calls.
|
|
17
19
|
*/
|
|
18
20
|
mustCallTool?: string[];
|
|
19
21
|
mustNotCallTool?: string[];
|
|
20
22
|
/**
|
|
21
|
-
* Substring that must appear in
|
|
23
|
+
* Substring that must appear in Read-family args or a Shell/Bash path access.
|
|
22
24
|
* Hallucination scoring stays in live `judge` questions — these are lightweight proxies.
|
|
23
25
|
*/
|
|
24
26
|
mustReadPath?: string[];
|
|
25
27
|
/** Substring that must not appear in any Read tool's JSON args. */
|
|
26
28
|
mustNotReadPath?: string[];
|
|
27
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Skill folder names. Deterministic match is a SKILL.md / references read
|
|
31
|
+
* on a host skill path. Live judge then scores whether the agent followed it.
|
|
32
|
+
*/
|
|
28
33
|
mustInvokeSkill?: string[];
|
|
29
34
|
mustNotInvokeSkill?: string[];
|
|
30
35
|
routingBlock?: boolean;
|
|
31
36
|
reviewDepth?: "quick" | "standard" | "thorough" | "full";
|
|
32
|
-
/**
|
|
37
|
+
/** Any transcript outcome. The judge sees assistant text, tool args, and tool results. */
|
|
33
38
|
judge?: JudgeRubricItem[];
|
|
34
39
|
}
|
|
40
|
+
export type CompareArmId = "a" | "b";
|
|
41
|
+
/** Overrides for one side of a compare scenario. Omitted fields inherit the scenario. */
|
|
42
|
+
export interface CompareArm {
|
|
43
|
+
label?: string;
|
|
44
|
+
prompt?: string;
|
|
45
|
+
host?: AgentHost;
|
|
46
|
+
profile?: ContextProfile;
|
|
47
|
+
workspace?: string;
|
|
48
|
+
skills?: SkillContextSetting;
|
|
49
|
+
contextSources?: string[];
|
|
50
|
+
mcpServers?: Record<string, McpServerConfig>;
|
|
51
|
+
allowUserSkills?: boolean;
|
|
52
|
+
seedPatch?: string;
|
|
53
|
+
seedStageOnly?: boolean;
|
|
54
|
+
/**
|
|
55
|
+
* Extra deterministic checks for this arm only.
|
|
56
|
+
* Arrays append onto the scenario rubric. Do not set judge here.
|
|
57
|
+
*/
|
|
58
|
+
rubric?: ScenarioRubric;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Two live arms in one scenario.
|
|
62
|
+
* `faster` and `cheaper` are optional metric gates.
|
|
63
|
+
* `rubric.judge` is optional and scores both transcripts.
|
|
64
|
+
*/
|
|
65
|
+
export interface ScenarioCompare {
|
|
66
|
+
a: CompareArm;
|
|
67
|
+
b: CompareArm;
|
|
68
|
+
/** Named arm must have a shorter duration. */
|
|
69
|
+
faster?: CompareArmId;
|
|
70
|
+
/** Named arm must use fewer total tokens. */
|
|
71
|
+
cheaper?: CompareArmId;
|
|
72
|
+
}
|
|
73
|
+
export interface CompareArmResult {
|
|
74
|
+
id: CompareArmId;
|
|
75
|
+
label: string;
|
|
76
|
+
prompt: string;
|
|
77
|
+
trace?: AgentTrace;
|
|
78
|
+
durationMs?: number;
|
|
79
|
+
}
|
|
80
|
+
export interface ScenarioCompareResult {
|
|
81
|
+
a: CompareArmResult;
|
|
82
|
+
b: CompareArmResult;
|
|
83
|
+
}
|
|
35
84
|
export interface AgentScenario {
|
|
36
85
|
name: string;
|
|
37
|
-
/**
|
|
38
|
-
|
|
86
|
+
/** Run two arms. Optional metric gates and an optional pairwise judge. */
|
|
87
|
+
compare?: ScenarioCompare;
|
|
39
88
|
prompt: string;
|
|
40
89
|
host?: AgentHost;
|
|
41
90
|
profile?: ContextProfile;
|
|
42
|
-
/**
|
|
91
|
+
/** Extra skill folders to overlay. Project skills in the git repo load without this. */
|
|
43
92
|
skills?: SkillContextSetting;
|
|
44
93
|
/** Additive context paths (merged after suite defaults.contextSources). */
|
|
45
94
|
contextSources?: string[];
|
|
46
95
|
/** Inline MCP servers for live Cursor (merged over suite defaults by server name). */
|
|
47
96
|
mcpServers?: Record<string, McpServerConfig>;
|
|
97
|
+
/**
|
|
98
|
+
* Caller-relative folder that becomes the sealed repo.
|
|
99
|
+
* Omit or `"."` copies caller HEAD. A subfolder copies only that tree.
|
|
100
|
+
*/
|
|
101
|
+
workspace?: string;
|
|
102
|
+
/**
|
|
103
|
+
* Load host-global user skills from the developer machine.
|
|
104
|
+
* Default false. Keep false for a custom `workspace` fixture.
|
|
105
|
+
*/
|
|
106
|
+
allowUserSkills?: boolean;
|
|
48
107
|
/** Live-only: apply patch + commit in worktree so pr-mode branch diff exists. */
|
|
49
108
|
seedPatch?: string;
|
|
50
109
|
/** Live-only: with seedPatch, stage changes without committing (staged review mode). */
|
|
51
110
|
seedStageOnly?: boolean;
|
|
52
|
-
replayTrace?: string;
|
|
53
111
|
/**
|
|
54
112
|
* Assertions / judge criteria. May be omitted in scenarios.json when supplied via
|
|
55
113
|
* sibling `rubrics.json` / `scenarios.rubric.json` or `--rubrics-dir` (harness-only);
|
|
@@ -58,22 +116,35 @@ export interface AgentScenario {
|
|
|
58
116
|
rubric: ScenarioRubric;
|
|
59
117
|
skip?: boolean;
|
|
60
118
|
}
|
|
119
|
+
export interface AgentSuiteDefaults {
|
|
120
|
+
host?: AgentHost;
|
|
121
|
+
profile?: ContextProfile;
|
|
122
|
+
/** Extra skill folders to overlay, or `"none"`. */
|
|
123
|
+
skills?: SkillContextSetting;
|
|
124
|
+
/** Additive repo-relative context paths or `.skeleton/customize/` basenames. */
|
|
125
|
+
contextSources?: string[];
|
|
126
|
+
/** Inline MCP servers for direct agent runs. */
|
|
127
|
+
mcpServers?: Record<string, McpServerConfig>;
|
|
128
|
+
/**
|
|
129
|
+
* Caller-relative folder that becomes the sealed repo.
|
|
130
|
+
* Scenario `workspace` wins when both are set.
|
|
131
|
+
*/
|
|
132
|
+
workspace?: string;
|
|
133
|
+
/**
|
|
134
|
+
* Load host-global user skills from the developer machine.
|
|
135
|
+
* Default false. Scenario `allowUserSkills` wins when both are set.
|
|
136
|
+
*/
|
|
137
|
+
allowUserSkills?: boolean;
|
|
138
|
+
}
|
|
61
139
|
export interface AgentSuiteFile {
|
|
62
140
|
name: string;
|
|
63
141
|
description?: string;
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
* Additive repo-relative context paths (or `.skeleton/customize/` basenames).
|
|
71
|
-
* Use with `profile: "skeleton"` or to extend shared/cursor/claude profiles.
|
|
72
|
-
*/
|
|
73
|
-
contextSources?: string[];
|
|
74
|
-
/** Inline MCP servers for live Cursor runs. */
|
|
75
|
-
mcpServers?: Record<string, McpServerConfig>;
|
|
76
|
-
};
|
|
142
|
+
/**
|
|
143
|
+
* Host matrix for this suite (Playwright-style projects).
|
|
144
|
+
* When set, a run expands once per host. `--host` then filters the list.
|
|
145
|
+
*/
|
|
146
|
+
hosts?: AgentHost[];
|
|
147
|
+
defaults?: AgentSuiteDefaults;
|
|
77
148
|
scenarios: AgentScenario[];
|
|
78
149
|
}
|
|
79
150
|
export interface AssertionFailure {
|
|
@@ -102,11 +173,16 @@ export interface JudgeVerdictResult {
|
|
|
102
173
|
promptChars?: number;
|
|
103
174
|
usage?: AgentUsage;
|
|
104
175
|
}
|
|
176
|
+
/** CLI and HTML summary of rubric checks and what the run produced. */
|
|
177
|
+
export interface ScenarioStory {
|
|
178
|
+
criteria: string[];
|
|
179
|
+
result: string[];
|
|
180
|
+
/** Failures and judge notes. Shown under Result. */
|
|
181
|
+
verdict: string[];
|
|
182
|
+
}
|
|
105
183
|
export interface ScenarioResult {
|
|
106
184
|
suite: string;
|
|
107
185
|
scenario: string;
|
|
108
|
-
/** Stable compare key when scenario JSON defines compareId. */
|
|
109
|
-
compareId?: string;
|
|
110
186
|
passed: boolean;
|
|
111
187
|
failures: AssertionFailure[];
|
|
112
188
|
skipped?: boolean;
|
|
@@ -121,8 +197,14 @@ export interface ScenarioResult {
|
|
|
121
197
|
agentUsage?: AgentUsage;
|
|
122
198
|
/** Denormalized judge usage (sum of criteria). */
|
|
123
199
|
judgeUsage?: AgentUsage;
|
|
200
|
+
/** Scenario prompt sent to the host (for HTML conversation). */
|
|
201
|
+
prompt?: string;
|
|
124
202
|
/** Full agent transcript when available (for HTML reports / debug bundles). */
|
|
125
203
|
trace?: AgentTrace;
|
|
204
|
+
/** Both arms when this scenario is a compare run. */
|
|
205
|
+
compare?: ScenarioCompareResult;
|
|
206
|
+
/** Plain-language summary of the check, the agent run, and the verdict. */
|
|
207
|
+
story?: ScenarioStory;
|
|
126
208
|
/** Absolute path to the debug bundle directory when --debug wrote one. */
|
|
127
209
|
debugBundleDir?: string;
|
|
128
210
|
}
|
|
@@ -132,6 +214,7 @@ export interface UsageStats {
|
|
|
132
214
|
sumTotalTokens?: number;
|
|
133
215
|
p50TotalTokens?: number;
|
|
134
216
|
p95TotalTokens?: number;
|
|
217
|
+
maxTotalTokens?: number;
|
|
135
218
|
sumInputTokens?: number;
|
|
136
219
|
sumOutputTokens?: number;
|
|
137
220
|
}
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,kFAAkF;IAClF,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,qHAAqH;IACrH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,0FAA0F;IAC1F,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,MAAM,MAAM,YAAY,GAAG,GAAG,GAAG,GAAG,CAAC;AAErC,yFAAyF;AACzF,MAAM,WAAW,UAAU;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,CAAC,EAAE,UAAU,CAAC;IACd,CAAC,EAAE,UAAU,CAAC;IACd,8CAA8C;IAC9C,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,YAAY,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACrC,CAAC,EAAE,gBAAgB,CAAC;IACpB,CAAC,EAAE,gBAAgB,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sFAAsF;IACtF,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,mDAAmD;IACnD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,SAAS,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,eAAe,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,eAAe,GACxB,aAAa,GACb,aAAa,GACb,aAAa,GACb,eAAe,GACf,eAAe,GACf,iBAAiB,CAAC;AAErB,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,oDAAoD;IACpD,OAAO,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,uDAAuD;IACvD,KAAK,CAAC,EAAE,sBAAsB,CAAC;IAC/B,oCAAoC;IACpC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,kDAAkD;IAClD,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,qDAAqD;IACrD,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,wBAAwB,EAAE,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,UAAU,CAAC;CACrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-seeds.d.ts","sourceRoot":"","sources":["../src/validate-seeds.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"validate-seeds.d.ts","sourceRoot":"","sources":["../src/validate-seeds.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IAClD,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAsChC;AAqCD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,oBAAoB,GAAG,MAAM,CAQ/E"}
|
package/dist/validate-seeds.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
|
-
import { createScenarioWorktree } from "@post-print/agent-harness";
|
|
2
|
+
import { createScenarioWorktree, createSealedWorkspace } from "@post-print/agent-harness";
|
|
3
3
|
import { discoverSuites } from "./discover-suites.js";
|
|
4
4
|
import { loadSuiteFile } from "./load-suite.js";
|
|
5
5
|
import { seedScenarioWorktree } from "./scenario-seed.js";
|
|
6
|
+
import { resolveScenarioWorkspaceRel } from "./validate-suite.js";
|
|
6
7
|
/** Apply each seedPatch in a temp worktree to verify hunks match the current baseline. */
|
|
7
8
|
export async function validateSeedPatches(options) {
|
|
8
9
|
const suitesDir = resolve(options.cwd, options.suitesDir);
|
|
@@ -23,6 +24,7 @@ export async function validateSeedPatches(options) {
|
|
|
23
24
|
try {
|
|
24
25
|
await validateOneSeed(options.cwd, suite.name, scenario.name, scenario.seedPatch, {
|
|
25
26
|
stageOnly: scenario.seedStageOnly,
|
|
27
|
+
workspace: resolveScenarioWorkspaceRel(suite, scenario),
|
|
26
28
|
});
|
|
27
29
|
}
|
|
28
30
|
catch (error) {
|
|
@@ -38,16 +40,29 @@ export async function validateSeedPatches(options) {
|
|
|
38
40
|
return { ok: issues.length === 0, issues, checked };
|
|
39
41
|
}
|
|
40
42
|
async function validateOneSeed(repoRoot, suiteName, scenarioName, seedPatch, options) {
|
|
41
|
-
let
|
|
43
|
+
let cleanup;
|
|
44
|
+
let applyPath;
|
|
42
45
|
try {
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
if (options.workspace) {
|
|
47
|
+
const sealed = await createSealedWorkspace({
|
|
48
|
+
callerCwd: repoRoot,
|
|
49
|
+
workspace: options.workspace,
|
|
50
|
+
});
|
|
51
|
+
cleanup = sealed.cleanup;
|
|
52
|
+
applyPath = sealed.path;
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const worktree = await createScenarioWorktree(repoRoot, `seed-validate-${suiteName}-${scenarioName}`);
|
|
56
|
+
cleanup = worktree.cleanup;
|
|
57
|
+
applyPath = worktree.path;
|
|
58
|
+
}
|
|
59
|
+
await seedScenarioWorktree(repoRoot, applyPath, seedPatch, {
|
|
45
60
|
stageOnly: options.stageOnly,
|
|
46
61
|
});
|
|
47
62
|
}
|
|
48
63
|
finally {
|
|
49
|
-
if (
|
|
50
|
-
await
|
|
64
|
+
if (cleanup) {
|
|
65
|
+
await cleanup().catch(() => undefined);
|
|
51
66
|
}
|
|
52
67
|
}
|
|
53
68
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-seeds.js","sourceRoot":"","sources":["../src/validate-seeds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"validate-seeds.js","sourceRoot":"","sources":["../src/validate-seeds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAE1F,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAelE,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAKzC;IACA,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM;QAC9B,CAAC,CAAC,UAAU,CAAC,MAAM,CACjB,CAAC,IAAI,EAAE,EAAE,CACR,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,iBAAiB,CAAC,CACnD;QACF,CAAC,CAAC,UAAU,CAAC;IAEd,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;QACjF,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACzB,SAAS;YACV,CAAC;YACD,OAAO,EAAE,CAAC;YACV,IAAI,CAAC;gBACJ,MAAM,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,SAAS,EAAE;oBACjF,SAAS,EAAE,QAAQ,CAAC,aAAa;oBACjC,SAAS,EAAE,2BAA2B,CAAC,KAAK,EAAE,QAAQ,CAAC;iBACvD,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK,EAAE,KAAK,CAAC,IAAI;oBACjB,QAAQ,EAAE,QAAQ,CAAC,IAAI;oBACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;oBAC7B,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC/D,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,eAAe,CAC7B,QAAgB,EAChB,SAAiB,EACjB,YAAoB,EACpB,SAAiB,EACjB,OAAoD;IAEpD,IAAI,OAA0C,CAAC;IAC/C,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACJ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;gBAC1C,SAAS,EAAE,QAAQ;gBACnB,SAAS,EAAE,OAAO,CAAC,SAAS;aAC5B,CAAC,CAAC;YACH,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YACzB,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAC5C,QAAQ,EACR,iBAAiB,SAAS,IAAI,YAAY,EAAE,CAC5C,CAAC;YACF,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3B,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,CAAC;QACD,MAAM,oBAAoB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE;YAC1D,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC,CAAC;IACJ,CAAC;YAAS,CAAC;QACV,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAA4B;IACtE,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,aAAa,MAAM,CAAC,OAAO,qBAAqB,CAAC;IACzD,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,OAAO,EAAE,CACxF,CAAC;IACF,OAAO,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,MAAM,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5F,CAAC"}
|
package/dist/validate-suite.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AgentSuiteFile } from "./types.js";
|
|
1
|
+
import type { AgentScenario, AgentSuiteFile } from "./types.js";
|
|
2
2
|
export interface SuiteValidationIssue {
|
|
3
3
|
suitePath: string;
|
|
4
4
|
scenario?: string;
|
|
@@ -19,4 +19,6 @@ export declare function validateSuitePaths(suitePaths: string[], options?: {
|
|
|
19
19
|
rubricsDir?: string;
|
|
20
20
|
}): Promise<SuiteValidationReport>;
|
|
21
21
|
export declare function formatValidationReport(report: SuiteValidationReport): string;
|
|
22
|
+
/** Scenario workspace wins over suite defaults. Caller HEAD when omit or `"."`. */
|
|
23
|
+
export declare function resolveScenarioWorkspaceRel(suite: Pick<AgentSuiteFile, "defaults">, scenario: Pick<AgentScenario, "workspace">): string | undefined;
|
|
22
24
|
//# sourceMappingURL=validate-suite.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAA8B,MAAM,YAAY,CAAC;AA2D5F,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAkZD,kEAAkE;AAClE,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,cAAc,GACnB,oBAAoB,EAAE,CAQxB;AAED,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3E,OAAO,CAAC,qBAAqB,CAAC,CAqIhC;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAY5E;AAED,mFAAmF;AACnF,wBAAgB,2BAA2B,CAC1C,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,EACvC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,GACxC,MAAM,GAAG,SAAS,CAIpB"}
|