@post-print/agent-test 0.3.7 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.js +2 -2
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +37 -13
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +236 -63
- package/dist/compare-scenario.js.map +1 -1
- package/dist/expect.d.ts +2 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +8 -1
- package/dist/expect.js.map +1 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +89 -35
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/record-trace.d.ts +5 -4
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js.map +1 -1
- package/dist/run-suite.d.ts +2 -2
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +102 -94
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-story.d.ts +17 -20
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +92 -39
- package/dist/scenario-story.js.map +1 -1
- package/dist/types.d.ts +46 -15
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +103 -35
- package/dist/validate-suite.js.map +1 -1
- package/package.json +2 -2
package/dist/scenario-story.d.ts
CHANGED
|
@@ -1,16 +1,24 @@
|
|
|
1
1
|
import type { AgentTrace } from "@post-print/agent-harness";
|
|
2
|
-
import
|
|
2
|
+
import { type CompareStoryFields } from "./compare-scenario.js";
|
|
3
|
+
import type { AssertionFailure, CompareMetricGate, JudgeVerdictResult, ScenarioRubric, ScenarioStory } from "./types.js";
|
|
3
4
|
export declare function quoteExcerpt(text: string): string;
|
|
4
5
|
export declare function pathFromArgs(args: Record<string, unknown> | undefined): string | undefined;
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
export type StoryCompareInput = Partial<CompareStoryFields> & {
|
|
7
|
+
aLabel?: string;
|
|
8
|
+
bLabel?: string;
|
|
9
|
+
aDescription?: string;
|
|
10
|
+
bDescription?: string;
|
|
11
|
+
faster?: CompareMetricGate;
|
|
12
|
+
cheaper?: CompareMetricGate;
|
|
11
13
|
aRubric?: ScenarioRubric;
|
|
12
14
|
bRubric?: ScenarioRubric;
|
|
13
|
-
|
|
15
|
+
aTrace?: AgentTrace;
|
|
16
|
+
bTrace?: AgentTrace;
|
|
17
|
+
aDurationMs?: number;
|
|
18
|
+
bDurationMs?: number;
|
|
19
|
+
};
|
|
20
|
+
/** Rubric checks in one short line each. */
|
|
21
|
+
export declare function describeRubricChecks(rubric?: ScenarioRubric, compare?: StoryCompareInput): string[];
|
|
14
22
|
/** What the agent said and which tools it used. */
|
|
15
23
|
export declare function describeTraceHappened(trace?: AgentTrace): string[];
|
|
16
24
|
/** Pass or fail in plain language. */
|
|
@@ -27,17 +35,6 @@ export declare function buildScenarioStory(options: {
|
|
|
27
35
|
skipped?: boolean;
|
|
28
36
|
failures: AssertionFailure[];
|
|
29
37
|
judgeVerdicts?: JudgeVerdictResult[];
|
|
30
|
-
compare?:
|
|
31
|
-
aLabel: string;
|
|
32
|
-
bLabel: string;
|
|
33
|
-
faster?: CompareArmId;
|
|
34
|
-
cheaper?: CompareArmId;
|
|
35
|
-
aRubric?: ScenarioRubric;
|
|
36
|
-
bRubric?: ScenarioRubric;
|
|
37
|
-
aTrace?: AgentTrace;
|
|
38
|
-
bTrace?: AgentTrace;
|
|
39
|
-
aDurationMs?: number;
|
|
40
|
-
bDurationMs?: number;
|
|
41
|
-
};
|
|
38
|
+
compare?: StoryCompareInput;
|
|
42
39
|
}): ScenarioStory;
|
|
43
40
|
//# sourceMappingURL=scenario-story.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-story.d.ts","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"scenario-story.d.ts","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,OAAO,EAGN,KAAK,kBAAkB,EAGvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACX,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,aAAa,EACb,MAAM,YAAY,CAAC;AAKpB,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASjD;AAeD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW1F;AAMD,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,GAAG;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAqEF,4CAA4C;AAC5C,wBAAgB,oBAAoB,CACnC,MAAM,CAAC,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,iBAAiB,GACzB,MAAM,EAAE,CA4EV;AAED,mDAAmD;AACnD,wBAAgB,qBAAqB,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG,MAAM,EAAE,CAyBlE;AAyBD,sCAAsC;AACtC,wBAAgB,eAAe,CAAC,OAAO,EAAE;IACxC,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACrC,GAAG,MAAM,EAAE,CAiBX;AAQD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC3C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B,GAAG,aAAa,CA4BhB"}
|
package/dist/scenario-story.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describeCompareOutcome } from "./compare-scenario.js";
|
|
1
|
+
import { buildCompareResult, describeCompareOutcome, resolveCompareMetricPairs, } from "./compare-scenario.js";
|
|
2
2
|
const PATH_ARG_KEYS = ["path", "file_path", "filePath", "target_file", "uri", "cwd"];
|
|
3
3
|
const EXCERPT_CHARS = 80;
|
|
4
4
|
export function quoteExcerpt(text) {
|
|
@@ -38,6 +38,70 @@ export function pathFromArgs(args) {
|
|
|
38
38
|
function countLabel(count, singular, plural) {
|
|
39
39
|
return count === 1 ? `1 ${singular}` : `${count} ${plural}`;
|
|
40
40
|
}
|
|
41
|
+
function compareHeading(labels) {
|
|
42
|
+
if (labels.length === 2 && labels[0] && labels[1]) {
|
|
43
|
+
return `compare ${labels[0]} vs ${labels[1]}`;
|
|
44
|
+
}
|
|
45
|
+
if (labels.length <= 1) {
|
|
46
|
+
return labels[0] ? `compare ${labels[0]}` : "compare arms";
|
|
47
|
+
}
|
|
48
|
+
const last = labels[labels.length - 1];
|
|
49
|
+
return `compare ${labels.slice(0, -1).join(", ")}, and ${last}`;
|
|
50
|
+
}
|
|
51
|
+
function labelForArm(arms, id) {
|
|
52
|
+
return arms.find((arm) => arm.id === id)?.label ?? id;
|
|
53
|
+
}
|
|
54
|
+
function normalizeStoryCompare(compare) {
|
|
55
|
+
if (compare.arms && compare.arms.length > 0) {
|
|
56
|
+
const armIds = compare.arms.map((arm) => arm.id);
|
|
57
|
+
return {
|
|
58
|
+
arms: compare.arms,
|
|
59
|
+
faster: Array.isArray(compare.faster)
|
|
60
|
+
? compare.faster
|
|
61
|
+
: resolveCompareMetricPairs(compare.faster, armIds),
|
|
62
|
+
cheaper: Array.isArray(compare.cheaper)
|
|
63
|
+
? compare.cheaper
|
|
64
|
+
: resolveCompareMetricPairs(compare.cheaper, armIds),
|
|
65
|
+
aLabel: compare.aLabel ?? compare.arms[0]?.label ?? "control",
|
|
66
|
+
bLabel: compare.bLabel ?? compare.arms[1]?.label ?? "experimental",
|
|
67
|
+
aDescription: compare.aDescription ?? compare.arms[0]?.description,
|
|
68
|
+
bDescription: compare.bDescription ?? compare.arms[1]?.description,
|
|
69
|
+
aRubric: compare.aRubric ?? compare.arms[0]?.rubric,
|
|
70
|
+
bRubric: compare.bRubric ?? compare.arms[1]?.rubric,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
const aLabel = compare.aLabel ?? "control";
|
|
74
|
+
const bLabel = compare.bLabel ?? "experimental";
|
|
75
|
+
const arms = [
|
|
76
|
+
{
|
|
77
|
+
id: "a",
|
|
78
|
+
label: aLabel,
|
|
79
|
+
description: compare.aDescription,
|
|
80
|
+
rubric: compare.aRubric,
|
|
81
|
+
trace: compare.aTrace,
|
|
82
|
+
durationMs: compare.aDurationMs,
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
id: "b",
|
|
86
|
+
label: bLabel,
|
|
87
|
+
description: compare.bDescription,
|
|
88
|
+
rubric: compare.bRubric,
|
|
89
|
+
trace: compare.bTrace,
|
|
90
|
+
durationMs: compare.bDurationMs,
|
|
91
|
+
},
|
|
92
|
+
];
|
|
93
|
+
return {
|
|
94
|
+
arms,
|
|
95
|
+
faster: resolveCompareMetricPairs(compare.faster, ["a", "b"]),
|
|
96
|
+
cheaper: resolveCompareMetricPairs(compare.cheaper, ["a", "b"]),
|
|
97
|
+
aLabel,
|
|
98
|
+
bLabel,
|
|
99
|
+
aDescription: compare.aDescription,
|
|
100
|
+
bDescription: compare.bDescription,
|
|
101
|
+
aRubric: compare.aRubric,
|
|
102
|
+
bRubric: compare.bRubric,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
41
105
|
/** Rubric checks in one short line each. */
|
|
42
106
|
export function describeRubricChecks(rubric, compare) {
|
|
43
107
|
if (!rubric) {
|
|
@@ -81,33 +145,31 @@ export function describeRubricChecks(rubric, compare) {
|
|
|
81
145
|
lines.push(`announce review depth ${rubric.reviewDepth}`);
|
|
82
146
|
}
|
|
83
147
|
if (compare) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
148
|
+
const normalized = normalizeStoryCompare(compare);
|
|
149
|
+
lines.push(compareHeading(normalized.arms.map((arm) => arm.label)));
|
|
150
|
+
for (const arm of normalized.arms) {
|
|
151
|
+
if (arm.description) {
|
|
152
|
+
lines.push(`${arm.label}: ${arm.description}`);
|
|
153
|
+
}
|
|
89
154
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
const loser = compare.cheaper === "a" ? compare.bLabel : compare.aLabel;
|
|
93
|
-
lines.push(`${winner} uses fewer tokens than ${loser}`);
|
|
155
|
+
for (const pair of normalized.faster ?? []) {
|
|
156
|
+
lines.push(`${labelForArm(normalized.arms, pair.winner)} is faster than ${labelForArm(normalized.arms, pair.loser)}`);
|
|
94
157
|
}
|
|
95
|
-
for (const
|
|
96
|
-
|
|
97
|
-
continue;
|
|
98
|
-
}
|
|
99
|
-
lines.push(`${compare.aLabel}: ${line}`);
|
|
158
|
+
for (const pair of normalized.cheaper ?? []) {
|
|
159
|
+
lines.push(`${labelForArm(normalized.arms, pair.winner)} uses fewer tokens than ${labelForArm(normalized.arms, pair.loser)}`);
|
|
100
160
|
}
|
|
101
|
-
for (const
|
|
102
|
-
|
|
103
|
-
|
|
161
|
+
for (const arm of normalized.arms) {
|
|
162
|
+
for (const line of describeRubricChecks(arm.rubric)) {
|
|
163
|
+
if (line === "no rubric checks" || line === "no rubric recorded") {
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
lines.push(`${arm.label}: ${line}`);
|
|
104
167
|
}
|
|
105
|
-
lines.push(`${compare.bLabel}: ${line}`);
|
|
106
168
|
}
|
|
107
169
|
}
|
|
108
170
|
if (rubric.judge && rubric.judge.length > 0) {
|
|
109
171
|
lines.push(compare
|
|
110
|
-
? `judge answers ${countLabel(rubric.judge.length, "question", "questions")} on both arms`
|
|
172
|
+
? `judge answers ${countLabel(rubric.judge.length, "question", "questions")} on ${normalizeStoryCompare(compare).arms.length === 2 ? "both arms" : "every arm"}`
|
|
111
173
|
: `judge answers ${countLabel(rubric.judge.length, "question", "questions")}`);
|
|
112
174
|
}
|
|
113
175
|
return lines.length > 0 ? lines : ["no rubric checks"];
|
|
@@ -182,28 +244,19 @@ function storyVerdict(happened, outcome) {
|
|
|
182
244
|
return outcome.filter((line) => !REDUNDANT_VERDICT.has(line) && !happened.includes(line));
|
|
183
245
|
}
|
|
184
246
|
export function buildScenarioStory(options) {
|
|
247
|
+
const compare = options.compare ? normalizeStoryCompare(options.compare) : undefined;
|
|
185
248
|
const result = options.skipped
|
|
186
249
|
? ["scenario skipped"]
|
|
187
|
-
:
|
|
250
|
+
: compare
|
|
188
251
|
? [
|
|
189
|
-
...describeTraceHappened(
|
|
190
|
-
...
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
durationMs: options.compare.aDurationMs,
|
|
198
|
-
},
|
|
199
|
-
b: {
|
|
200
|
-
id: "b",
|
|
201
|
-
label: options.compare.bLabel,
|
|
202
|
-
prompt: "",
|
|
203
|
-
trace: options.compare.bTrace,
|
|
204
|
-
durationMs: options.compare.bDurationMs,
|
|
205
|
-
},
|
|
206
|
-
}),
|
|
252
|
+
...compare.arms.flatMap((arm) => describeTraceHappened(arm.trace).map((line) => `${arm.label}: ${line}`)),
|
|
253
|
+
...describeCompareOutcome(buildCompareResult(compare.arms.map((arm) => ({
|
|
254
|
+
id: arm.id,
|
|
255
|
+
label: arm.label,
|
|
256
|
+
prompt: "",
|
|
257
|
+
trace: arm.trace,
|
|
258
|
+
durationMs: arm.durationMs,
|
|
259
|
+
})), { faster: compare.faster, cheaper: compare.cheaper })),
|
|
207
260
|
]
|
|
208
261
|
: describeTraceHappened(options.trace);
|
|
209
262
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-story.js","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAS/D,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AAC9F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QACrC,OAAO,IAAI,OAAO,GAAG,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAyC;IACrE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,QAAgB,EAAE,MAAc;IAClE,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,oBAAoB,CACnC,MAAuB,EACvB,OAOC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,kBAAkB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,0BAA0B,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACxE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,mBAAmB,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACzE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;gBAClE,SAAS;YACV,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;gBAClE,SAAS;YACV,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CACT,OAAO;YACN,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,eAAe;YAC1F,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAC9E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACxD,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,eAAe,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;SACnC,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACpD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,IAAI,EAAE,EAAE,CACtF,eAAe,CAAC,IAAI,CAAC,CACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IACjD,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,mBAAmB,OAAO,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC9E,OAAO,kBAAkB,OAAO,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;QAC5C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAK/B;IACA,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,SAAS,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpE,SAAS,YAAY,CAAC,QAAkB,EAAE,OAAiB;IAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAmBlC;IACA,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;QAC7B,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC,OAAO;YAChB,CAAC,CAAC;gBACA,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CACnD,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,CAC/C;gBACD,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CACnD,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,CAC/C;gBACD,GAAG,sBAAsB,CAAC;oBACzB,CAAC,EAAE;wBACF,EAAE,EAAE,GAAG;wBACP,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW;qBACvC;oBACD,CAAC,EAAE;wBACF,EAAE,EAAE,GAAG;wBACP,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW;qBACvC;iBACD,CAAC;aACF;YACF,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO;QACN,QAAQ,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM;QACN,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;KACvD,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"scenario-story.js","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,kBAAkB,EAGlB,sBAAsB,EACtB,yBAAyB,GACzB,MAAM,uBAAuB,CAAC;AAS/B,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AAC9F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QACrC,OAAO,IAAI,OAAO,GAAG,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAyC;IACrE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,QAAgB,EAAE,MAAc;IAClE,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAiBD,SAAS,cAAc,CAAC,MAAgB;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,WAAW,MAAM,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,OAAO,WAAW,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB,EAAE,EAAU;IACvD,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;AACvD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA0B;IACxD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjD,OAAO;YACN,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;gBACpC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAChB,CAAC,CAAC,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC;YACpD,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;gBACtC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACjB,CAAC,CAAC,yBAAyB,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC;YACrD,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;YAC7D,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,cAAc;YAClE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW;YAClE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW;YAClE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM;YACnD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM;SACnD,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC;IAChD,MAAM,IAAI,GAAsB;QAC/B;YACC,EAAE,EAAE,GAAG;YACP,KAAK,EAAE,MAAM;YACb,WAAW,EAAE,OAAO,CAAC,YAAY;YACjC,MAAM,EAAE,OAAO,CAAC,OAAO;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,UAAU,EAAE,OAAO,CAAC,WAAW;SAC/B;QACD;YACC,EAAE,EAAE,GAAG;YACP,KAAK,EAAE,MAAM;YACb,WAAW,EAAE,OAAO,CAAC,YAAY;YACjC,MAAM,EAAE,OAAO,CAAC,OAAO;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,UAAU,EAAE,OAAO,CAAC,WAAW;SAC/B;KACD,CAAC;IACF,OAAO;QACN,IAAI;QACJ,MAAM,EAAE,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC7D,OAAO,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC/D,MAAM;QACN,MAAM;QACN,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACxB,CAAC;AACH,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,oBAAoB,CACnC,MAAuB,EACvB,OAA2B;IAE3B,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,kBAAkB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,0BAA0B,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC;QACF,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CACT,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CACzG,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CACT,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,2BAA2B,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CACjH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;oBAClE,SAAS;gBACV,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CACT,OAAO;YACN,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,EAAE;YAChK,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAC9E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACxD,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,eAAe,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;SACnC,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACpD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,IAAI,EAAE,EAAE,CACtF,eAAe,CAAC,IAAI,CAAC,CACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IACjD,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,mBAAmB,OAAO,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC9E,OAAO,kBAAkB,OAAO,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;QAC5C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAK/B;IACA,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,SAAS,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpE,SAAS,YAAY,CAAC,QAAkB,EAAE,OAAiB;IAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAQlC;IACA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;QAC7B,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACtB,CAAC,CAAC,OAAO;YACR,CAAC,CAAC;gBACA,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/B,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CACvE;gBACD,GAAG,sBAAsB,CACxB,kBAAkB,CACjB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC1B,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,UAAU,EAAE,GAAG,CAAC,UAAU;iBAC1B,CAAC,CAAC,EACH,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CACpD,CACD;aACD;YACF,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO;QACN,QAAQ,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM;QACN,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;KACvD,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export interface ScenarioRubric {
|
|
|
8
8
|
tier?: "low" | "medium" | "high";
|
|
9
9
|
/** Chat-style tier announce — infer one-line routing from transcript when routing.tier absent. */
|
|
10
10
|
handsOnRouting?: boolean;
|
|
11
|
-
/** Substring
|
|
11
|
+
/** Substring that must appear in assistant reply text. */
|
|
12
12
|
must?: string[];
|
|
13
13
|
/** Substring that must not appear in assistant text, commands, artifacts, or tool args. Tool results are ignored. */
|
|
14
14
|
mustNot?: string[];
|
|
@@ -37,11 +37,25 @@ export interface ScenarioRubric {
|
|
|
37
37
|
/** Any transcript outcome. The judge sees assistant text, tool args, and tool results. */
|
|
38
38
|
judge?: JudgeRubricItem[];
|
|
39
39
|
}
|
|
40
|
-
|
|
40
|
+
/** Arm id. Two-arm form uses `a` and `b`. Named arms use a lowercase slug. */
|
|
41
|
+
export type CompareArmId = string;
|
|
42
|
+
/** Named winner-versus-loser pair for a metric gate. */
|
|
43
|
+
export interface CompareMetricPair {
|
|
44
|
+
winner: CompareArmId;
|
|
45
|
+
loser: CompareArmId;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Two-arm form names the winner. Named-arm form lists winner-versus-loser pairs.
|
|
49
|
+
* A 2x2 must not require one arm to beat every other arm.
|
|
50
|
+
*/
|
|
51
|
+
export type CompareMetricGate = CompareArmId | CompareMetricPair[];
|
|
41
52
|
/** Overrides for one side of a compare scenario. Omitted fields inherit the scenario. */
|
|
42
53
|
export interface CompareArm {
|
|
54
|
+
/** Required on `compare.arms`. Implicit `a` / `b` in the two-arm form. */
|
|
55
|
+
id?: CompareArmId;
|
|
56
|
+
/** Display name. Omitted labels are control (a), experimental (b), or the arm id. */
|
|
43
57
|
label?: string;
|
|
44
|
-
/**
|
|
58
|
+
/** Required. Says what this arm tests. */
|
|
45
59
|
description?: string;
|
|
46
60
|
prompt?: string;
|
|
47
61
|
host?: AgentHost;
|
|
@@ -60,17 +74,26 @@ export interface CompareArm {
|
|
|
60
74
|
rubric?: ScenarioRubric;
|
|
61
75
|
}
|
|
62
76
|
/**
|
|
63
|
-
* Two live arms in one scenario.
|
|
77
|
+
* Two or more live arms in one scenario.
|
|
78
|
+
* Keep `a` and `b` for the two-arm form. Use `arms` when the scenario has more than two workspaces.
|
|
64
79
|
* `faster` and `cheaper` are optional metric gates.
|
|
65
|
-
* `rubric.judge` is optional and scores
|
|
80
|
+
* `rubric.judge` is optional and scores every arm transcript.
|
|
66
81
|
*/
|
|
67
82
|
export interface ScenarioCompare {
|
|
68
|
-
a
|
|
69
|
-
b
|
|
70
|
-
/** Named
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
|
|
83
|
+
a?: CompareArm;
|
|
84
|
+
b?: CompareArm;
|
|
85
|
+
/** Named arms. Each row needs `id`, `description`, and a workspace override when the trees differ. */
|
|
86
|
+
arms?: CompareArm[];
|
|
87
|
+
/**
|
|
88
|
+
* Two-arm form: named arm must have a shorter duration than the other arm.
|
|
89
|
+
* Named-arm form: winner-versus-loser pairs.
|
|
90
|
+
*/
|
|
91
|
+
faster?: CompareMetricGate;
|
|
92
|
+
/**
|
|
93
|
+
* Two-arm form: named arm must use fewer total tokens than the other arm.
|
|
94
|
+
* Named-arm form: winner-versus-loser pairs.
|
|
95
|
+
*/
|
|
96
|
+
cheaper?: CompareMetricGate;
|
|
74
97
|
}
|
|
75
98
|
export interface CompareArmResult {
|
|
76
99
|
id: CompareArmId;
|
|
@@ -82,14 +105,22 @@ export interface CompareArmResult {
|
|
|
82
105
|
durationMs?: number;
|
|
83
106
|
}
|
|
84
107
|
export interface ScenarioCompareResult {
|
|
85
|
-
|
|
86
|
-
|
|
108
|
+
/** Every arm in author order. */
|
|
109
|
+
arms: CompareArmResult[];
|
|
110
|
+
/** Present when an arm id is `a`. */
|
|
111
|
+
a?: CompareArmResult;
|
|
112
|
+
/** Present when an arm id is `b`. */
|
|
113
|
+
b?: CompareArmResult;
|
|
114
|
+
/** Resolved faster pairs after load. */
|
|
115
|
+
faster?: CompareMetricPair[];
|
|
116
|
+
/** Resolved cheaper pairs after load. */
|
|
117
|
+
cheaper?: CompareMetricPair[];
|
|
87
118
|
}
|
|
88
119
|
export interface AgentScenario {
|
|
89
120
|
name: string;
|
|
90
121
|
/** Plain-language note. Says what this scenario tests. */
|
|
91
122
|
description?: string;
|
|
92
|
-
/** Run two arms. Optional metric gates and an optional
|
|
123
|
+
/** Run two or more arms. Optional metric gates and an optional shared judge. */
|
|
93
124
|
compare?: ScenarioCompare;
|
|
94
125
|
prompt: string;
|
|
95
126
|
host?: AgentHost;
|
|
@@ -209,7 +240,7 @@ export interface ScenarioResult {
|
|
|
209
240
|
description?: string;
|
|
210
241
|
/** Full agent transcript when available (for HTML reports / debug bundles). */
|
|
211
242
|
trace?: AgentTrace;
|
|
212
|
-
/**
|
|
243
|
+
/** Compare arms when this scenario is a compare run. */
|
|
213
244
|
compare?: ScenarioCompareResult;
|
|
214
245
|
/** Plain-language summary of the check, the agent run, and the verdict. */
|
|
215
246
|
story?: ScenarioStory;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,qHAAqH;IACrH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,0FAA0F;IAC1F,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,8EAA8E;AAC9E,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,wDAAwD;AACxD,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,YAAY,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG,iBAAiB,EAAE,CAAC;AAEnE,yFAAyF;AACzF,MAAM,WAAW,UAAU;IAC1B,0EAA0E;IAC1E,EAAE,CAAC,EAAE,YAAY,CAAC;IAClB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;CACxB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC/B,CAAC,CAAC,EAAE,UAAU,CAAC;IACf,CAAC,CAAC,EAAE,UAAU,CAAC;IACf,sGAAsG;IACtG,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B;;;OAGG;IACH,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,YAAY,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACrC,iCAAiC;IACjC,IAAI,EAAE,gBAAgB,EAAE,CAAC;IACzB,qCAAqC;IACrC,CAAC,CAAC,EAAE,gBAAgB,CAAC;IACrB,qCAAqC;IACrC,CAAC,CAAC,EAAE,gBAAgB,CAAC;IACrB,wCAAwC;IACxC,MAAM,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC7B,yCAAyC;IACzC,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sFAAsF;IACtF,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,mDAAmD;IACnD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,SAAS,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,eAAe,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,eAAe,GACxB,aAAa,GACb,aAAa,GACb,aAAa,GACb,eAAe,GACf,eAAe,GACf,iBAAiB,CAAC;AAErB,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,oDAAoD;IACpD,OAAO,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,uDAAuD;IACvD,KAAK,CAAC,EAAE,sBAAsB,CAAC;IAC/B,oCAAoC;IACpC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,kDAAkD;IAClD,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,wDAAwD;IACxD,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,wBAAwB,EAAE,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,UAAU,CAAC;CACrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EACX,aAAa,EACb,cAAc,EAKd,MAAM,YAAY,CAAC;AA2DpB,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AA2kBD,kEAAkE;AAClE,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,cAAc,GACnB,oBAAoB,EAAE,CAQxB;AAED,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3E,OAAO,CAAC,qBAAqB,CAAC,CAyHhC;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAY5E;AAED,mFAAmF;AACnF,wBAAgB,2BAA2B,CAC1C,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,EACvC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,GACxC,MAAM,GAAG,SAAS,CAIpB"}
|
package/dist/validate-suite.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { access, stat } from "node:fs/promises";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { isHostSlug, isKnownAgentHost, isRepoRelativeSkillPath, parseScenarioWorkspace, skillManifestRelPath, skillPathsFromSetting, } from "@post-print/agent-harness";
|
|
4
|
-
import { applyCompareArm } from "./compare-scenario.js";
|
|
4
|
+
import { applyCompareArm, parseCompareArmId, resolveCompareArms } from "./compare-scenario.js";
|
|
5
5
|
import { loadSuiteFile } from "./load-suite.js";
|
|
6
6
|
const REPLAY_DEPRECATION = "Replay-based testing is deprecated and no longer supported; use Cursor, Claude, or OpenAI.";
|
|
7
7
|
function isKnownHostId(host) {
|
|
@@ -139,7 +139,9 @@ function validateCompareArmFields(issues, suitePath, scenarioName, fieldPrefix,
|
|
|
139
139
|
if (arm.label !== undefined && (typeof arm.label !== "string" || !arm.label.trim())) {
|
|
140
140
|
pushIssue(issues, suitePath, fieldPrefix, "label must be a non-empty string", scenarioName);
|
|
141
141
|
}
|
|
142
|
-
|
|
142
|
+
if (typeof arm.description !== "string" || !arm.description.trim()) {
|
|
143
|
+
pushIssue(issues, suitePath, `${fieldPrefix}.description`, "description is required", scenarioName);
|
|
144
|
+
}
|
|
143
145
|
if (arm.prompt !== undefined && typeof arm.prompt !== "string") {
|
|
144
146
|
pushIssue(issues, suitePath, `${fieldPrefix}.prompt`, "prompt must be a string", scenarioName);
|
|
145
147
|
}
|
|
@@ -178,30 +180,108 @@ function validateCompare(issues, suitePath, scenario) {
|
|
|
178
180
|
return;
|
|
179
181
|
}
|
|
180
182
|
if (typeof scenario.compare !== "object" || scenario.compare === null) {
|
|
181
|
-
pushIssue(issues, suitePath, "compare", "compare must be an object with a and b", scenario.name);
|
|
183
|
+
pushIssue(issues, suitePath, "compare", "compare must be an object with a and b, or arms", scenario.name);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const compare = scenario.compare;
|
|
187
|
+
const hasLegacyArms = compare.a !== undefined || compare.b !== undefined;
|
|
188
|
+
const hasNamedArms = compare.arms !== undefined;
|
|
189
|
+
if (hasLegacyArms && hasNamedArms) {
|
|
190
|
+
pushIssue(issues, suitePath, "compare", "compare must use a and b, or arms, not both", scenario.name);
|
|
182
191
|
return;
|
|
183
192
|
}
|
|
184
|
-
if (
|
|
185
|
-
|
|
193
|
+
if (hasNamedArms) {
|
|
194
|
+
validateNamedCompareArms(issues, suitePath, scenario.name, compare);
|
|
186
195
|
}
|
|
187
196
|
else {
|
|
188
|
-
|
|
197
|
+
if (typeof compare.a !== "object" || compare.a === null) {
|
|
198
|
+
pushIssue(issues, suitePath, "compare.a", "compare.a must be an object", scenario.name);
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
validateCompareArmFields(issues, suitePath, scenario.name, "compare.a", compare.a);
|
|
202
|
+
}
|
|
203
|
+
if (typeof compare.b !== "object" || compare.b === null) {
|
|
204
|
+
pushIssue(issues, suitePath, "compare.b", "compare.b must be an object", scenario.name);
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
validateCompareArmFields(issues, suitePath, scenario.name, "compare.b", compare.b);
|
|
208
|
+
}
|
|
189
209
|
}
|
|
190
|
-
|
|
191
|
-
|
|
210
|
+
const armIds = resolveCompareArms(compare)
|
|
211
|
+
.map((entry) => entry.id)
|
|
212
|
+
.filter(Boolean);
|
|
213
|
+
validateCompareMetricGate(issues, suitePath, scenario.name, "compare.faster", compare.faster, armIds);
|
|
214
|
+
validateCompareMetricGate(issues, suitePath, scenario.name, "compare.cheaper", compare.cheaper, armIds);
|
|
215
|
+
}
|
|
216
|
+
function validateNamedCompareArms(issues, suitePath, scenarioName, compare) {
|
|
217
|
+
if (!Array.isArray(compare.arms) || compare.arms.length < 2) {
|
|
218
|
+
pushIssue(issues, suitePath, "compare.arms", "compare.arms must be an array of two or more arms", scenarioName);
|
|
219
|
+
return;
|
|
192
220
|
}
|
|
193
|
-
|
|
194
|
-
|
|
221
|
+
const seen = new Set();
|
|
222
|
+
for (const [index, arm] of compare.arms.entries()) {
|
|
223
|
+
const fieldPrefix = `compare.arms[${index}]`;
|
|
224
|
+
if (typeof arm !== "object" || arm === null) {
|
|
225
|
+
pushIssue(issues, suitePath, fieldPrefix, "arm must be an object", scenarioName);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
const id = parseCompareArmId(arm.id);
|
|
229
|
+
if (!id) {
|
|
230
|
+
pushIssue(issues, suitePath, `${fieldPrefix}.id`, "id must be a lowercase slug such as skel-clean", scenarioName);
|
|
231
|
+
}
|
|
232
|
+
else if (seen.has(id)) {
|
|
233
|
+
pushIssue(issues, suitePath, `${fieldPrefix}.id`, `duplicate arm id ${id}`, scenarioName);
|
|
234
|
+
}
|
|
235
|
+
else {
|
|
236
|
+
seen.add(id);
|
|
237
|
+
}
|
|
238
|
+
validateCompareArmFields(issues, suitePath, scenarioName, fieldPrefix, arm);
|
|
195
239
|
}
|
|
196
|
-
validateCompareArmPick(issues, suitePath, scenario.name, "compare.faster", scenario.compare.faster);
|
|
197
|
-
validateCompareArmPick(issues, suitePath, scenario.name, "compare.cheaper", scenario.compare.cheaper);
|
|
198
240
|
}
|
|
199
|
-
function
|
|
241
|
+
function validateCompareMetricGate(issues, suitePath, scenarioName, field, value, armIds) {
|
|
200
242
|
if (value === undefined) {
|
|
201
243
|
return;
|
|
202
244
|
}
|
|
203
|
-
if (
|
|
204
|
-
|
|
245
|
+
if (typeof value === "string") {
|
|
246
|
+
if (armIds.length !== 2) {
|
|
247
|
+
pushIssue(issues, suitePath, field, `${field} must be an array of { winner, loser } pairs when compare has more than two arms`, scenarioName);
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
if (!armIds.includes(value)) {
|
|
251
|
+
pushIssue(issues, suitePath, field, `${field} must be ${armIds.map((id) => JSON.stringify(id)).join(" or ")}`, scenarioName);
|
|
252
|
+
}
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
256
|
+
pushIssue(issues, suitePath, field, armIds.length === 2
|
|
257
|
+
? `${field} must be ${armIds.map((id) => JSON.stringify(id)).join(" or ")} or an array of { winner, loser } pairs`
|
|
258
|
+
: `${field} must be an array of { winner, loser } pairs`, scenarioName);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
const seen = new Set();
|
|
262
|
+
for (const [index, pair] of value.entries()) {
|
|
263
|
+
if (typeof pair !== "object" || pair === null) {
|
|
264
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "pair must be { winner, loser }", scenarioName);
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
const typed = pair;
|
|
268
|
+
validateMetricPairEnd(issues, suitePath, scenarioName, `${field}[${index}].winner`, typed.winner, armIds);
|
|
269
|
+
validateMetricPairEnd(issues, suitePath, scenarioName, `${field}[${index}].loser`, typed.loser, armIds);
|
|
270
|
+
if (typeof typed.winner === "string" && typed.winner === typed.loser) {
|
|
271
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "winner and loser must be different arms", scenarioName);
|
|
272
|
+
}
|
|
273
|
+
if (typeof typed.winner === "string" && typeof typed.loser === "string") {
|
|
274
|
+
const key = `${typed.winner}>${typed.loser}`;
|
|
275
|
+
if (seen.has(key)) {
|
|
276
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "duplicate pair", scenarioName);
|
|
277
|
+
}
|
|
278
|
+
seen.add(key);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
function validateMetricPairEnd(issues, suitePath, scenarioName, field, value, armIds) {
|
|
283
|
+
if (typeof value !== "string" || !armIds.includes(value)) {
|
|
284
|
+
pushIssue(issues, suitePath, field, `must be one of ${armIds.join(", ")}`, scenarioName);
|
|
205
285
|
}
|
|
206
286
|
}
|
|
207
287
|
function validateHosts(issues, suitePath, suite) {
|
|
@@ -302,16 +382,10 @@ export async function validateSuitePaths(suitePaths, options) {
|
|
|
302
382
|
if (options?.validatePaths && repoRoot) {
|
|
303
383
|
for (const scenario of suite.scenarios) {
|
|
304
384
|
const workspaceChecks = scenario.compare
|
|
305
|
-
?
|
|
306
|
-
{
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
},
|
|
310
|
-
{
|
|
311
|
-
field: "compare.b.workspace",
|
|
312
|
-
rel: resolveScenarioWorkspaceRel(suite, applyCompareArm(scenario, "b")),
|
|
313
|
-
},
|
|
314
|
-
]
|
|
385
|
+
? resolveCompareArms(scenario.compare).map((entry) => ({
|
|
386
|
+
field: `compare.${entry.id}.workspace`,
|
|
387
|
+
rel: resolveScenarioWorkspaceRel(suite, applyCompareArm(scenario, entry.id)),
|
|
388
|
+
}))
|
|
315
389
|
: [{ field: "workspace", rel: resolveScenarioWorkspaceRel(suite, scenario) }];
|
|
316
390
|
for (const check of workspaceChecks) {
|
|
317
391
|
const workspaceRel = check.rel;
|
|
@@ -330,16 +404,10 @@ export async function validateSuitePaths(suitePaths, options) {
|
|
|
330
404
|
}
|
|
331
405
|
}
|
|
332
406
|
const skillTargets = scenario.compare
|
|
333
|
-
?
|
|
334
|
-
{
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
},
|
|
338
|
-
{
|
|
339
|
-
field: "compare.b.skills",
|
|
340
|
-
scenario: applyCompareArm(scenario, "b"),
|
|
341
|
-
},
|
|
342
|
-
]
|
|
407
|
+
? resolveCompareArms(scenario.compare).map((entry) => ({
|
|
408
|
+
field: `compare.${entry.id}.skills`,
|
|
409
|
+
scenario: applyCompareArm(scenario, entry.id),
|
|
410
|
+
}))
|
|
343
411
|
: [{ field: "skills", scenario }];
|
|
344
412
|
for (const target of skillTargets) {
|
|
345
413
|
const skills = target.scenario.skills ?? suite.defaults?.skills;
|