@post-print/agent-test 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.js +2 -2
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +37 -15
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +236 -65
- package/dist/compare-scenario.js.map +1 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +89 -35
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/record-trace.d.ts +5 -4
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js.map +1 -1
- package/dist/run-suite.d.ts +2 -2
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +102 -94
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-story.d.ts +15 -22
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +92 -45
- package/dist/scenario-story.js.map +1 -1
- package/dist/types.d.ts +44 -14
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +100 -34
- package/dist/validate-suite.js.map +1 -1
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-story.d.ts","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"scenario-story.d.ts","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,OAAO,EAGN,KAAK,kBAAkB,EAGvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACX,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,aAAa,EACb,MAAM,YAAY,CAAC;AAKpB,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASjD;AAeD,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW1F;AAMD,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,GAAG;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAqEF,4CAA4C;AAC5C,wBAAgB,oBAAoB,CACnC,MAAM,CAAC,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,iBAAiB,GACzB,MAAM,EAAE,CA4EV;AAED,mDAAmD;AACnD,wBAAgB,qBAAqB,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG,MAAM,EAAE,CAyBlE;AAyBD,sCAAsC;AACtC,wBAAgB,eAAe,CAAC,OAAO,EAAE;IACxC,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACrC,GAAG,MAAM,EAAE,CAiBX;AAQD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC3C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B,GAAG,aAAa,CA4BhB"}
|
package/dist/scenario-story.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describeCompareOutcome } from "./compare-scenario.js";
|
|
1
|
+
import { buildCompareResult, describeCompareOutcome, resolveCompareMetricPairs, } from "./compare-scenario.js";
|
|
2
2
|
const PATH_ARG_KEYS = ["path", "file_path", "filePath", "target_file", "uri", "cwd"];
|
|
3
3
|
const EXCERPT_CHARS = 80;
|
|
4
4
|
export function quoteExcerpt(text) {
|
|
@@ -38,6 +38,70 @@ export function pathFromArgs(args) {
|
|
|
38
38
|
function countLabel(count, singular, plural) {
|
|
39
39
|
return count === 1 ? `1 ${singular}` : `${count} ${plural}`;
|
|
40
40
|
}
|
|
41
|
+
function compareHeading(labels) {
|
|
42
|
+
if (labels.length === 2 && labels[0] && labels[1]) {
|
|
43
|
+
return `compare ${labels[0]} vs ${labels[1]}`;
|
|
44
|
+
}
|
|
45
|
+
if (labels.length <= 1) {
|
|
46
|
+
return labels[0] ? `compare ${labels[0]}` : "compare arms";
|
|
47
|
+
}
|
|
48
|
+
const last = labels[labels.length - 1];
|
|
49
|
+
return `compare ${labels.slice(0, -1).join(", ")}, and ${last}`;
|
|
50
|
+
}
|
|
51
|
+
function labelForArm(arms, id) {
|
|
52
|
+
return arms.find((arm) => arm.id === id)?.label ?? id;
|
|
53
|
+
}
|
|
54
|
+
function normalizeStoryCompare(compare) {
|
|
55
|
+
if (compare.arms && compare.arms.length > 0) {
|
|
56
|
+
const armIds = compare.arms.map((arm) => arm.id);
|
|
57
|
+
return {
|
|
58
|
+
arms: compare.arms,
|
|
59
|
+
faster: Array.isArray(compare.faster)
|
|
60
|
+
? compare.faster
|
|
61
|
+
: resolveCompareMetricPairs(compare.faster, armIds),
|
|
62
|
+
cheaper: Array.isArray(compare.cheaper)
|
|
63
|
+
? compare.cheaper
|
|
64
|
+
: resolveCompareMetricPairs(compare.cheaper, armIds),
|
|
65
|
+
aLabel: compare.aLabel ?? compare.arms[0]?.label ?? "control",
|
|
66
|
+
bLabel: compare.bLabel ?? compare.arms[1]?.label ?? "experimental",
|
|
67
|
+
aDescription: compare.aDescription ?? compare.arms[0]?.description,
|
|
68
|
+
bDescription: compare.bDescription ?? compare.arms[1]?.description,
|
|
69
|
+
aRubric: compare.aRubric ?? compare.arms[0]?.rubric,
|
|
70
|
+
bRubric: compare.bRubric ?? compare.arms[1]?.rubric,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
const aLabel = compare.aLabel ?? "control";
|
|
74
|
+
const bLabel = compare.bLabel ?? "experimental";
|
|
75
|
+
const arms = [
|
|
76
|
+
{
|
|
77
|
+
id: "a",
|
|
78
|
+
label: aLabel,
|
|
79
|
+
description: compare.aDescription,
|
|
80
|
+
rubric: compare.aRubric,
|
|
81
|
+
trace: compare.aTrace,
|
|
82
|
+
durationMs: compare.aDurationMs,
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
id: "b",
|
|
86
|
+
label: bLabel,
|
|
87
|
+
description: compare.bDescription,
|
|
88
|
+
rubric: compare.bRubric,
|
|
89
|
+
trace: compare.bTrace,
|
|
90
|
+
durationMs: compare.bDurationMs,
|
|
91
|
+
},
|
|
92
|
+
];
|
|
93
|
+
return {
|
|
94
|
+
arms,
|
|
95
|
+
faster: resolveCompareMetricPairs(compare.faster, ["a", "b"]),
|
|
96
|
+
cheaper: resolveCompareMetricPairs(compare.cheaper, ["a", "b"]),
|
|
97
|
+
aLabel,
|
|
98
|
+
bLabel,
|
|
99
|
+
aDescription: compare.aDescription,
|
|
100
|
+
bDescription: compare.bDescription,
|
|
101
|
+
aRubric: compare.aRubric,
|
|
102
|
+
bRubric: compare.bRubric,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
41
105
|
/** Rubric checks in one short line each. */
|
|
42
106
|
export function describeRubricChecks(rubric, compare) {
|
|
43
107
|
if (!rubric) {
|
|
@@ -81,39 +145,31 @@ export function describeRubricChecks(rubric, compare) {
|
|
|
81
145
|
lines.push(`announce review depth ${rubric.reviewDepth}`);
|
|
82
146
|
}
|
|
83
147
|
if (compare) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
}
|
|
91
|
-
if (compare.faster) {
|
|
92
|
-
const winner = compare.faster === "a" ? compare.aLabel : compare.bLabel;
|
|
93
|
-
const loser = compare.faster === "a" ? compare.bLabel : compare.aLabel;
|
|
94
|
-
lines.push(`${winner} is faster than ${loser}`);
|
|
148
|
+
const normalized = normalizeStoryCompare(compare);
|
|
149
|
+
lines.push(compareHeading(normalized.arms.map((arm) => arm.label)));
|
|
150
|
+
for (const arm of normalized.arms) {
|
|
151
|
+
if (arm.description) {
|
|
152
|
+
lines.push(`${arm.label}: ${arm.description}`);
|
|
153
|
+
}
|
|
95
154
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const loser = compare.cheaper === "a" ? compare.bLabel : compare.aLabel;
|
|
99
|
-
lines.push(`${winner} uses fewer tokens than ${loser}`);
|
|
155
|
+
for (const pair of normalized.faster ?? []) {
|
|
156
|
+
lines.push(`${labelForArm(normalized.arms, pair.winner)} is faster than ${labelForArm(normalized.arms, pair.loser)}`);
|
|
100
157
|
}
|
|
101
|
-
for (const
|
|
102
|
-
|
|
103
|
-
continue;
|
|
104
|
-
}
|
|
105
|
-
lines.push(`${compare.aLabel}: ${line}`);
|
|
158
|
+
for (const pair of normalized.cheaper ?? []) {
|
|
159
|
+
lines.push(`${labelForArm(normalized.arms, pair.winner)} uses fewer tokens than ${labelForArm(normalized.arms, pair.loser)}`);
|
|
106
160
|
}
|
|
107
|
-
for (const
|
|
108
|
-
|
|
109
|
-
|
|
161
|
+
for (const arm of normalized.arms) {
|
|
162
|
+
for (const line of describeRubricChecks(arm.rubric)) {
|
|
163
|
+
if (line === "no rubric checks" || line === "no rubric recorded") {
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
lines.push(`${arm.label}: ${line}`);
|
|
110
167
|
}
|
|
111
|
-
lines.push(`${compare.bLabel}: ${line}`);
|
|
112
168
|
}
|
|
113
169
|
}
|
|
114
170
|
if (rubric.judge && rubric.judge.length > 0) {
|
|
115
171
|
lines.push(compare
|
|
116
|
-
? `judge answers ${countLabel(rubric.judge.length, "question", "questions")} on both arms`
|
|
172
|
+
? `judge answers ${countLabel(rubric.judge.length, "question", "questions")} on ${normalizeStoryCompare(compare).arms.length === 2 ? "both arms" : "every arm"}`
|
|
117
173
|
: `judge answers ${countLabel(rubric.judge.length, "question", "questions")}`);
|
|
118
174
|
}
|
|
119
175
|
return lines.length > 0 ? lines : ["no rubric checks"];
|
|
@@ -188,28 +244,19 @@ function storyVerdict(happened, outcome) {
|
|
|
188
244
|
return outcome.filter((line) => !REDUNDANT_VERDICT.has(line) && !happened.includes(line));
|
|
189
245
|
}
|
|
190
246
|
export function buildScenarioStory(options) {
|
|
247
|
+
const compare = options.compare ? normalizeStoryCompare(options.compare) : undefined;
|
|
191
248
|
const result = options.skipped
|
|
192
249
|
? ["scenario skipped"]
|
|
193
|
-
:
|
|
250
|
+
: compare
|
|
194
251
|
? [
|
|
195
|
-
...describeTraceHappened(
|
|
196
|
-
...
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
durationMs: options.compare.aDurationMs,
|
|
204
|
-
},
|
|
205
|
-
b: {
|
|
206
|
-
id: "b",
|
|
207
|
-
label: options.compare.bLabel,
|
|
208
|
-
prompt: "",
|
|
209
|
-
trace: options.compare.bTrace,
|
|
210
|
-
durationMs: options.compare.bDurationMs,
|
|
211
|
-
},
|
|
212
|
-
}),
|
|
252
|
+
...compare.arms.flatMap((arm) => describeTraceHappened(arm.trace).map((line) => `${arm.label}: ${line}`)),
|
|
253
|
+
...describeCompareOutcome(buildCompareResult(compare.arms.map((arm) => ({
|
|
254
|
+
id: arm.id,
|
|
255
|
+
label: arm.label,
|
|
256
|
+
prompt: "",
|
|
257
|
+
trace: arm.trace,
|
|
258
|
+
durationMs: arm.durationMs,
|
|
259
|
+
})), { faster: compare.faster, cheaper: compare.cheaper })),
|
|
213
260
|
]
|
|
214
261
|
: describeTraceHappened(options.trace);
|
|
215
262
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario-story.js","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAS/D,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AAC9F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QACrC,OAAO,IAAI,OAAO,GAAG,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAyC;IACrE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,QAAgB,EAAE,MAAc;IAClE,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,oBAAoB,CACnC,MAAuB,EACvB,OASC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,kBAAkB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,0BAA0B,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACxE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,mBAAmB,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACzE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACxE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;gBAClE,SAAS;YACV,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;gBAClE,SAAS;YACV,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CACT,OAAO;YACN,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,eAAe;YAC1F,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAC9E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACxD,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,eAAe,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;SACnC,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACpD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,IAAI,EAAE,EAAE,CACtF,eAAe,CAAC,IAAI,CAAC,CACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IACjD,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,mBAAmB,OAAO,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC9E,OAAO,kBAAkB,OAAO,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;QAC5C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAK/B;IACA,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,SAAS,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpE,SAAS,YAAY,CAAC,QAAkB,EAAE,OAAiB;IAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAqBlC;IACA,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;QAC7B,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC,OAAO;YAChB,CAAC,CAAC;gBACA,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CACnD,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,CAC/C;gBACD,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CACnD,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,CAC/C;gBACD,GAAG,sBAAsB,CAAC;oBACzB,CAAC,EAAE;wBACF,EAAE,EAAE,GAAG;wBACP,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW;qBACvC;oBACD,CAAC,EAAE;wBACF,EAAE,EAAE,GAAG;wBACP,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;wBAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW;qBACvC;iBACD,CAAC;aACF;YACF,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO;QACN,QAAQ,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM;QACN,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;KACvD,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"scenario-story.js","sourceRoot":"","sources":["../src/scenario-story.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,kBAAkB,EAGlB,sBAAsB,EACtB,yBAAyB,GACzB,MAAM,uBAAuB,CAAC;AAS/B,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AAC9F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QACrC,OAAO,IAAI,OAAO,GAAG,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,MAAM,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAyC;IACrE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,QAAgB,EAAE,MAAc;IAClE,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAiBD,SAAS,cAAc,CAAC,MAAgB;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,WAAW,MAAM,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,OAAO,WAAW,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB,EAAE,EAAU;IACvD,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;AACvD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA0B;IACxD,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjD,OAAO;YACN,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;gBACpC,CAAC,CAAC,OAAO,CAAC,MAAM;gBAChB,CAAC,CAAC,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC;YACpD,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;gBACtC,CAAC,CAAC,OAAO,CAAC,OAAO;gBACjB,CAAC,CAAC,yBAAyB,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC;YACrD,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;YAC7D,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,cAAc;YAClE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW;YAClE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW;YAClE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM;YACnD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM;SACnD,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC;IAChD,MAAM,IAAI,GAAsB;QAC/B;YACC,EAAE,EAAE,GAAG;YACP,KAAK,EAAE,MAAM;YACb,WAAW,EAAE,OAAO,CAAC,YAAY;YACjC,MAAM,EAAE,OAAO,CAAC,OAAO;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,UAAU,EAAE,OAAO,CAAC,WAAW;SAC/B;QACD;YACC,EAAE,EAAE,GAAG;YACP,KAAK,EAAE,MAAM;YACb,WAAW,EAAE,OAAO,CAAC,YAAY;YACjC,MAAM,EAAE,OAAO,CAAC,OAAO;YACvB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,UAAU,EAAE,OAAO,CAAC,WAAW;SAC/B;KACD,CAAC;IACF,OAAO;QACN,IAAI;QACJ,MAAM,EAAE,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC7D,OAAO,EAAE,yBAAyB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC/D,MAAM;QACN,MAAM;QACN,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;KACxB,CAAC;AACH,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,oBAAoB,CACnC,MAAuB,EACvB,OAA2B;IAE3B,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,kBAAkB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,0BAA0B,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC;QACF,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CACT,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CACzG,CAAC;QACH,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CACT,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,2BAA2B,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CACjH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;oBAClE,SAAS;gBACV,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CACT,OAAO;YACN,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,EAAE;YAChK,CAAC,CAAC,iBAAiB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAC9E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;AACxD,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,eAAe,CAAC,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;SACnC,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACpD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,kBAAkB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,IAAI,EAAE,EAAE,CACtF,eAAe,CAAC,IAAI,CAAC,CACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IACjD,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,mBAAmB,OAAO,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QAC1C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC9E,OAAO,kBAAkB,OAAO,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,iBAAiB,EAAE,CAAC;QAC5C,OAAO,sBAAsB,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,eAAe,CAAC,OAK/B;IACA,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,SAAS,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpE,SAAS,YAAY,CAAC,QAAkB,EAAE,OAAiB;IAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAQlC;IACA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;QAC7B,CAAC,CAAC,CAAC,kBAAkB,CAAC;QACtB,CAAC,CAAC,OAAO;YACR,CAAC,CAAC;gBACA,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC/B,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC,CACvE;gBACD,GAAG,sBAAsB,CACxB,kBAAkB,CACjB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC1B,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,UAAU,EAAE,GAAG,CAAC,UAAU;iBAC1B,CAAC,CAAC,EACH,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CACpD,CACD;aACD;YACF,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO;QACN,QAAQ,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM;QACN,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;KACvD,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -37,10 +37,23 @@ export interface ScenarioRubric {
|
|
|
37
37
|
/** Any transcript outcome. The judge sees assistant text, tool args, and tool results. */
|
|
38
38
|
judge?: JudgeRubricItem[];
|
|
39
39
|
}
|
|
40
|
-
|
|
40
|
+
/** Arm id. Two-arm form uses `a` and `b`. Named arms use a lowercase slug. */
|
|
41
|
+
export type CompareArmId = string;
|
|
42
|
+
/** Named winner-versus-loser pair for a metric gate. */
|
|
43
|
+
export interface CompareMetricPair {
|
|
44
|
+
winner: CompareArmId;
|
|
45
|
+
loser: CompareArmId;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Two-arm form names the winner. Named-arm form lists winner-versus-loser pairs.
|
|
49
|
+
* A 2x2 must not require one arm to beat every other arm.
|
|
50
|
+
*/
|
|
51
|
+
export type CompareMetricGate = CompareArmId | CompareMetricPair[];
|
|
41
52
|
/** Overrides for one side of a compare scenario. Omitted fields inherit the scenario. */
|
|
42
53
|
export interface CompareArm {
|
|
43
|
-
/**
|
|
54
|
+
/** Required on `compare.arms`. Implicit `a` / `b` in the two-arm form. */
|
|
55
|
+
id?: CompareArmId;
|
|
56
|
+
/** Display name. Omitted labels are control (a), experimental (b), or the arm id. */
|
|
44
57
|
label?: string;
|
|
45
58
|
/** Required. Says what this arm tests. */
|
|
46
59
|
description?: string;
|
|
@@ -61,17 +74,26 @@ export interface CompareArm {
|
|
|
61
74
|
rubric?: ScenarioRubric;
|
|
62
75
|
}
|
|
63
76
|
/**
|
|
64
|
-
* Two live arms in one scenario.
|
|
77
|
+
* Two or more live arms in one scenario.
|
|
78
|
+
* Keep `a` and `b` for the two-arm form. Use `arms` when the scenario has more than two workspaces.
|
|
65
79
|
* `faster` and `cheaper` are optional metric gates.
|
|
66
|
-
* `rubric.judge` is optional and scores
|
|
80
|
+
* `rubric.judge` is optional and scores every arm transcript.
|
|
67
81
|
*/
|
|
68
82
|
export interface ScenarioCompare {
|
|
69
|
-
a
|
|
70
|
-
b
|
|
71
|
-
/** Named
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
|
|
83
|
+
a?: CompareArm;
|
|
84
|
+
b?: CompareArm;
|
|
85
|
+
/** Named arms. Each row needs `id`, `description`, and a workspace override when the trees differ. */
|
|
86
|
+
arms?: CompareArm[];
|
|
87
|
+
/**
|
|
88
|
+
* Two-arm form: named arm must have a shorter duration than the other arm.
|
|
89
|
+
* Named-arm form: winner-versus-loser pairs.
|
|
90
|
+
*/
|
|
91
|
+
faster?: CompareMetricGate;
|
|
92
|
+
/**
|
|
93
|
+
* Two-arm form: named arm must use fewer total tokens than the other arm.
|
|
94
|
+
* Named-arm form: winner-versus-loser pairs.
|
|
95
|
+
*/
|
|
96
|
+
cheaper?: CompareMetricGate;
|
|
75
97
|
}
|
|
76
98
|
export interface CompareArmResult {
|
|
77
99
|
id: CompareArmId;
|
|
@@ -83,14 +105,22 @@ export interface CompareArmResult {
|
|
|
83
105
|
durationMs?: number;
|
|
84
106
|
}
|
|
85
107
|
export interface ScenarioCompareResult {
|
|
86
|
-
|
|
87
|
-
|
|
108
|
+
/** Every arm in author order. */
|
|
109
|
+
arms: CompareArmResult[];
|
|
110
|
+
/** Present when an arm id is `a`. */
|
|
111
|
+
a?: CompareArmResult;
|
|
112
|
+
/** Present when an arm id is `b`. */
|
|
113
|
+
b?: CompareArmResult;
|
|
114
|
+
/** Resolved faster pairs after load. */
|
|
115
|
+
faster?: CompareMetricPair[];
|
|
116
|
+
/** Resolved cheaper pairs after load. */
|
|
117
|
+
cheaper?: CompareMetricPair[];
|
|
88
118
|
}
|
|
89
119
|
export interface AgentScenario {
|
|
90
120
|
name: string;
|
|
91
121
|
/** Plain-language note. Says what this scenario tests. */
|
|
92
122
|
description?: string;
|
|
93
|
-
/** Run two arms. Optional metric gates and an optional
|
|
123
|
+
/** Run two or more arms. Optional metric gates and an optional shared judge. */
|
|
94
124
|
compare?: ScenarioCompare;
|
|
95
125
|
prompt: string;
|
|
96
126
|
host?: AgentHost;
|
|
@@ -210,7 +240,7 @@ export interface ScenarioResult {
|
|
|
210
240
|
description?: string;
|
|
211
241
|
/** Full agent transcript when available (for HTML reports / debug bundles). */
|
|
212
242
|
trace?: AgentTrace;
|
|
213
|
-
/**
|
|
243
|
+
/** Compare arms when this scenario is a compare run. */
|
|
214
244
|
compare?: ScenarioCompareResult;
|
|
215
245
|
/** Plain-language summary of the check, the agent run, and the verdict. */
|
|
216
246
|
story?: ScenarioStory;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,qHAAqH;IACrH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,0FAA0F;IAC1F,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,MAAM,MAAM,YAAY,GAAG,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EACX,UAAU,EACV,eAAe,EACf,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,qHAAqH;IACrH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,0FAA0F;IAC1F,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,8EAA8E;AAC9E,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,wDAAwD;AACxD,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,YAAY,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG,iBAAiB,EAAE,CAAC;AAEnE,yFAAyF;AACzF,MAAM,WAAW,UAAU;IAC1B,0EAA0E;IAC1E,EAAE,CAAC,EAAE,YAAY,CAAC;IAClB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;CACxB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC/B,CAAC,CAAC,EAAE,UAAU,CAAC;IACf,CAAC,CAAC,EAAE,UAAU,CAAC;IACf,sGAAsG;IACtG,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB;;;OAGG;IACH,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B;;;OAGG;IACH,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,YAAY,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACrC,iCAAiC;IACjC,IAAI,EAAE,gBAAgB,EAAE,CAAC;IACzB,qCAAqC;IACrC,CAAC,CAAC,EAAE,gBAAgB,CAAC;IACrB,qCAAqC;IACrC,CAAC,CAAC,EAAE,gBAAgB,CAAC;IACrB,wCAAwC;IACxC,MAAM,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC7B,yCAAyC;IACzC,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,wFAAwF;IACxF,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sFAAsF;IACtF,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,mDAAmD;IACnD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,gFAAgF;IAChF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,SAAS,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,eAAe,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,eAAe,GACxB,aAAa,GACb,aAAa,GACb,aAAa,GACb,eAAe,GACf,eAAe,GACf,iBAAiB,CAAC;AAErB,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,oDAAoD;IACpD,OAAO,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,uDAAuD;IACvD,KAAK,CAAC,EAAE,sBAAsB,CAAC;IAC/B,oCAAoC;IACpC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,kDAAkD;IAClD,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,wDAAwD;IACxD,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAChC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,wBAAwB,EAAE,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,UAAU,CAAC;CACrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EACX,aAAa,EACb,cAAc,EAKd,MAAM,YAAY,CAAC;AA2DpB,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AA2kBD,kEAAkE;AAClE,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,cAAc,GACnB,oBAAoB,EAAE,CAQxB;AAED,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3E,OAAO,CAAC,qBAAqB,CAAC,CAyHhC;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAY5E;AAED,mFAAmF;AACnF,wBAAgB,2BAA2B,CAC1C,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,EACvC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,GACxC,MAAM,GAAG,SAAS,CAIpB"}
|
package/dist/validate-suite.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { access, stat } from "node:fs/promises";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { isHostSlug, isKnownAgentHost, isRepoRelativeSkillPath, parseScenarioWorkspace, skillManifestRelPath, skillPathsFromSetting, } from "@post-print/agent-harness";
|
|
4
|
-
import { applyCompareArm } from "./compare-scenario.js";
|
|
4
|
+
import { applyCompareArm, parseCompareArmId, resolveCompareArms } from "./compare-scenario.js";
|
|
5
5
|
import { loadSuiteFile } from "./load-suite.js";
|
|
6
6
|
const REPLAY_DEPRECATION = "Replay-based testing is deprecated and no longer supported; use Cursor, Claude, or OpenAI.";
|
|
7
7
|
function isKnownHostId(host) {
|
|
@@ -180,30 +180,108 @@ function validateCompare(issues, suitePath, scenario) {
|
|
|
180
180
|
return;
|
|
181
181
|
}
|
|
182
182
|
if (typeof scenario.compare !== "object" || scenario.compare === null) {
|
|
183
|
-
pushIssue(issues, suitePath, "compare", "compare must be an object with a and b", scenario.name);
|
|
183
|
+
pushIssue(issues, suitePath, "compare", "compare must be an object with a and b, or arms", scenario.name);
|
|
184
184
|
return;
|
|
185
185
|
}
|
|
186
|
-
|
|
187
|
-
|
|
186
|
+
const compare = scenario.compare;
|
|
187
|
+
const hasLegacyArms = compare.a !== undefined || compare.b !== undefined;
|
|
188
|
+
const hasNamedArms = compare.arms !== undefined;
|
|
189
|
+
if (hasLegacyArms && hasNamedArms) {
|
|
190
|
+
pushIssue(issues, suitePath, "compare", "compare must use a and b, or arms, not both", scenario.name);
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
if (hasNamedArms) {
|
|
194
|
+
validateNamedCompareArms(issues, suitePath, scenario.name, compare);
|
|
188
195
|
}
|
|
189
196
|
else {
|
|
190
|
-
|
|
197
|
+
if (typeof compare.a !== "object" || compare.a === null) {
|
|
198
|
+
pushIssue(issues, suitePath, "compare.a", "compare.a must be an object", scenario.name);
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
validateCompareArmFields(issues, suitePath, scenario.name, "compare.a", compare.a);
|
|
202
|
+
}
|
|
203
|
+
if (typeof compare.b !== "object" || compare.b === null) {
|
|
204
|
+
pushIssue(issues, suitePath, "compare.b", "compare.b must be an object", scenario.name);
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
validateCompareArmFields(issues, suitePath, scenario.name, "compare.b", compare.b);
|
|
208
|
+
}
|
|
191
209
|
}
|
|
192
|
-
|
|
193
|
-
|
|
210
|
+
const armIds = resolveCompareArms(compare)
|
|
211
|
+
.map((entry) => entry.id)
|
|
212
|
+
.filter(Boolean);
|
|
213
|
+
validateCompareMetricGate(issues, suitePath, scenario.name, "compare.faster", compare.faster, armIds);
|
|
214
|
+
validateCompareMetricGate(issues, suitePath, scenario.name, "compare.cheaper", compare.cheaper, armIds);
|
|
215
|
+
}
|
|
216
|
+
function validateNamedCompareArms(issues, suitePath, scenarioName, compare) {
|
|
217
|
+
if (!Array.isArray(compare.arms) || compare.arms.length < 2) {
|
|
218
|
+
pushIssue(issues, suitePath, "compare.arms", "compare.arms must be an array of two or more arms", scenarioName);
|
|
219
|
+
return;
|
|
194
220
|
}
|
|
195
|
-
|
|
196
|
-
|
|
221
|
+
const seen = new Set();
|
|
222
|
+
for (const [index, arm] of compare.arms.entries()) {
|
|
223
|
+
const fieldPrefix = `compare.arms[${index}]`;
|
|
224
|
+
if (typeof arm !== "object" || arm === null) {
|
|
225
|
+
pushIssue(issues, suitePath, fieldPrefix, "arm must be an object", scenarioName);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
const id = parseCompareArmId(arm.id);
|
|
229
|
+
if (!id) {
|
|
230
|
+
pushIssue(issues, suitePath, `${fieldPrefix}.id`, "id must be a lowercase slug such as skel-clean", scenarioName);
|
|
231
|
+
}
|
|
232
|
+
else if (seen.has(id)) {
|
|
233
|
+
pushIssue(issues, suitePath, `${fieldPrefix}.id`, `duplicate arm id ${id}`, scenarioName);
|
|
234
|
+
}
|
|
235
|
+
else {
|
|
236
|
+
seen.add(id);
|
|
237
|
+
}
|
|
238
|
+
validateCompareArmFields(issues, suitePath, scenarioName, fieldPrefix, arm);
|
|
197
239
|
}
|
|
198
|
-
validateCompareArmPick(issues, suitePath, scenario.name, "compare.faster", scenario.compare.faster);
|
|
199
|
-
validateCompareArmPick(issues, suitePath, scenario.name, "compare.cheaper", scenario.compare.cheaper);
|
|
200
240
|
}
|
|
201
|
-
function
|
|
241
|
+
function validateCompareMetricGate(issues, suitePath, scenarioName, field, value, armIds) {
|
|
202
242
|
if (value === undefined) {
|
|
203
243
|
return;
|
|
204
244
|
}
|
|
205
|
-
if (
|
|
206
|
-
|
|
245
|
+
if (typeof value === "string") {
|
|
246
|
+
if (armIds.length !== 2) {
|
|
247
|
+
pushIssue(issues, suitePath, field, `${field} must be an array of { winner, loser } pairs when compare has more than two arms`, scenarioName);
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
if (!armIds.includes(value)) {
|
|
251
|
+
pushIssue(issues, suitePath, field, `${field} must be ${armIds.map((id) => JSON.stringify(id)).join(" or ")}`, scenarioName);
|
|
252
|
+
}
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
256
|
+
pushIssue(issues, suitePath, field, armIds.length === 2
|
|
257
|
+
? `${field} must be ${armIds.map((id) => JSON.stringify(id)).join(" or ")} or an array of { winner, loser } pairs`
|
|
258
|
+
: `${field} must be an array of { winner, loser } pairs`, scenarioName);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
const seen = new Set();
|
|
262
|
+
for (const [index, pair] of value.entries()) {
|
|
263
|
+
if (typeof pair !== "object" || pair === null) {
|
|
264
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "pair must be { winner, loser }", scenarioName);
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
const typed = pair;
|
|
268
|
+
validateMetricPairEnd(issues, suitePath, scenarioName, `${field}[${index}].winner`, typed.winner, armIds);
|
|
269
|
+
validateMetricPairEnd(issues, suitePath, scenarioName, `${field}[${index}].loser`, typed.loser, armIds);
|
|
270
|
+
if (typeof typed.winner === "string" && typed.winner === typed.loser) {
|
|
271
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "winner and loser must be different arms", scenarioName);
|
|
272
|
+
}
|
|
273
|
+
if (typeof typed.winner === "string" && typeof typed.loser === "string") {
|
|
274
|
+
const key = `${typed.winner}>${typed.loser}`;
|
|
275
|
+
if (seen.has(key)) {
|
|
276
|
+
pushIssue(issues, suitePath, `${field}[${index}]`, "duplicate pair", scenarioName);
|
|
277
|
+
}
|
|
278
|
+
seen.add(key);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
function validateMetricPairEnd(issues, suitePath, scenarioName, field, value, armIds) {
|
|
283
|
+
if (typeof value !== "string" || !armIds.includes(value)) {
|
|
284
|
+
pushIssue(issues, suitePath, field, `must be one of ${armIds.join(", ")}`, scenarioName);
|
|
207
285
|
}
|
|
208
286
|
}
|
|
209
287
|
function validateHosts(issues, suitePath, suite) {
|
|
@@ -304,16 +382,10 @@ export async function validateSuitePaths(suitePaths, options) {
|
|
|
304
382
|
if (options?.validatePaths && repoRoot) {
|
|
305
383
|
for (const scenario of suite.scenarios) {
|
|
306
384
|
const workspaceChecks = scenario.compare
|
|
307
|
-
?
|
|
308
|
-
{
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
},
|
|
312
|
-
{
|
|
313
|
-
field: "compare.b.workspace",
|
|
314
|
-
rel: resolveScenarioWorkspaceRel(suite, applyCompareArm(scenario, "b")),
|
|
315
|
-
},
|
|
316
|
-
]
|
|
385
|
+
? resolveCompareArms(scenario.compare).map((entry) => ({
|
|
386
|
+
field: `compare.${entry.id}.workspace`,
|
|
387
|
+
rel: resolveScenarioWorkspaceRel(suite, applyCompareArm(scenario, entry.id)),
|
|
388
|
+
}))
|
|
317
389
|
: [{ field: "workspace", rel: resolveScenarioWorkspaceRel(suite, scenario) }];
|
|
318
390
|
for (const check of workspaceChecks) {
|
|
319
391
|
const workspaceRel = check.rel;
|
|
@@ -332,16 +404,10 @@ export async function validateSuitePaths(suitePaths, options) {
|
|
|
332
404
|
}
|
|
333
405
|
}
|
|
334
406
|
const skillTargets = scenario.compare
|
|
335
|
-
?
|
|
336
|
-
{
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
},
|
|
340
|
-
{
|
|
341
|
-
field: "compare.b.skills",
|
|
342
|
-
scenario: applyCompareArm(scenario, "b"),
|
|
343
|
-
},
|
|
344
|
-
]
|
|
407
|
+
? resolveCompareArms(scenario.compare).map((entry) => ({
|
|
408
|
+
field: `compare.${entry.id}.skills`,
|
|
409
|
+
scenario: applyCompareArm(scenario, entry.id),
|
|
410
|
+
}))
|
|
345
411
|
: [{ field: "skills", scenario }];
|
|
346
412
|
for (const target of skillTargets) {
|
|
347
413
|
const skills = target.scenario.skills ?? suite.defaults?.skills;
|