@post-print/agent-test 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -15
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +167 -11
- package/dist/cli.js.map +1 -1
- package/dist/compare.d.ts +74 -0
- package/dist/compare.d.ts.map +1 -0
- package/dist/compare.js +201 -0
- package/dist/compare.js.map +1 -0
- package/dist/debug-bundle.d.ts.map +1 -1
- package/dist/debug-bundle.js +10 -1
- package/dist/debug-bundle.js.map +1 -1
- package/dist/expect.d.ts +4 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +44 -0
- package/dist/expect.js.map +1 -1
- package/dist/html-report.d.ts +13 -0
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +368 -18
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/record-trace.d.ts.map +1 -1
- package/dist/record-trace.js +1 -0
- package/dist/record-trace.js.map +1 -1
- package/dist/run-suite.d.ts +3 -0
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +162 -70
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-retry.d.ts +15 -0
- package/dist/scenario-retry.d.ts.map +1 -0
- package/dist/scenario-retry.js +52 -0
- package/dist/scenario-retry.js.map +1 -0
- package/dist/suite-summary.d.ts +6 -10
- package/dist/suite-summary.d.ts.map +1 -1
- package/dist/suite-summary.js +93 -12
- package/dist/suite-summary.js.map +1 -1
- package/dist/types.d.ts +33 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +18 -4
- package/dist/validate-suite.js.map +1 -1
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"suite-summary.d.ts","sourceRoot":"","sources":["../src/suite-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"suite-summary.d.ts","sourceRoot":"","sources":["../src/suite-summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,gBAAgB,EAEhB,UAAU,EACV,cAAc,EACd,cAAc,EACd,UAAU,EACV,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,UAAU,GAAG,YAAY,CAAC;AAiB3D,0EAA0E;AAC1E,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOxF;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,UAAU,GAAG,SAAS,CAgDhF;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,GAAG,UAAU,CAyB1E;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,UAAU,CAiBtE;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,UAAU,CAc5E;AAED,sFAAsF;AACtF,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAS1F;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAkB1D;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAe5D"}
|
package/dist/suite-summary.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const INFRA_CATEGORIES = new Set(["judge_infra", "agent_runtime"]);
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
function emptyRunSummary() {
|
|
3
|
+
return {
|
|
4
4
|
infraFailures: 0,
|
|
5
5
|
rubricFailures: 0,
|
|
6
6
|
agentRuntimeFailures: 0,
|
|
@@ -8,7 +8,66 @@ export function summarizeFailures(failures) {
|
|
|
8
8
|
recordingErrors: 0,
|
|
9
9
|
judgeParseFailures: 0,
|
|
10
10
|
retriedScenarios: 0,
|
|
11
|
+
scenarioRetriedScenarios: 0,
|
|
11
12
|
};
|
|
13
|
+
}
|
|
14
|
+
/** Nearest-rank percentile (p in 0–100) over a sorted ascending array. */
|
|
15
|
+
export function percentileNearestRank(sortedAsc, p) {
|
|
16
|
+
if (sortedAsc.length === 0) {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
const clamped = Math.min(100, Math.max(0, p));
|
|
20
|
+
const rank = Math.ceil((clamped / 100) * sortedAsc.length) - 1;
|
|
21
|
+
return sortedAsc[Math.max(0, Math.min(sortedAsc.length - 1, rank))];
|
|
22
|
+
}
|
|
23
|
+
export function summarizeUsage(results) {
|
|
24
|
+
const totals = [];
|
|
25
|
+
let scenariosWithUsage = 0;
|
|
26
|
+
let sumTotal = 0;
|
|
27
|
+
let sumInput = 0;
|
|
28
|
+
let sumOutput = 0;
|
|
29
|
+
let sawInput = false;
|
|
30
|
+
let sawOutput = false;
|
|
31
|
+
for (const result of results) {
|
|
32
|
+
const usage = result.usage ?? result.trace?.usage;
|
|
33
|
+
if (!usage) {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
scenariosWithUsage++;
|
|
37
|
+
const total = usage.totalTokens;
|
|
38
|
+
if (typeof total === "number" && Number.isFinite(total)) {
|
|
39
|
+
totals.push(total);
|
|
40
|
+
sumTotal += total;
|
|
41
|
+
}
|
|
42
|
+
if (typeof usage.inputTokens === "number" && Number.isFinite(usage.inputTokens)) {
|
|
43
|
+
sumInput += usage.inputTokens;
|
|
44
|
+
sawInput = true;
|
|
45
|
+
}
|
|
46
|
+
if (typeof usage.outputTokens === "number" && Number.isFinite(usage.outputTokens)) {
|
|
47
|
+
sumOutput += usage.outputTokens;
|
|
48
|
+
sawOutput = true;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (scenariosWithUsage === 0) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
totals.sort((a, b) => a - b);
|
|
55
|
+
const stats = { scenariosWithUsage };
|
|
56
|
+
if (totals.length > 0) {
|
|
57
|
+
stats.sumTotalTokens = sumTotal;
|
|
58
|
+
stats.p50TotalTokens = percentileNearestRank(totals, 50);
|
|
59
|
+
stats.p95TotalTokens = percentileNearestRank(totals, 95);
|
|
60
|
+
}
|
|
61
|
+
if (sawInput) {
|
|
62
|
+
stats.sumInputTokens = sumInput;
|
|
63
|
+
}
|
|
64
|
+
if (sawOutput) {
|
|
65
|
+
stats.sumOutputTokens = sumOutput;
|
|
66
|
+
}
|
|
67
|
+
return stats;
|
|
68
|
+
}
|
|
69
|
+
export function summarizeFailures(failures) {
|
|
70
|
+
const summary = emptyRunSummary();
|
|
12
71
|
for (const failure of failures) {
|
|
13
72
|
switch (failure.category) {
|
|
14
73
|
case "judge_infra":
|
|
@@ -34,15 +93,8 @@ export function summarizeFailures(failures) {
|
|
|
34
93
|
return summary;
|
|
35
94
|
}
|
|
36
95
|
export function summarizeReports(reports) {
|
|
37
|
-
const combined =
|
|
38
|
-
|
|
39
|
-
rubricFailures: 0,
|
|
40
|
-
agentRuntimeFailures: 0,
|
|
41
|
-
worktreeLeaks: 0,
|
|
42
|
-
recordingErrors: 0,
|
|
43
|
-
judgeParseFailures: 0,
|
|
44
|
-
retriedScenarios: 0,
|
|
45
|
-
};
|
|
96
|
+
const combined = emptyRunSummary();
|
|
97
|
+
const allResults = [];
|
|
46
98
|
for (const report of reports) {
|
|
47
99
|
const partial = report.summary ?? summarizeReportResults(report.results);
|
|
48
100
|
combined.infraFailures += partial.infraFailures;
|
|
@@ -52,13 +104,18 @@ export function summarizeReports(reports) {
|
|
|
52
104
|
combined.recordingErrors += partial.recordingErrors;
|
|
53
105
|
combined.judgeParseFailures += partial.judgeParseFailures;
|
|
54
106
|
combined.retriedScenarios += partial.retriedScenarios;
|
|
107
|
+
combined.scenarioRetriedScenarios += partial.scenarioRetriedScenarios;
|
|
108
|
+
allResults.push(...report.results);
|
|
55
109
|
}
|
|
110
|
+
combined.usage = summarizeUsage(allResults);
|
|
56
111
|
return combined;
|
|
57
112
|
}
|
|
58
113
|
export function summarizeReportResults(results) {
|
|
59
114
|
const allFailures = results.flatMap((result) => result.failures);
|
|
60
115
|
const summary = summarizeFailures(allFailures);
|
|
61
116
|
summary.retriedScenarios = results.reduce((count, result) => count + (result.judgeVerdicts?.some((verdict) => (verdict.attempt ?? 1) > 1) ? 1 : 0), 0);
|
|
117
|
+
summary.scenarioRetriedScenarios = results.reduce((count, result) => count + ((result.attempts ?? 1) > 1 ? 1 : 0), 0);
|
|
118
|
+
summary.usage = summarizeUsage(results);
|
|
62
119
|
return summary;
|
|
63
120
|
}
|
|
64
121
|
/** Whether a failed scenario should fail the process under the given fail-on mode. */
|
|
@@ -72,8 +129,27 @@ export function shouldFailScenario(failures, mode) {
|
|
|
72
129
|
// behavior: ignore pure infra flakes
|
|
73
130
|
return failures.some((failure) => !INFRA_CATEGORIES.has(failure.category));
|
|
74
131
|
}
|
|
132
|
+
export function formatUsageStats(usage) {
|
|
133
|
+
const parts = [`usage_n=${usage.scenariosWithUsage}`];
|
|
134
|
+
if (usage.sumTotalTokens !== undefined) {
|
|
135
|
+
parts.push(`tokens_sum=${usage.sumTotalTokens}`);
|
|
136
|
+
}
|
|
137
|
+
if (usage.p50TotalTokens !== undefined) {
|
|
138
|
+
parts.push(`p50=${usage.p50TotalTokens}`);
|
|
139
|
+
}
|
|
140
|
+
if (usage.p95TotalTokens !== undefined) {
|
|
141
|
+
parts.push(`p95=${usage.p95TotalTokens}`);
|
|
142
|
+
}
|
|
143
|
+
if (usage.sumInputTokens !== undefined) {
|
|
144
|
+
parts.push(`in=${usage.sumInputTokens}`);
|
|
145
|
+
}
|
|
146
|
+
if (usage.sumOutputTokens !== undefined) {
|
|
147
|
+
parts.push(`out=${usage.sumOutputTokens}`);
|
|
148
|
+
}
|
|
149
|
+
return parts.join(" · ");
|
|
150
|
+
}
|
|
75
151
|
export function formatRunSummary(summary) {
|
|
76
|
-
|
|
152
|
+
const base = [
|
|
77
153
|
`summary: rubric=${summary.rubricFailures}`,
|
|
78
154
|
`infra=${summary.infraFailures}`,
|
|
79
155
|
`runtime=${summary.agentRuntimeFailures}`,
|
|
@@ -81,6 +157,11 @@ export function formatRunSummary(summary) {
|
|
|
81
157
|
`worktree=${summary.worktreeLeaks}`,
|
|
82
158
|
`recording=${summary.recordingErrors}`,
|
|
83
159
|
`retried=${summary.retriedScenarios}`,
|
|
160
|
+
`scenario_retried=${summary.scenarioRetriedScenarios}`,
|
|
84
161
|
].join(" · ");
|
|
162
|
+
if (!summary.usage) {
|
|
163
|
+
return base;
|
|
164
|
+
}
|
|
165
|
+
return `${base}\n${formatUsageStats(summary.usage)}`;
|
|
85
166
|
}
|
|
86
167
|
//# sourceMappingURL=suite-summary.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"suite-summary.js","sourceRoot":"","sources":["../src/suite-summary.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"suite-summary.js","sourceRoot":"","sources":["../src/suite-summary.ts"],"names":[],"mappings":"AAaA,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAkB,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC;AAEpF,SAAS,eAAe;IACvB,OAAO;QACN,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,oBAAoB,EAAE,CAAC;QACvB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;QAClB,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAC;QACnB,wBAAwB,EAAE,CAAC;KAC3B,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,qBAAqB,CAAC,SAAmB,EAAE,CAAS;IACnE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAyB;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC;QAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,SAAS;QACV,CAAC;QACD,kBAAkB,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC;QAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,QAAQ,IAAI,KAAK,CAAC;QACnB,CAAC;QACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACjF,QAAQ,IAAI,KAAK,CAAC,WAAW,CAAC;YAC9B,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;YACnF,SAAS,IAAI,KAAK,CAAC,YAAY,CAAC;YAChC,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;IACF,CAAC;IAED,IAAI,kBAAkB,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAe,EAAE,kBAAkB,EAAE,CAAC;IACjD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC;QAChC,KAAK,CAAC,cAAc,GAAG,qBAAqB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzD,KAAK,CAAC,cAAc,GAAG,qBAAqB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC;IACjC,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACf,KAAK,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAA4B;IAC7D,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;IAClC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1B,KAAK,aAAa;gBACjB,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,MAAM;YACP,KAAK,aAAa;gBACjB,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBAC7B,MAAM;YACP,KAAK,eAAe;gBACnB,OAAO,CAAC,oBAAoB,EAAE,CAAC;gBAC/B,MAAM;YACP,KAAK,eAAe;gBACnB,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,MAAM;YACP,KAAK,iBAAiB;gBACrB,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC1B,MAAM;YACP;gBACC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACzB,MAAM;QACR,CAAC;IACF,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACzD,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzE,QAAQ,CAAC,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC;QAChD,QAAQ,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC;QAClD,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC;QAC9D,QAAQ,CAAC,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC;QAChD,QAAQ,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC;QACpD,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,kBAAkB,CAAC;QAC1D,QAAQ,CAAC,gBAAgB,IAAI,OAAO,CAAC,gBAAgB,CAAC;QACtD,QAAQ,CAAC,wBAAwB,IAAI,OAAO,CAAC,wBAAwB,CAAC;QACtE,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IACD,QAAQ,CAAC,KAAK,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC5C,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC/D,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAC/C,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,MAAM,CACxC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CACjB,KAAK,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtF,CAAC,CACD,CAAC;IACF,OAAO,CAAC,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAChD,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/D,CAAC,CACD,CAAC;IACF,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,kBAAkB,CAAC,QAA4B,EAAE,IAAgB;IAChF,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,qCAAqC;IACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAiB;IACjD,MAAM,KAAK,GAAG,CAAC,WAAW,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,IAAI,GAAG;QACZ,mBAAmB,OAAO,CAAC,cAAc,EAAE;QAC3C,SAAS,OAAO,CAAC,aAAa,EAAE;QAChC,WAAW,OAAO,CAAC,oBAAoB,EAAE;QACzC,eAAe,OAAO,CAAC,kBAAkB,EAAE;QAC3C,YAAY,OAAO,CAAC,aAAa,EAAE;QACnC,aAAa,OAAO,CAAC,eAAe,EAAE;QACtC,WAAW,OAAO,CAAC,gBAAgB,EAAE;QACrC,oBAAoB,OAAO,CAAC,wBAAwB,EAAE;KACtD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACd,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,GAAG,IAAI,KAAK,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;AACtD,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { AgentHost, AgentTrace, ContextProfile, McpServerConfig, SkillContextSetting } from "@post-print/agent-harness";
|
|
2
|
-
export type { McpServerConfig } from "@post-print/agent-harness";
|
|
1
|
+
import type { AgentHost, AgentTrace, AgentUsage, ContextProfile, McpServerConfig, SkillContextSetting } from "@post-print/agent-harness";
|
|
2
|
+
export type { AgentUsage, McpServerConfig } from "@post-print/agent-harness";
|
|
3
3
|
export type JudgeRubricItem = string | {
|
|
4
4
|
id?: string;
|
|
5
5
|
question: string;
|
|
@@ -17,6 +17,13 @@ export interface ScenarioRubric {
|
|
|
17
17
|
*/
|
|
18
18
|
mustCallTool?: string[];
|
|
19
19
|
mustNotCallTool?: string[];
|
|
20
|
+
/**
|
|
21
|
+
* Substring that must appear in a Read tool's JSON args (registry-first / path grounding).
|
|
22
|
+
* Hallucination scoring stays in live `judge` questions — these are lightweight proxies.
|
|
23
|
+
*/
|
|
24
|
+
mustReadPath?: string[];
|
|
25
|
+
/** Substring that must not appear in any Read tool's JSON args. */
|
|
26
|
+
mustNotReadPath?: string[];
|
|
20
27
|
/** Skill folder names the agent must read via SKILL.md (e.g. grill, crystallize). */
|
|
21
28
|
mustInvokeSkill?: string[];
|
|
22
29
|
mustNotInvokeSkill?: string[];
|
|
@@ -32,6 +39,8 @@ export interface AgentScenario {
|
|
|
32
39
|
profile?: ContextProfile;
|
|
33
40
|
/** Override suite defaults for skill catalog loading. */
|
|
34
41
|
skills?: SkillContextSetting;
|
|
42
|
+
/** Additive context paths (merged after suite defaults.contextSources). */
|
|
43
|
+
contextSources?: string[];
|
|
35
44
|
/** Inline MCP servers for live Cursor (merged over suite defaults by server name). */
|
|
36
45
|
mcpServers?: Record<string, McpServerConfig>;
|
|
37
46
|
/** Live-only: apply patch + commit in worktree so pr-mode branch diff exists. */
|
|
@@ -50,6 +59,11 @@ export interface AgentSuiteFile {
|
|
|
50
59
|
profile?: ContextProfile;
|
|
51
60
|
/** none | catalog | full — ambient-routing uses full for IDE parity. */
|
|
52
61
|
skills?: SkillContextSetting;
|
|
62
|
+
/**
|
|
63
|
+
* Additive repo-relative context paths (or `.skeleton/customize/` basenames).
|
|
64
|
+
* Use with `profile: "skeleton"` or to extend shared/cursor/claude profiles.
|
|
65
|
+
*/
|
|
66
|
+
contextSources?: string[];
|
|
53
67
|
/** Inline MCP servers for live Cursor runs. */
|
|
54
68
|
mcpServers?: Record<string, McpServerConfig>;
|
|
55
69
|
};
|
|
@@ -87,13 +101,26 @@ export interface ScenarioResult {
|
|
|
87
101
|
failures: AssertionFailure[];
|
|
88
102
|
skipped?: boolean;
|
|
89
103
|
durationMs: number;
|
|
104
|
+
/** Total live attempts including announce-stop retries (omit or 1 when no retry). */
|
|
105
|
+
attempts?: number;
|
|
90
106
|
/** LLM judge verdicts when judge criteria were evaluated. */
|
|
91
107
|
judgeVerdicts?: JudgeVerdictResult[];
|
|
108
|
+
/** Token usage when the host reported it (mirrored from trace.usage). */
|
|
109
|
+
usage?: AgentUsage;
|
|
92
110
|
/** Full agent transcript when available (for HTML reports / debug bundles). */
|
|
93
111
|
trace?: AgentTrace;
|
|
94
112
|
/** Absolute path to the debug bundle directory when --debug wrote one. */
|
|
95
113
|
debugBundleDir?: string;
|
|
96
114
|
}
|
|
115
|
+
/** Aggregate token usage across scenarios that reported it. */
|
|
116
|
+
export interface UsageStats {
|
|
117
|
+
scenariosWithUsage: number;
|
|
118
|
+
sumTotalTokens?: number;
|
|
119
|
+
p50TotalTokens?: number;
|
|
120
|
+
p95TotalTokens?: number;
|
|
121
|
+
sumInputTokens?: number;
|
|
122
|
+
sumOutputTokens?: number;
|
|
123
|
+
}
|
|
97
124
|
export interface RunSummary {
|
|
98
125
|
infraFailures: number;
|
|
99
126
|
rubricFailures: number;
|
|
@@ -101,7 +128,11 @@ export interface RunSummary {
|
|
|
101
128
|
worktreeLeaks: number;
|
|
102
129
|
recordingErrors: number;
|
|
103
130
|
judgeParseFailures: number;
|
|
131
|
+
/** Scenarios where the LLM judge used more than one attempt. */
|
|
104
132
|
retriedScenarios: number;
|
|
133
|
+
/** Scenarios where announce-stop scenario retry re-ran the agent. */
|
|
134
|
+
scenarioRetriedScenarios: number;
|
|
135
|
+
usage?: UsageStats;
|
|
105
136
|
}
|
|
106
137
|
export interface SuiteRunReport {
|
|
107
138
|
suite: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACjC,kGAAkG;IAClG,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,qFAAqF;IACrF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACzD,4DAA4D;IAC5D,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,yDAAyD;IACzD,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sFAAsF;IACtF,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE;QACV,IAAI,CAAC,EAAE,SAAS,CAAC;QACjB,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,wEAAwE;QACxE,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,+CAA+C;QAC/C,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;KAC7C,CAAC;IACF,SAAS,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,QAAQ,EAAE,eAAe,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,eAAe,GACxB,aAAa,GACb,aAAa,GACb,aAAa,GACb,eAAe,GACf,eAAe,GACf,iBAAiB,CAAC;AAErB,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACrC,yEAAyE;IACzE,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,+EAA+E;IAC/E,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,wBAAwB,EAAE,MAAM,CAAC;IACjC,KAAK,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,UAAU,CAAC;CACrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAiB,cAAc,EAAkB,MAAM,YAAY,CAAC;AAahF,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;
|
|
1
|
+
{"version":3,"file":"validate-suite.d.ts","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAiB,cAAc,EAAkB,MAAM,YAAY,CAAC;AAahF,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAgMD,kEAAkE;AAClE,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,cAAc,GACnB,oBAAoB,EAAE,CAOxB;AAED,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACtD,OAAO,CAAC,qBAAqB,CAAC,CA6DhC;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAY5E"}
|
package/dist/validate-suite.js
CHANGED
|
@@ -2,7 +2,7 @@ import { access } from "node:fs/promises";
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { loadSuiteFile } from "./load-suite.js";
|
|
4
4
|
const VALID_HOSTS = new Set(["cursor", "claude", "replay"]);
|
|
5
|
-
const VALID_PROFILES = new Set(["shared", "cursor", "claude"]);
|
|
5
|
+
const VALID_PROFILES = new Set(["shared", "cursor", "claude", "skeleton"]);
|
|
6
6
|
const VALID_SKILLS = new Set(["none", "catalog", "full"]);
|
|
7
7
|
const VALID_TIERS = new Set(["low", "medium", "high"]);
|
|
8
8
|
const VALID_REVIEW_DEPTHS = new Set([
|
|
@@ -21,7 +21,15 @@ function validateRubric(issues, suitePath, scenarioName, rubric) {
|
|
|
21
21
|
if (rubric.reviewDepth !== undefined && !VALID_REVIEW_DEPTHS.has(rubric.reviewDepth)) {
|
|
22
22
|
pushIssue(issues, suitePath, "rubric.reviewDepth", `reviewDepth must be quick|standard|thorough|full, got ${JSON.stringify(rubric.reviewDepth)}`, scenarioName);
|
|
23
23
|
}
|
|
24
|
-
for (const key of [
|
|
24
|
+
for (const key of [
|
|
25
|
+
"must",
|
|
26
|
+
"mustNot",
|
|
27
|
+
"mustRun",
|
|
28
|
+
"mustCallTool",
|
|
29
|
+
"mustNotCallTool",
|
|
30
|
+
"mustReadPath",
|
|
31
|
+
"mustNotReadPath",
|
|
32
|
+
]) {
|
|
25
33
|
const value = rubric[key];
|
|
26
34
|
if (value !== undefined && !Array.isArray(value)) {
|
|
27
35
|
pushIssue(issues, suitePath, `rubric.${key}`, `${key} must be an array of strings`, scenarioName);
|
|
@@ -54,7 +62,10 @@ function validateScenario(issues, suitePath, scenario) {
|
|
|
54
62
|
pushIssue(issues, suitePath, "host", `host must be cursor|claude|replay, got ${JSON.stringify(scenario.host)}`, scenario.name);
|
|
55
63
|
}
|
|
56
64
|
if (scenario.profile !== undefined && !VALID_PROFILES.has(scenario.profile)) {
|
|
57
|
-
pushIssue(issues, suitePath, "profile", `profile must be shared|cursor|claude, got ${JSON.stringify(scenario.profile)}`, scenario.name);
|
|
65
|
+
pushIssue(issues, suitePath, "profile", `profile must be shared|cursor|claude|skeleton, got ${JSON.stringify(scenario.profile)}`, scenario.name);
|
|
66
|
+
}
|
|
67
|
+
if (scenario.contextSources !== undefined && !Array.isArray(scenario.contextSources)) {
|
|
68
|
+
pushIssue(issues, suitePath, "contextSources", "contextSources must be an array of strings", scenario.name);
|
|
58
69
|
}
|
|
59
70
|
if (scenario.skills !== undefined && !VALID_SKILLS.has(scenario.skills)) {
|
|
60
71
|
pushIssue(issues, suitePath, "skills", `skills must be none|catalog|full, got ${JSON.stringify(scenario.skills)}`, scenario.name);
|
|
@@ -73,7 +84,10 @@ function validateDefaults(issues, suitePath, suite) {
|
|
|
73
84
|
pushIssue(issues, suitePath, "defaults.host", `host must be cursor|claude|replay, got ${JSON.stringify(defaults.host)}`);
|
|
74
85
|
}
|
|
75
86
|
if (defaults.profile !== undefined && !VALID_PROFILES.has(defaults.profile)) {
|
|
76
|
-
pushIssue(issues, suitePath, "defaults.profile", `profile must be shared|cursor|claude, got ${JSON.stringify(defaults.profile)}`);
|
|
87
|
+
pushIssue(issues, suitePath, "defaults.profile", `profile must be shared|cursor|claude|skeleton, got ${JSON.stringify(defaults.profile)}`);
|
|
88
|
+
}
|
|
89
|
+
if (defaults.contextSources !== undefined && !Array.isArray(defaults.contextSources)) {
|
|
90
|
+
pushIssue(issues, suitePath, "defaults.contextSources", "contextSources must be an array of strings");
|
|
77
91
|
}
|
|
78
92
|
if (defaults.skills !== undefined && !VALID_SKILLS.has(defaults.skills)) {
|
|
79
93
|
pushIssue(issues, suitePath, "defaults.skills", `skills must be none|catalog|full, got ${JSON.stringify(defaults.skills)}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-suite.js","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"validate-suite.js","sourceRoot":"","sources":["../src/validate-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACvE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;AAC3F,MAAM,YAAY,GAAG,IAAI,GAAG,CAAsB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAsC,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5F,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAA6C;IAC/E,OAAO;IACP,UAAU;IACV,UAAU;IACV,MAAM;CACN,CAAC,CAAC;AAgBH,SAAS,SAAS,CACjB,MAA8B,EAC9B,SAAiB,EACjB,KAAa,EACb,OAAe,EACf,QAAiB;IAEjB,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,cAAc,CACtB,MAA8B,EAC9B,SAAiB,EACjB,YAAoB,EACpB,MAAsB;IAEtB,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChE,SAAS,CACR,MAAM,EACN,SAAS,EACT,aAAa,EACb,qCAAqC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAClE,YAAY,CACZ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACtF,SAAS,CACR,MAAM,EACN,SAAS,EACT,oBAAoB,EACpB,yDAAyD,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAC7F,YAAY,CACZ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI;QACjB,MAAM;QACN,SAAS;QACT,SAAS;QACT,cAAc;QACd,iBAAiB;QACjB,cAAc;QACd,iBAAiB;KACR,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,SAAS,CACR,MAAM,EACN,SAAS,EACT,UAAU,GAAG,EAAE,EACf,GAAG,GAAG,8BAA8B,EACpC,YAAY,CACZ,CAAC;QACH,CAAC;IACF,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,oBAAoB,CAAU,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,SAAS,CACR,MAAM,EACN,SAAS,EACT,UAAU,GAAG,EAAE,EACf,GAAG,GAAG,yCAAyC,EAC/C,YAAY,CACZ,CAAC;QACH,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,SAAS,CACR,MAAM,EACN,SAAS,EACT,cAAc,EACd,2DAA2D,EAC3D,YAAY,CACZ,CAAC;QACH,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC9B,SAAS;gBACV,CAAC;gBACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBACpF,SAAS,CACR,MAAM,EACN,SAAS,EACT,gBAAgB,KAAK,GAAG,EACxB,qDAAqD,EACrD,YAAY,CACZ,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,gBAAgB,CACxB,MAA8B,EAC9B,SAAiB,EACjB,QAAuB;IAEvB,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,SAAS,CACR,MAAM,EACN,SAAS,EACT,MAAM,EACN,0CAA0C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EACzE,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,SAAS,CACR,MAAM,EACN,SAAS,EACT,SAAS,EACT,sDAAsD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EACxF,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACtF,SAAS,CACR,MAAM,EACN,SAAS,EACT,gBAAgB,EAChB,4CAA4C,EAC5C,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,SAAS,CACR,MAAM,EACN,SAAS,EACT,QAAQ,EACR,yCAAyC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAC1E,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QACnD,SAAS,CACR,MAAM,EACN,SAAS,EACT,eAAe,EACf,kCAAkC,EAClC,QAAQ,CAAC,IAAI,CACb,CAAC;IACH,CAAC;IACD,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CACxB,MAA8B,EAC9B,SAAiB,EACjB,KAAqB;IAErB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO;IACR,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,SAAS,CACR,MAAM,EACN,SAAS,EACT,eAAe,EACf,0CAA0C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CACzE,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,SAAS,CACR,MAAM,EACN,SAAS,EACT,kBAAkB,EAClB,sDAAsD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CACxF,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACtF,SAAS,CACR,MAAM,EACN,SAAS,EACT,yBAAyB,EACzB,4CAA4C,CAC5C,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,SAAS,CACR,MAAM,EACN,SAAS,EACT,iBAAiB,EACjB,yCAAyC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAC1E,CAAC;IACH,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,iBAAiB,CAChC,SAAiB,EACjB,KAAqB;IAErB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACxC,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,UAAoB,EACpB,OAAwD;IAExD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IAEnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,IAAI,KAAqB,CAAC;QAC1B,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,CACR,MAAM,EACN,SAAS,EACT,MAAM,EACN,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CACpE,CAAC;YACF,SAAS;QACV,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACpD,gBAAgB,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;QAE3C,IAAI,OAAO,EAAE,aAAa,IAAI,QAAQ,EAAE,CAAC;YACxC,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACxC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC1B,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;oBAC3D,IAAI,CAAC;wBACJ,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;oBAC1B,CAAC;oBAAC,MAAM,CAAC;wBACR,SAAS,CACR,MAAM,EACN,SAAS,EACT,aAAa,EACb,2BAA2B,QAAQ,CAAC,WAAW,EAAE,EACjD,QAAQ,CAAC,IAAI,CACb,CAAC;oBACH,CAAC;gBACF,CAAC;gBACD,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACxD,IAAI,CAAC;wBACJ,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;oBACzB,CAAC;oBAAC,MAAM,CAAC;wBACR,SAAS,CACR,MAAM,EACN,SAAS,EACT,WAAW,EACX,yBAAyB,QAAQ,CAAC,SAAS,EAAE,EAC7C,QAAQ,CAAC,IAAI,CACb,CAAC;oBACH,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM;QACN,aAAa,EAAE,UAAU,CAAC,MAAM;QAChC,gBAAgB;KAChB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAA6B;IACnE,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,aAAa,MAAM,CAAC,aAAa,cAAc,MAAM,CAAC,gBAAgB,kBAAkB,CAAC;IACjG,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;QAC1F,OAAO,GAAG,KAAK,MAAM,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;IACtD,CAAC,CAAC,CAAC;IACH,OAAO;QACN,sBAAsB,MAAM,CAAC,MAAM,CAAC,MAAM,aAAa;QACvD,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KAC/B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@post-print/agent-test",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": {
|
|
6
6
|
"agent-test": "./dist/cli.js"
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"clean": "rm -rf dist tsconfig.tsbuildinfo"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@post-print/agent-harness": "0.2.
|
|
28
|
+
"@post-print/agent-harness": "0.2.7",
|
|
29
29
|
"chalk": "5"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|