@netlify/axis 1.1.7 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/adapters/base/agent-adapter.d.ts.map +1 -1
- package/dist/adapters/base/agent-adapter.js +43 -2
- package/dist/adapters/base/agent-adapter.js.map +1 -1
- package/dist/adapters/registry.d.ts +2 -0
- package/dist/adapters/registry.d.ts.map +1 -1
- package/dist/adapters/registry.js +6 -2
- package/dist/adapters/registry.js.map +1 -1
- package/dist/cli.js +109 -9
- package/dist/cli.js.map +1 -1
- package/dist/config/loader.d.ts +2 -2
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +229 -28
- package/dist/config/loader.js.map +1 -1
- package/dist/config/validator.d.ts +9 -1
- package/dist/config/validator.d.ts.map +1 -1
- package/dist/config/validator.js +75 -4
- package/dist/config/validator.js.map +1 -1
- package/dist/report-ui/index.html +201 -151
- package/dist/reports/html.d.ts.map +1 -1
- package/dist/reports/html.js +2 -1
- package/dist/reports/html.js.map +1 -1
- package/dist/reports/writer.d.ts.map +1 -1
- package/dist/reports/writer.js +3 -0
- package/dist/reports/writer.js.map +1 -1
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +152 -13
- package/dist/runner/runner.js.map +1 -1
- package/dist/scoring/index.d.ts.map +1 -1
- package/dist/scoring/index.js +4 -1
- package/dist/scoring/index.js.map +1 -1
- package/dist/scoring/sparse-index.d.ts +6 -1
- package/dist/scoring/sparse-index.d.ts.map +1 -1
- package/dist/scoring/sparse-index.js +43 -15
- package/dist/scoring/sparse-index.js.map +1 -1
- package/dist/transcript/extract.d.ts.map +1 -1
- package/dist/transcript/extract.js +33 -3
- package/dist/transcript/extract.js.map +1 -1
- package/dist/types/agent.d.ts +4 -0
- package/dist/types/agent.d.ts.map +1 -1
- package/dist/types/config.d.ts +27 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/output.d.ts +12 -2
- package/dist/types/output.d.ts.map +1 -1
- package/dist/types/output.js.map +1 -1
- package/dist/types/report.d.ts +3 -1
- package/dist/types/report.d.ts.map +1 -1
- package/dist/types/scenario.d.ts +16 -2
- package/dist/types/scenario.d.ts.map +1 -1
- package/dist/types/scoring.d.ts +4 -0
- package/dist/types/scoring.d.ts.map +1 -1
- package/dist/ui/LiveStatus.js +10 -5
- package/dist/ui/LiveStatus.js.map +1 -1
- package/dist/ui/format.d.ts +4 -0
- package/dist/ui/format.d.ts.map +1 -1
- package/dist/ui/format.js +34 -18
- package/dist/ui/format.js.map +1 -1
- package/package.json +2 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/transcript/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG,eAAe,
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/transcript/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG,eAAe,CAgFrE;AAMD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,IAAI,CA0B/E"}
|
|
@@ -41,16 +41,21 @@ export function extractFields(entry) {
|
|
|
41
41
|
errorMessage: null,
|
|
42
42
|
toolId: extractToolId(content),
|
|
43
43
|
};
|
|
44
|
-
case "tool_result":
|
|
44
|
+
case "tool_result": {
|
|
45
|
+
// Some adapters (e.g. codex web_search) emit the tool_use BEFORE the input
|
|
46
|
+
// is finalized — the populated input arrives on the tool_result. Extract it
|
|
47
|
+
// so downstream logic can recover the input from the completed entry.
|
|
48
|
+
const toolInput = extractToolInput(content);
|
|
45
49
|
return {
|
|
46
50
|
text: null,
|
|
47
51
|
toolName: extractToolName(content),
|
|
48
|
-
toolInput
|
|
49
|
-
toolInputSummary:
|
|
52
|
+
toolInput,
|
|
53
|
+
toolInputSummary: summarizeInput(toolInput),
|
|
50
54
|
toolResultText: extractToolResultText(content),
|
|
51
55
|
errorMessage: null,
|
|
52
56
|
toolId: extractToolId(content),
|
|
53
57
|
};
|
|
58
|
+
}
|
|
54
59
|
case "error":
|
|
55
60
|
return {
|
|
56
61
|
text: extractGenericText(content),
|
|
@@ -100,6 +105,9 @@ export function extractToolName(content) {
|
|
|
100
105
|
if (typeof content.type === "string" && content.type === "web_search") {
|
|
101
106
|
return "web_search";
|
|
102
107
|
}
|
|
108
|
+
if (typeof content.type === "string" && content.type === "file_change") {
|
|
109
|
+
return "file_change";
|
|
110
|
+
}
|
|
103
111
|
// Claude Code nested: { message: { content: [{ type: "tool_use", name: "..." }] } }
|
|
104
112
|
const nested = extractNestedBlock(content, "tool_use");
|
|
105
113
|
if (nested && typeof nested.name === "string")
|
|
@@ -123,6 +131,25 @@ function extractToolInput(content) {
|
|
|
123
131
|
if (typeof content.command === "string") {
|
|
124
132
|
return { command: content.command };
|
|
125
133
|
}
|
|
134
|
+
// Codex web_search: { type: "web_search", query: "..." }
|
|
135
|
+
if (typeof content.query === "string" && content.query.length > 0) {
|
|
136
|
+
return { query: content.query };
|
|
137
|
+
}
|
|
138
|
+
// Codex generic args field: { arguments: "..." } (function_call payloads)
|
|
139
|
+
if (typeof content.arguments === "string") {
|
|
140
|
+
return { arguments: content.arguments };
|
|
141
|
+
}
|
|
142
|
+
// Codex file_change: { changes: [{ path, kind }, ...] }
|
|
143
|
+
if (Array.isArray(content.changes) && content.changes.length > 0) {
|
|
144
|
+
const summary = content.changes
|
|
145
|
+
.map((c) => {
|
|
146
|
+
const kind = typeof c.kind === "string" ? c.kind : "change";
|
|
147
|
+
const p = typeof c.path === "string" ? c.path : "(unknown)";
|
|
148
|
+
return `${kind}: ${p}`;
|
|
149
|
+
})
|
|
150
|
+
.join("; ");
|
|
151
|
+
return { changes: summary };
|
|
152
|
+
}
|
|
126
153
|
return null;
|
|
127
154
|
}
|
|
128
155
|
function summarizeInput(input) {
|
|
@@ -160,6 +187,9 @@ function extractToolResultText(content) {
|
|
|
160
187
|
return parts.join(" | ");
|
|
161
188
|
}
|
|
162
189
|
}
|
|
190
|
+
// Codex command_execution: { aggregated_output: "..." }
|
|
191
|
+
if (typeof content.aggregated_output === "string")
|
|
192
|
+
return content.aggregated_output;
|
|
163
193
|
// Gemini / Codex: { output: "..." }
|
|
164
194
|
if (typeof content.output === "string")
|
|
165
195
|
return content.output;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/transcript/extract.ts"],"names":[],"mappings":"AAYA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAsB;IAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAE1B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,gEAAgE;YAChE,oDAAoD;YACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC5C,OAAO;oBACL,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC;oBACnC,QAAQ;oBACR,SAAS;oBACT,gBAAgB,EAAE,cAAc,CAAC,SAAS,CAAC;oBAC3C,cAAc,EAAE,IAAI;oBACpB,YAAY,EAAE,IAAI;oBAClB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;iBAC/B,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC;gBACnC,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE,IAAI;gBACtB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,IAAI;aACb,CAAC;QACJ,CAAC;QAED,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC;gBAClC,SAAS,EAAE,gBAAgB,CAAC,OAAO,CAAC;gBACpC,gBAAgB,EAAE,cAAc,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC3D,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;aAC/B,CAAC;QAEJ,KAAK,aAAa;
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/transcript/extract.ts"],"names":[],"mappings":"AAYA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAsB;IAClD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAE1B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,gEAAgE;YAChE,oDAAoD;YACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC5C,OAAO;oBACL,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC;oBACnC,QAAQ;oBACR,SAAS;oBACT,gBAAgB,EAAE,cAAc,CAAC,SAAS,CAAC;oBAC3C,cAAc,EAAE,IAAI;oBACpB,YAAY,EAAE,IAAI;oBAClB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;iBAC/B,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC;gBACnC,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE,IAAI;gBACtB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,IAAI;aACb,CAAC;QACJ,CAAC;QAED,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC;gBAClC,SAAS,EAAE,gBAAgB,CAAC,OAAO,CAAC;gBACpC,gBAAgB,EAAE,cAAc,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC3D,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;aAC/B,CAAC;QAEJ,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,2EAA2E;YAC3E,4EAA4E;YAC5E,sEAAsE;YACtE,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO;gBACL,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC;gBAClC,SAAS;gBACT,gBAAgB,EAAE,cAAc,CAAC,SAAS,CAAC;gBAC3C,cAAc,EAAE,qBAAqB,CAAC,OAAO,CAAC;gBAC9C,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;aAC/B,CAAC;QACJ,CAAC;QAED,KAAK,OAAO;YACV,OAAO;gBACL,IAAI,EAAE,kBAAkB,CAAC,OAAO,CAAC;gBACjC,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE,IAAI;gBACtB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,mBAAmB,CAAC,OAAO,CAAC;gBAC1C,MAAM,EAAE,IAAI;aACb,CAAC;QAEJ;YACE,OAAO;gBACL,IAAI,EAAE,kBAAkB,CAAC,OAAO,CAAC;gBACjC,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE,IAAI;gBACtB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,IAAI;aACb,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,OAAgC;IAC9D,gCAAgC;IAChC,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,SAAS,CAAC;IAEpE,yCAAyC;IACzC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAE1D,kEAAkE;IAClE,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAC7E,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACtE,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACvE,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,oFAAoF;IACpF,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACvD,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IAElE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,OAAgC;IACxD,4CAA4C;IAC5C,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjE,OAAO,OAAO,CAAC,UAAqC,CAAC;IACvD,CAAC;IAED,qFAAqF;IACrF,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACvD,IAAI,MAAM,EAAE,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,KAAgC,CAAC;IACjD,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IAED,yDAAyD;IACzD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,0EAA0E;IAC1E,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QAC1C,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC;IAED,wDAAwD;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjE,MAAM,OAAO,GAAI,OAAO,CAAC,OAA0C;aAChE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC5D,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;YAC5D,OAAO,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;QACzB,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,KAAqC;IAC3D,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,SAAS,qBAAqB,CAAC,OAAgC;IAC7D,0FAA0F;IAC1F,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC1D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;QACrC,IAAI,OAAO,aAAa,KAAK,QAAQ;YAAE,OAAO,aAAa,CAAC;QAC5D,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACjC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;gBACjC,MAAM,CAAC,GAAG,IAA+B,CAAC;gBAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;oBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrD,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,IAAI,OAAO,OAAO,CAAC,iBAAiB,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,iBAAiB,CAAC;IAEpF,oCAAoC;IACpC,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,MAAM,CAAC;IAE9D,8BAA8B;IAC9B,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAEhE,2BAA2B;IAC3B,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAE1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,0CAA0C;AAC1C,8EAA8E;AAE9E,SAAS,oBAAoB,CAAC,OAAgC;IAC5D,gFAAgF;IAChF,MAAM,OAAO,GAAG,OAAO,CAAC,OAA8C,CAAC;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,OAAqD,CAAC;QAC7E,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBACrC,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAEhE,mCAAmC;IACnC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAE1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgC;IAC1D,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAC1D,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAChE,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAChE,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,OAAgC;IAC3D,+BAA+B;IAC/B,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC;IAE5D,0CAA0C;IAC1C,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAEhE,qDAAqD;IACrD,MAAM,MAAM,GAAG,OAAO,CAAC,KAA4C,CAAC;IACpE,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC;IAExE,gBAAgB;IAChB,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC;IAE1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,OAAgC;IACrD,kCAAkC;IAClC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC;IAEhE,yBAAyB;IACzB,IAAI,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,EAAE,CAAC;IAEtD,mDAAmD;IACnD,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC1D,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,WAAW,CAAC;IAEhF,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1D,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC,EAAE,CAAC;IAEvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,gGAAgG;AAChG,SAAS,kBAAkB,CAAC,OAAgC,EAAE,SAAiB;IAC7E,MAAM,OAAO,GAAG,OAAO,CAAC,OAA8C,CAAC;IACvE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAqD,CAAC;IAC7E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAExC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,IAAI,CAAC;AAC1D,CAAC"}
|
package/dist/types/agent.d.ts
CHANGED
|
@@ -46,6 +46,10 @@ export interface AgentInput {
|
|
|
46
46
|
* below the true count so the UI never has to reverse.
|
|
47
47
|
*/
|
|
48
48
|
onTokenProgress?: (estimatedTokens: number) => void;
|
|
49
|
+
/** Override the adapter's default timeout (in ms). Set by the runner from resolved scenario limits. */
|
|
50
|
+
timeoutMs?: number;
|
|
51
|
+
/** Abort signal. When fired, the adapter kills the child process with SIGTERM → SIGKILL. */
|
|
52
|
+
signal?: AbortSignal;
|
|
49
53
|
}
|
|
50
54
|
export interface AgentOutput {
|
|
51
55
|
transcript: TranscriptEntry[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/types/agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7C;;;;OAIG;IACH,YAAY,CAAC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzD;;;;OAIG;IACH,WAAW,CAAC,IAAI,MAAM,EAAE,CAAC;IACzB;;;;OAIG;IACH,eAAe,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,6FAA6F;IAC7F,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,mFAAmF;IACnF,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IAC3C,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,qDAAqD;IACrD,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;IACjC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/types/agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7C;;;;OAIG;IACH,YAAY,CAAC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzD;;;;OAIG;IACH,WAAW,CAAC,IAAI,MAAM,EAAE,CAAC;IACzB;;;;OAIG;IACH,eAAe,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,6FAA6F;IAC7F,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,mFAAmF;IACnF,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;IAC3C,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC7C,qDAAqD;IACrD,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;IACjC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,uGAAuG;IACvG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,WAAW,GAAG,MAAM,GAAG,UAAU,GAAG,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC"}
|
package/dist/types/config.d.ts
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
|
+
import type { ScenarioInput } from "./scenario.js";
|
|
1
2
|
export interface AxisConfig {
|
|
2
3
|
/** Human-readable project name. Shown in report headers. */
|
|
3
4
|
name?: string;
|
|
4
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Where scenarios come from. Supports three forms:
|
|
7
|
+
* - A path string — directory walked for `*.json`/`*.{js,ts,...}` scenario files.
|
|
8
|
+
* - An array of strings — each entry is a directory or single scenario file path.
|
|
9
|
+
* - An array mixing path strings and inline {@link ScenarioInput} objects, useful
|
|
10
|
+
* when authoring `axis.config.{js,ts}` and generating scenarios programmatically.
|
|
11
|
+
*
|
|
12
|
+
* When omitted, the loader defaults to `"./scenarios"` (relative to the config file).
|
|
13
|
+
*/
|
|
14
|
+
scenarios?: string | (string | ScenarioInput)[];
|
|
5
15
|
agents: (string | AgentConfig)[];
|
|
6
16
|
settings?: SettingsConfig;
|
|
7
17
|
/** Custom adapter modules. Keys are adapter names, values are paths (relative to config) to JS/TS modules that export an AgentAdapter. */
|
|
@@ -34,6 +44,22 @@ export interface SettingsConfig {
|
|
|
34
44
|
scoring_weights?: ScoringWeights;
|
|
35
45
|
/** Maximum number of parallel jobs. Defaults to unlimited (all jobs run simultaneously). */
|
|
36
46
|
concurrency?: number;
|
|
47
|
+
/** Time and token spend limits for the run and individual scenarios. */
|
|
48
|
+
limits?: LimitsConfig;
|
|
49
|
+
}
|
|
50
|
+
/** Overall run limits and default per-scenario limits. */
|
|
51
|
+
export interface LimitsConfig {
|
|
52
|
+
/** When hit, ALL remaining and in-progress jobs are aborted. */
|
|
53
|
+
run?: ScenarioLimitsConfig;
|
|
54
|
+
/** Default limits applied to each individual job. Per-scenario limits override these. */
|
|
55
|
+
scenario?: ScenarioLimitsConfig;
|
|
56
|
+
}
|
|
57
|
+
/** Limits for an individual scenario/job. Reused across settings and scenario configs. */
|
|
58
|
+
export interface ScenarioLimitsConfig {
|
|
59
|
+
/** Maximum execution time in minutes. Accepts fractional values. */
|
|
60
|
+
time_minutes?: number;
|
|
61
|
+
/** Maximum total tokens (input + output + cache). Must be a positive integer. */
|
|
62
|
+
tokens?: number;
|
|
37
63
|
}
|
|
38
64
|
export interface ScoringWeights {
|
|
39
65
|
goal_achievement: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/types/config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/types/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,UAAU;IACzB,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC;IAChD,MAAM,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,0IAA0I;IAC1I,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,oHAAoH;IACpH,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,8IAA8I;IAC9I,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;CAC1C;AAED,MAAM,WAAW,cAAc;IAC7B,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4FAA4F;IAC5F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,gEAAgE;IAChE,GAAG,CAAC,EAAE,oBAAoB,CAAC;IAC3B,yFAAyF;IACzF,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAED,0FAA0F;AAC1F,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,aAAa,CAAC;AAE7D,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oDAAoD;IACpD,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC"}
|
package/dist/types/output.d.ts
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import type { AgentOutput } from "./agent.js";
|
|
2
|
-
import type { AgentConfig } from "./config.js";
|
|
3
|
-
import type { RubricCriterion } from "./scenario.js";
|
|
2
|
+
import type { AgentConfig, McpServerConfig, ScenarioLimitsConfig } from "./config.js";
|
|
3
|
+
import type { LifecycleAction, RubricCriterion } from "./scenario.js";
|
|
4
4
|
import type { ScoredRunResult } from "./scoring.js";
|
|
5
|
+
/** Materialized scenario configuration for a single run — limits, skills, lifecycle, and MCP, with defaults already applied. */
|
|
6
|
+
export interface ResolvedRunConfig {
|
|
7
|
+
limits?: ScenarioLimitsConfig;
|
|
8
|
+
skills?: string[];
|
|
9
|
+
setup?: LifecycleAction[];
|
|
10
|
+
teardown?: LifecycleAction[];
|
|
11
|
+
mcpServers?: Record<string, McpServerConfig>;
|
|
12
|
+
}
|
|
5
13
|
export interface RunOutput {
|
|
6
14
|
version: string;
|
|
7
15
|
timestamp: string;
|
|
@@ -20,6 +28,8 @@ export interface BaseRunResult {
|
|
|
20
28
|
output: AgentOutput;
|
|
21
29
|
/** Path to the agent's workspace directory (available during scoring, before cleanup). */
|
|
22
30
|
workingDirectory?: string;
|
|
31
|
+
/** Materialized scenario settings (limits, skills, lifecycle, MCP) actually applied to this run. */
|
|
32
|
+
resolvedConfig?: ResolvedRunConfig;
|
|
23
33
|
}
|
|
24
34
|
export interface RunResult extends BaseRunResult {
|
|
25
35
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/types/output.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/types/output.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,gIAAgI;AAChI,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,OAAO,EAAE,UAAU,CAAC;CACrB;AAED,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,eAAe,EAAE,CAAC;IACnC,WAAW,EAAE,WAAW,CAAC;IACzB,MAAM,EAAE,WAAW,CAAC;IACpB,0FAA0F;IAC1F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oGAAoG;IACpG,cAAc,CAAC,EAAE,iBAAiB,CAAC;CACpC;AAED,MAAM,WAAW,SAAU,SAAQ,aAAa;CAAG;AAEnD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAErG,MAAM,WAAW,QAAQ;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,2EAA2E;IAC3E,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,2EAA2E;IAC3E,WAAW,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACnE;AAED,eAAO,MAAM,YAAY,EAAE,MAG1B,CAAC;AAEF,0DAA0D;AAC1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,IAAI,eAAe,CAE/E;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAGhD"}
|
package/dist/types/output.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/types/output.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/types/output.ts"],"names":[],"mappings":"AAqFA,MAAM,CAAC,MAAM,YAAY,GAAW;IAClC,IAAI,KAAI,CAAC;IACT,KAAK,KAAI,CAAC;CACX,CAAC;AAEF,0DAA0D;AAC1D,MAAM,UAAU,cAAc,CAAC,MAAqB;IAClD,OAAO,OAAO,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;AACnD,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,IAAI,GAAG,YAAY,KAAK;QAAE,OAAO,GAAG,CAAC,OAAO,CAAC;IAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC"}
|
package/dist/types/report.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { TokenUsage } from "./agent.js";
|
|
2
2
|
import type { AgentConfig } from "./config.js";
|
|
3
3
|
import type { ScoreResult } from "./scoring.js";
|
|
4
|
-
import type { RunSummary } from "./output.js";
|
|
4
|
+
import type { ResolvedRunConfig, RunSummary } from "./output.js";
|
|
5
5
|
import type { ScoredSummary } from "./scoring.js";
|
|
6
6
|
import type { RubricCriterion } from "./scenario.js";
|
|
7
7
|
/** Lightweight report manifest — no transcripts, just summary data. */
|
|
@@ -35,5 +35,7 @@ export interface ReportResultEntry {
|
|
|
35
35
|
rubric?: string | RubricCriterion[];
|
|
36
36
|
/** Agent configuration. */
|
|
37
37
|
agentConfig?: AgentConfig;
|
|
38
|
+
/** Materialized scenario configuration (limits, skills, lifecycle, MCP) actually used for this run. */
|
|
39
|
+
resolvedConfig?: ResolvedRunConfig;
|
|
38
40
|
}
|
|
39
41
|
//# sourceMappingURL=report.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/types/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/types/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,uEAAuE;AACvE,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,aAAa,GAAG,UAAU,CAAC;IACpC,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,iEAAiE;AACjE,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,GAAG,eAAe,EAAE,CAAC;IACpC,2BAA2B;IAC3B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,uGAAuG;IACvG,cAAc,CAAC,EAAE,iBAAiB,CAAC;CACpC"}
|
package/dist/types/scenario.d.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import type { McpServerConfig } from "./config.js";
|
|
1
|
+
import type { McpServerConfig, ScenarioLimitsConfig } from "./config.js";
|
|
2
2
|
export interface Scenario {
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* Stable identifier. For on-disk JSON scenarios the loader derives it from the file path
|
|
5
|
+
* (relative to the scenarios root, sans extension). For inline scenarios declared in
|
|
6
|
+
* `axis.config.{js,ts,json}`, authors must provide it themselves.
|
|
7
|
+
*/
|
|
4
8
|
key: string;
|
|
5
9
|
name: string;
|
|
6
10
|
/** When true, the scenario is excluded from runs. */
|
|
@@ -15,6 +19,8 @@ export interface Scenario {
|
|
|
15
19
|
skills?: string[];
|
|
16
20
|
/** MCP servers specific to this scenario, merged with top-level servers. */
|
|
17
21
|
mcp_servers?: Record<string, McpServerConfig>;
|
|
22
|
+
/** Per-scenario time/token limits. Overrides settings.limits.scenario defaults. */
|
|
23
|
+
limits?: ScenarioLimitsConfig;
|
|
18
24
|
/**
|
|
19
25
|
* When defined, the scenario becomes a template. Only variants run;
|
|
20
26
|
* the base scenario does not execute on its own. Each variant inherits
|
|
@@ -33,7 +39,15 @@ export interface ScenarioVariant {
|
|
|
33
39
|
agents?: string[];
|
|
34
40
|
skills?: string[];
|
|
35
41
|
mcp_servers?: Record<string, McpServerConfig>;
|
|
42
|
+
/** Per-variant time/token limits. Overrides parent scenario and default limits. */
|
|
43
|
+
limits?: ScenarioLimitsConfig;
|
|
36
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* User-facing shape for inline scenarios declared in `axis.config.{js,ts,json}`.
|
|
47
|
+
* Identical to {@link Scenario} but explicit about the contract: `key` is required
|
|
48
|
+
* (the loader does not derive it from a file path for inline entries).
|
|
49
|
+
*/
|
|
50
|
+
export type ScenarioInput = Scenario;
|
|
37
51
|
export interface LifecycleAction {
|
|
38
52
|
action: "run_script";
|
|
39
53
|
command: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scenario.d.ts","sourceRoot":"","sources":["../../src/types/scenario.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"scenario.d.ts","sourceRoot":"","sources":["../../src/types/scenario.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEzE,MAAM,WAAW,QAAQ;IACvB;;;;OAIG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,eAAe,EAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;IAC7B,wFAAwF;IACxF,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,oFAAoF;IACpF,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,iHAAiH;IACjH,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,eAAe,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,CAAC;AAErC,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
package/dist/types/scoring.d.ts
CHANGED
|
@@ -53,6 +53,10 @@ export interface SparseIndex {
|
|
|
53
53
|
totalDurationMs: number;
|
|
54
54
|
/** Wall-clock elapsed time from first to last interaction end. */
|
|
55
55
|
wallClockMs: number;
|
|
56
|
+
/** Time from agent process spawn to the first traced interaction (model warmup, auth, etc.). */
|
|
57
|
+
startupMs?: number;
|
|
58
|
+
/** Time from the last traced interaction to agent process exit (final response, teardown). */
|
|
59
|
+
shutdownMs?: number;
|
|
56
60
|
};
|
|
57
61
|
}
|
|
58
62
|
/** Per-interaction audit result from the deep evaluation pass. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../src/types/scoring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIrE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAID,2EAA2E;AAC3E,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG,SAAS,GAAG,OAAO,CAAC;AAEtE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,8CAA8C;IAC9C,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,yFAAyF;IACzF,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,6CAA6C;IAC7C,QAAQ,EAAE,OAAO,CAAC;IAClB,oEAAoE;IACpE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,mFAAmF;IACnF,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,mFAAmF;IACnF,YAAY,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,+DAA+D;AAC/D,MAAM,WAAW,WAAW;IAC1B,qDAAqD;IACrD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,8EAA8E;IAC9E,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,qBAAqB;IACrB,KAAK,EAAE;QACL,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;QAChD,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,kEAAkE;QAClE,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAID,kEAAkE;AAClE,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,oEAAoE;IACpE,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,uEAAuE;AACvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CACrC;AAED,mCAAmC;AACnC,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,qCAAqC;IACrC,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,+DAA+D;IAC/D,QAAQ,EAAE,WAAW,EAAE,CAAC;CACzB;AAED,0DAA0D;AAC1D,MAAM,WAAW,kBAAkB;IACjC,yCAAyC;IACzC,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,sEAAsE;IACtE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,4CAA4C;IAC5C,SAAS,EAAE,iBAAiB,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,EAAE,WAAW,EAAE,CAAC;CACzB;AAID,iFAAiF;AACjF,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,gBAAgB,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,UAAU,EAAE;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uCAAuC;IACvC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,0BAA0B;IAC1B,SAAS,EAAE,iBAAiB,CAAC;CAC9B;AAID,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,eAAe,EAAE,oBAAoB,CAAC;IACtC,oDAAoD;IACpD,WAAW,EAAE,aAAa,CAAC;IAC3B,gDAAgD;IAChD,OAAO,EAAE,aAAa,CAAC;IACvB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,qCAAqC;IACrC,OAAO,EAAE,cAAc,CAAC;IACxB,wEAAwE;IACxE,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAID,MAAM,WAAW,eAAgB,SAAQ,aAAa;IACpD,KAAK,EAAE,WAAW,CAAC;CACpB;AAID,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAID,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,UAAU,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC;IACvF,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
|
|
1
|
+
{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../src/types/scoring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIrE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAID,2EAA2E;AAC3E,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG,SAAS,GAAG,OAAO,CAAC;AAEtE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,8CAA8C;IAC9C,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,yFAAyF;IACzF,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,6CAA6C;IAC7C,QAAQ,EAAE,OAAO,CAAC;IAClB,oEAAoE;IACpE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,mFAAmF;IACnF,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,mFAAmF;IACnF,YAAY,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,+DAA+D;AAC/D,MAAM,WAAW,WAAW;IAC1B,qDAAqD;IACrD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,8EAA8E;IAC9E,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,qBAAqB;IACrB,KAAK,EAAE;QACL,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;QAChD,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,kEAAkE;QAClE,WAAW,EAAE,MAAM,CAAC;QACpB,gGAAgG;QAChG,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,8FAA8F;QAC9F,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAID,kEAAkE;AAClE,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,oEAAoE;IACpE,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,uEAAuE;AACvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CACrC;AAED,mCAAmC;AACnC,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,qCAAqC;IACrC,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,+DAA+D;IAC/D,QAAQ,EAAE,WAAW,EAAE,CAAC;CACzB;AAED,0DAA0D;AAC1D,MAAM,WAAW,kBAAkB;IACjC,yCAAyC;IACzC,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,sEAAsE;IACtE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,4CAA4C;IAC5C,SAAS,EAAE,iBAAiB,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,EAAE,WAAW,EAAE,CAAC;CACzB;AAID,iFAAiF;AACjF,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,gBAAgB,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,UAAU,EAAE;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,uCAAuC;IACvC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,0BAA0B;IAC1B,SAAS,EAAE,iBAAiB,CAAC;CAC9B;AAID,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,eAAe,EAAE,oBAAoB,CAAC;IACtC,oDAAoD;IACpD,WAAW,EAAE,aAAa,CAAC;IAC3B,gDAAgD;IAChD,OAAO,EAAE,aAAa,CAAC;IACvB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,qCAAqC;IACrC,OAAO,EAAE,cAAc,CAAC;IACxB,wEAAwE;IACxE,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAID,MAAM,WAAW,eAAgB,SAAQ,aAAa;IACpD,KAAK,EAAE,WAAW,CAAC;CACpB;AAID,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;CACxB;AAED,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAID,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,UAAU,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,KAAK,IAAI,CAAC;IACvF,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
|
package/dist/ui/LiveStatus.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
import { Box, Text } from "ink";
|
|
3
|
-
import { STATUS_ICONS, STATUS_LABELS } from "./format.js";
|
|
3
|
+
import { getBaseKey, getVariantName, STATUS_ICONS, STATUS_LABELS } from "./format.js";
|
|
4
4
|
import { AnimatedTokens } from "./AnimatedTokens.js";
|
|
5
5
|
import { LiveDuration } from "./LiveDuration.js";
|
|
6
6
|
export function LiveStatus({ jobs, skippedCount = 0 }) {
|
|
@@ -17,10 +17,12 @@ export function LiveStatus({ jobs, skippedCount = 0 }) {
|
|
|
17
17
|
return (_jsxs(Box, { flexDirection: "column", paddingLeft: 2, children: [_jsx(Text, { children: " " }), _jsxs(Text, { bold: true, children: ["AXIS \u2014 ", scenarioCount, " scenario", scenarioCount !== 1 ? "s" : "", " \u00B7 ", agentCount, " agent", agentCount !== 1 ? "s" : ""] }), _jsx(Text, { children: "─".repeat(50) }), scenarios.map(({ scenarioKey, agents }) => (_jsx(ScenarioGroup, { scenarioKey: scenarioKey, agents: agents }, scenarioKey))), _jsx(Text, { children: "─".repeat(50) }), allFinished && avgScore !== null ? (_jsxs(Text, { bold: true, children: ["Average AXIS Result: ", avgScore, " / 100"] })) : scoring > 0 ? (_jsxs(Text, { children: [done + failed, "/", total, " complete \u00B7 scoring ", scoring, " result", scoring !== 1 ? "s" : "", "\u2026"] })) : (_jsxs(Text, { children: [done + failed, "/", total, " complete"] })), skippedCount > 0 ? _jsxs(Text, { dimColor: true, children: [skippedCount, " marked to be skipped"] }) : null, _jsx(Text, { children: " " })] }));
|
|
18
18
|
}
|
|
19
19
|
function ScenarioGroup({ scenarioKey, agents }) {
|
|
20
|
-
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { bold: true, children: scenarioKey }), agents.map((job) => (_jsx(AgentRow, { job: job }, job.agentName))), _jsx(Text, { children: " " })] }));
|
|
20
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { bold: true, children: scenarioKey }), agents.map((job) => (_jsx(AgentRow, { job: job }, `${job.scenarioKey}:${job.agentName}`))), _jsx(Text, { children: " " })] }));
|
|
21
21
|
}
|
|
22
|
+
const COL_AGENT_LIVE = 25;
|
|
22
23
|
function AgentRow({ job }) {
|
|
23
24
|
const icon = STATUS_ICONS[job.status] ?? "?";
|
|
25
|
+
const variant = getVariantName(job.scenarioKey);
|
|
24
26
|
const label = (job.status === "done" || job.status === "failed") && job.axisScore !== undefined
|
|
25
27
|
? `${job.axisScore} / 100`
|
|
26
28
|
: (STATUS_LABELS[job.status] ?? job.status);
|
|
@@ -38,14 +40,17 @@ function AgentRow({ job }) {
|
|
|
38
40
|
const active = job.status === "running" || job.status === "scoring";
|
|
39
41
|
const hasTime = job.runStartedAt !== undefined || job.durationMs !== undefined;
|
|
40
42
|
const hasTokens = (job.liveTokens ?? 0) > 0;
|
|
41
|
-
|
|
43
|
+
// Build the agent display name with optional variant suffix
|
|
44
|
+
const agentDisplay = variant ? `${job.agentName} @${variant}` : job.agentName;
|
|
45
|
+
return (_jsxs(Box, { children: [_jsxs(Text, { color: color, children: [" ", icon, " ", agentDisplay.padEnd(COL_AGENT_LIVE), " ", label.padEnd(15)] }), hasTime ? (_jsx(Box, { marginRight: 1, children: _jsx(LiveDuration, { startedAt: job.runStartedAt, finalMs: job.durationMs, active: active, color: color }) })) : null, hasTokens ? _jsx(AnimatedTokens, { target: job.liveTokens ?? 0, active: active, isFinal: job.tokensFinal }) : null] }));
|
|
42
46
|
}
|
|
43
47
|
function groupByScenario(jobs) {
|
|
44
48
|
const map = new Map();
|
|
45
49
|
for (const job of jobs) {
|
|
46
|
-
const
|
|
50
|
+
const baseKey = getBaseKey(job.scenarioKey);
|
|
51
|
+
const list = map.get(baseKey) ?? [];
|
|
47
52
|
list.push(job);
|
|
48
|
-
map.set(
|
|
53
|
+
map.set(baseKey, list);
|
|
49
54
|
}
|
|
50
55
|
return Array.from(map, ([scenarioKey, agents]) => ({ scenarioKey, agents }));
|
|
51
56
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LiveStatus.js","sourceRoot":"","sources":["../../src/ui/LiveStatus.tsx"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,KAAK,CAAC;AAEhC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"LiveStatus.js","sourceRoot":"","sources":["../../src/ui/LiveStatus.tsx"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,KAAK,CAAC;AAEhC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAOjD,MAAM,UAAU,UAAU,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,CAAC,EAAmB;IACpE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAE1B,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,MAAM,WAAW,GAAG,IAAI,GAAG,MAAM,KAAK,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACjE,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAU,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtH,OAAO,CACL,MAAC,GAAG,IAAC,aAAa,EAAC,QAAQ,EAAC,WAAW,EAAE,CAAC,aACxC,KAAC,IAAI,oBAAS,EACd,MAAC,IAAI,IAAC,IAAI,mCACA,aAAa,eAAW,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,cAAK,UAAU,YAC5E,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IACvB,EACP,KAAC,IAAI,cAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAQ,EAC5B,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAC1C,KAAC,aAAa,IAAmB,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,IAArD,WAAW,CAA8C,CAC9E,CAAC,EACF,KAAC,IAAI,cAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAQ,EAC5B,WAAW,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,CAClC,MAAC,IAAI,IAAC,IAAI,4CAAuB,QAAQ,cAAc,CACxD,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAChB,MAAC,IAAI,eACF,IAAI,GAAG,MAAM,OAAG,KAAK,+BAAsB,OAAO,aAAS,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,cAC/E,CACR,CAAC,CAAC,CAAC,CACF,MAAC,IAAI,eACF,IAAI,GAAG,MAAM,OAAG,KAAK,iBACjB,CACR,EACA,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,MAAC,IAAI,IAAC,QAAQ,mBAAE,YAAY,6BAA6B,CAAC,CAAC,CAAC,IAAI,EACpF,KAAC,IAAI,oBAAS,IACV,CACP,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,EAAE,WAAW,EAAE,MAAM,EAA+C;IACzF,OAAO,CACL,MAAC,GAAG,IAAC,aAAa,EAAC,QAAQ,aACzB,KAAC,IAAI,IAAC,IAAI,kBAAE,WAAW,GAAQ,EAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CACnB,KAAC,QAAQ,IAA6C,GAAG,EAAE,GAAG,IAA/C,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,SAAS,EAAE,CAAc,CACnE,CAAC,EACF,KAAC,IAAI,oBAAS,IACV,CACP,CAAC;AACJ,CAAC;AAED,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B,SAAS,QAAQ,CAAC,EAAE,GAAG,EAAqB;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;IAC7C,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAEhD,MAAM,KAAK,GACT,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS;QAC/E,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,QAAQ;QAC1B,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAEhD,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,MAAM;QACnB,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ;YACvB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;gBACpD,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,SAAS,CAAC;IAEpB,yEAAyE;IACzE,wEAAwE;IACxE,uEAAuE;IACvE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;IACpE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC;IAC/E,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAE5C,4DAA4D;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;IAE9E,OAAO,CACL,MAAC,GAAG,eACF,MAAC,IAAI,IAAC,KAAK,EAAE,KAAK,aACf,IAAI,EACJ,IAAI,OAAG,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,OAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IACzD,EACN,OAAO,CAAC,CAAC,CAAC,CACT,KAAC,GAAG,IAAC,WAAW,EAAE,CAAC,YACjB,KAAC,YAAY,IAAC,SAAS,EAAE,GAAG,CAAC,YAAY,EAAE,OAAO,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAI,GAChG,CACP,CAAC,CAAC,CAAC,IAAI,EACP,SAAS,CAAC,CAAC,CAAC,KAAC,cAAc,IAAC,MAAM,EAAE,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,WAAW,GAAI,CAAC,CAAC,CAAC,IAAI,IACzG,CACP,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAgB;IACvC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AAC/E,CAAC"}
|
package/dist/ui/format.d.ts
CHANGED
|
@@ -2,6 +2,10 @@ import type { RunOutput, RunResult, RunSummary } from "../types/output.js";
|
|
|
2
2
|
import type { ScoreResult, ScoredOutput, ScoredRunResult } from "../types/scoring.js";
|
|
3
3
|
import type { ReportManifest } from "../types/report.js";
|
|
4
4
|
import type { Baseline, BaselineComparison } from "../types/baseline.js";
|
|
5
|
+
/** Strip @variant suffix to get the base scenario key. */
|
|
6
|
+
export declare function getBaseKey(scenarioKey: string): string;
|
|
7
|
+
/** Extract variant name from a scenarioKey, or null if none. */
|
|
8
|
+
export declare function getVariantName(scenarioKey: string): string | null;
|
|
5
9
|
export declare const STATUS_ICONS: Record<string, string>;
|
|
6
10
|
export declare const STATUS_LABELS: Record<string, string>;
|
|
7
11
|
export declare function formatDuration(ms: number): string;
|
package/dist/ui/format.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/ui/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE3E,OAAO,KAAK,EAAmC,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/ui/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE3E,OAAO,KAAK,EAAmC,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAkBzE,0DAA0D;AAC1D,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAGtD;AAED,gEAAgE;AAChE,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGjE;AAYD,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ/C,CAAC;AAEF,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQhD,CAAC;AAEF,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAGjD;AAED,uFAAuF;AACvF,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAK/D;AAmBD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,SAAK,GAAG,MAAM,CAO9D;AASD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI,CA0BnE;AAyBD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAmBlG;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAmC5D;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAmC5D;AAID,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAiBtG;AAkFD,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAmDrE;AAID,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,MAAM,CAkClE;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CA0EjE;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,GAAG,eAAe,GAAG,MAAM,CAuChF;AASD,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,CA4BhE;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAoC7D;AAQD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAmCzE"}
|
package/dist/ui/format.js
CHANGED
|
@@ -1,17 +1,33 @@
|
|
|
1
1
|
import { isScoredResult } from "../types/output.js";
|
|
2
2
|
// --- Layout constants ---
|
|
3
|
-
const COL_SCENARIO =
|
|
3
|
+
const COL_SCENARIO = 22;
|
|
4
4
|
const COL_SCENARIO_SCORED = 22;
|
|
5
|
-
const COL_AGENT =
|
|
5
|
+
const COL_AGENT = 25;
|
|
6
6
|
const COL_STATUS = 10;
|
|
7
7
|
const COL_DURATION = 10;
|
|
8
8
|
const COL_SCORE = 7;
|
|
9
|
-
const SEP_SUMMARY =
|
|
10
|
-
const SEP_SCORED =
|
|
11
|
-
const SEP_REPORT =
|
|
9
|
+
const SEP_SUMMARY = 72;
|
|
10
|
+
const SEP_SCORED = 102;
|
|
11
|
+
const SEP_REPORT = 100;
|
|
12
12
|
const SEP_DETAIL = 50;
|
|
13
13
|
const RESULT_TRUNCATE_SHORT = 200;
|
|
14
14
|
const RESULT_TRUNCATE_LONG = 500;
|
|
15
|
+
// --- Variant display helpers ---
|
|
16
|
+
/** Strip @variant suffix to get the base scenario key. */
|
|
17
|
+
export function getBaseKey(scenarioKey) {
|
|
18
|
+
const idx = scenarioKey.indexOf("@");
|
|
19
|
+
return idx === -1 ? scenarioKey : scenarioKey.slice(0, idx);
|
|
20
|
+
}
|
|
21
|
+
/** Extract variant name from a scenarioKey, or null if none. */
|
|
22
|
+
export function getVariantName(scenarioKey) {
|
|
23
|
+
const idx = scenarioKey.indexOf("@");
|
|
24
|
+
return idx === -1 ? null : scenarioKey.slice(idx + 1);
|
|
25
|
+
}
|
|
26
|
+
/** Build display agent name: "claude-code @variant" or plain "claude-code". */
|
|
27
|
+
function displayAgent(scenarioKey, agentName) {
|
|
28
|
+
const variant = getVariantName(scenarioKey);
|
|
29
|
+
return variant ? `${agentName} @${variant}` : agentName;
|
|
30
|
+
}
|
|
15
31
|
/** Threshold for high/medium criterion scores (out of 10). */
|
|
16
32
|
const CRITERION_HIGH = 8;
|
|
17
33
|
const CRITERION_MEDIUM = 4;
|
|
@@ -134,9 +150,9 @@ export function renderFinalOutput(output, verbose, agentCount) {
|
|
|
134
150
|
return "\n No scenarios matched.\n\n";
|
|
135
151
|
}
|
|
136
152
|
const agents = agentCount ?? new Set(output.results.map((r) => r.agentName)).size;
|
|
137
|
-
const
|
|
153
|
+
const scenarios = new Set(output.results.map((r) => getBaseKey(r.scenarioKey))).size;
|
|
138
154
|
let out = "\n";
|
|
139
|
-
out += ` AXIS — ${
|
|
155
|
+
out += ` AXIS — ${scenarios} scenario${scenarios !== 1 ? "s" : ""} for ${agents} agent${agents !== 1 ? "s" : ""}\n`;
|
|
140
156
|
if (verbose) {
|
|
141
157
|
for (const result of output.results) {
|
|
142
158
|
out += renderResultDetail(result);
|
|
@@ -158,7 +174,7 @@ export function renderSummaryTable(output) {
|
|
|
158
174
|
const status = failed ? "✗ fail" : "✓ pass";
|
|
159
175
|
const cost = meta.totalCostUsd ?? 0;
|
|
160
176
|
totalCost += cost;
|
|
161
|
-
lines.push(` ${result.scenarioKey.padEnd(COL_SCENARIO)} ${result.agentName.padEnd(COL_AGENT)} ${status.padEnd(COL_STATUS)} ${formatDuration(meta.durationMs).padEnd(COL_DURATION)} ${cost > 0 ? "$" + cost.toFixed(4) : "—"}`);
|
|
177
|
+
lines.push(` ${getBaseKey(result.scenarioKey).padEnd(COL_SCENARIO)} ${displayAgent(result.scenarioKey, result.agentName).padEnd(COL_AGENT)} ${status.padEnd(COL_STATUS)} ${formatDuration(meta.durationMs).padEnd(COL_DURATION)} ${cost > 0 ? "$" + cost.toFixed(4) : "—"}`);
|
|
162
178
|
if (meta.error) {
|
|
163
179
|
lines.push(` ↳ ${friendlyError(meta.error)}`);
|
|
164
180
|
}
|
|
@@ -202,9 +218,9 @@ export function renderScoredOutput(output, verbose, agentCount) {
|
|
|
202
218
|
return "\n No scenarios matched.\n\n";
|
|
203
219
|
}
|
|
204
220
|
const agents = agentCount ?? new Set(output.results.map((r) => r.agentName)).size;
|
|
205
|
-
const
|
|
221
|
+
const scenarios = new Set(output.results.map((r) => getBaseKey(r.scenarioKey))).size;
|
|
206
222
|
let out = "\n";
|
|
207
|
-
out += ` AXIS — ${
|
|
223
|
+
out += ` AXIS — ${scenarios} scenario${scenarios !== 1 ? "s" : ""} for ${agents} agent${agents !== 1 ? "s" : ""}\n`;
|
|
208
224
|
for (const result of output.results) {
|
|
209
225
|
out += renderScoredResult(result, verbose);
|
|
210
226
|
}
|
|
@@ -289,7 +305,7 @@ export function renderScoredSummaryTable(output) {
|
|
|
289
305
|
const meta = result.output.metadata;
|
|
290
306
|
const cost = meta.totalCostUsd ?? 0;
|
|
291
307
|
totalCost += cost;
|
|
292
|
-
lines.push(` ${result.scenarioKey.padEnd(COL_SCENARIO_SCORED)} ${result.agentName.padEnd(COL_AGENT)} ` +
|
|
308
|
+
lines.push(` ${getBaseKey(result.scenarioKey).padEnd(COL_SCENARIO_SCORED)} ${displayAgent(result.scenarioKey, result.agentName).padEnd(COL_AGENT)} ` +
|
|
293
309
|
`${String(result.score.axisScore).padEnd(COL_SCORE)} ` +
|
|
294
310
|
`${String(result.score.goalAchievement.score).padEnd(COL_SCORE)} ` +
|
|
295
311
|
`${String(result.score.environment.score).padEnd(COL_SCORE)} ` +
|
|
@@ -308,7 +324,7 @@ export function renderScoredSummaryTable(output) {
|
|
|
308
324
|
}
|
|
309
325
|
}
|
|
310
326
|
lines.push(` ${sep}`);
|
|
311
|
-
lines.push(` Average AXIS Result: ${output.summary.averageAxisScore} / 100`.padEnd(
|
|
327
|
+
lines.push(` Average AXIS Result: ${output.summary.averageAxisScore} / 100`.padEnd(82) +
|
|
312
328
|
`${formatDuration(output.durationMs).padEnd(COL_DURATION)} ` +
|
|
313
329
|
`${totalCost > 0 ? "$" + totalCost.toFixed(4) : ""}`);
|
|
314
330
|
if (output.summary.skipped) {
|
|
@@ -365,7 +381,7 @@ export function renderReportDetail(report) {
|
|
|
365
381
|
for (const r of report.results) {
|
|
366
382
|
const s = r.score;
|
|
367
383
|
const cost = r.totalCostUsd ?? 0;
|
|
368
|
-
lines.push(` ${r.scenarioKey.padEnd(COL_SCENARIO_SCORED)} ${r.agentName.padEnd(COL_AGENT)} ` +
|
|
384
|
+
lines.push(` ${getBaseKey(r.scenarioKey).padEnd(COL_SCENARIO_SCORED)} ${displayAgent(r.scenarioKey, r.agentName).padEnd(COL_AGENT)} ` +
|
|
369
385
|
`${s ? String(s.axisScore).padEnd(COL_SCORE) : "\u2014".padEnd(COL_SCORE)} ` +
|
|
370
386
|
`${s ? String(s.goalAchievement.score).padEnd(COL_SCORE) : "\u2014".padEnd(COL_SCORE)} ` +
|
|
371
387
|
`${s ? String(s.environment.score).padEnd(COL_SCORE) : "\u2014".padEnd(COL_SCORE)} ` +
|
|
@@ -390,7 +406,7 @@ export function renderReportDetail(report) {
|
|
|
390
406
|
for (const r of report.results) {
|
|
391
407
|
const status = r.exitCode !== 0 || r.error ? "\u2717 fail" : "\u2713 pass";
|
|
392
408
|
const cost = r.totalCostUsd ?? 0;
|
|
393
|
-
lines.push(` ${r.scenarioKey.padEnd(COL_SCENARIO)} ${r.agentName.padEnd(COL_AGENT)} ` +
|
|
409
|
+
lines.push(` ${getBaseKey(r.scenarioKey).padEnd(COL_SCENARIO)} ${displayAgent(r.scenarioKey, r.agentName).padEnd(COL_AGENT)} ` +
|
|
394
410
|
`${status.padEnd(COL_STATUS)} ${formatDuration(r.durationMs).padEnd(COL_DURATION)} ` +
|
|
395
411
|
`${cost > 0 ? "$" + cost.toFixed(4) : "\u2014"}`);
|
|
396
412
|
if (r.error) {
|
|
@@ -438,9 +454,9 @@ export function renderScenarioDetail(result) {
|
|
|
438
454
|
return lines.join("\n") + "\n";
|
|
439
455
|
}
|
|
440
456
|
// --- Baseline rendering ---
|
|
441
|
-
const SEP_BASELINE =
|
|
457
|
+
const SEP_BASELINE = 100;
|
|
442
458
|
const COL_BASELINE_SCENARIO = 22;
|
|
443
|
-
const COL_BASELINE_AGENT =
|
|
459
|
+
const COL_BASELINE_AGENT = 25;
|
|
444
460
|
const COL_BASELINE_SCORE = 10;
|
|
445
461
|
export function renderBaselineList(baselines) {
|
|
446
462
|
const lines = [];
|
|
@@ -479,7 +495,7 @@ export function renderBaselineShow(baseline) {
|
|
|
479
495
|
lines.push(` ${sep}`);
|
|
480
496
|
for (const [scenarioKey, agents] of Object.entries(baseline.results)) {
|
|
481
497
|
for (const [agentName, entry] of Object.entries(agents)) {
|
|
482
|
-
lines.push(` ${scenarioKey.padEnd(COL_BASELINE_SCENARIO)} ${agentName.padEnd(COL_BASELINE_AGENT)} ` +
|
|
498
|
+
lines.push(` ${getBaseKey(scenarioKey).padEnd(COL_BASELINE_SCENARIO)} ${displayAgent(scenarioKey, agentName).padEnd(COL_BASELINE_AGENT)} ` +
|
|
483
499
|
`${String(entry.axisScore).padEnd(COL_SCORE)} ` +
|
|
484
500
|
`${String(entry.goalAchievement).padEnd(COL_SCORE)} ` +
|
|
485
501
|
`${String(entry.environment).padEnd(COL_SCORE)} ` +
|
|
@@ -511,7 +527,7 @@ export function renderBaselineComparison(diff) {
|
|
|
511
527
|
`${"Baseline".padEnd(COL_BASELINE_SCORE)} ${"Current".padEnd(COL_BASELINE_SCORE)} Delta`);
|
|
512
528
|
lines.push(` ${sep}`);
|
|
513
529
|
for (const entry of diff.entries) {
|
|
514
|
-
lines.push(` ${entry.scenarioKey.padEnd(COL_BASELINE_SCENARIO)} ${entry.agentName.padEnd(COL_BASELINE_AGENT)} ` +
|
|
530
|
+
lines.push(` ${getBaseKey(entry.scenarioKey).padEnd(COL_BASELINE_SCENARIO)} ${displayAgent(entry.scenarioKey, entry.agentName).padEnd(COL_BASELINE_AGENT)} ` +
|
|
515
531
|
`${String(entry.baseline).padEnd(COL_BASELINE_SCORE)} ` +
|
|
516
532
|
`${String(entry.current).padEnd(COL_BASELINE_SCORE)} ` +
|
|
517
533
|
deltaIndicator(entry.delta));
|