@sverka/arena 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +38 -1
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +198 -1
- package/dist/index.mjs.map +1 -0
- package/package.json +6 -3
package/dist/index.d.mts
CHANGED
|
@@ -2,6 +2,9 @@ import { ChildProcess } from "node:child_process";
|
|
|
2
2
|
import * as acp from "@agentclientprotocol/sdk";
|
|
3
3
|
import { PromptResponse, SessionUpdate, Usage } from "@agentclientprotocol/sdk";
|
|
4
4
|
import { z } from "zod";
|
|
5
|
+
import { ReportContext } from "@sverka/reporter";
|
|
6
|
+
import { RunEvent } from "@sverka/runtime";
|
|
7
|
+
import { DefinitionGraph } from "@sverka/workflow";
|
|
5
8
|
//#region src/types.d.ts
|
|
6
9
|
/** Agent adapter — pluggable interface for different agent CLIs */
|
|
7
10
|
interface AgentAdapter {
|
|
@@ -493,5 +496,39 @@ export declare function loadArenaConfig(configPath: string): Promise<ArenaConfig
|
|
|
493
496
|
/** Render an ArenaResult as human-readable text. */
|
|
494
497
|
export declare function renderReport(result: ArenaResult): string;
|
|
495
498
|
//#endregion
|
|
496
|
-
|
|
499
|
+
//#region src/trace-report.d.ts
|
|
500
|
+
/** Relative duration weight per step kind — shares of the real total. */
|
|
501
|
+
type StepKind = "tool" | "think" | "message";
|
|
502
|
+
/** A trace step drives one Gantt bar. */
|
|
503
|
+
interface ActionStep {
|
|
504
|
+
stepId: string;
|
|
505
|
+
kind: StepKind;
|
|
506
|
+
label: string;
|
|
507
|
+
detail: string;
|
|
508
|
+
/** Set when this step should render as failed (run-level evidence). */
|
|
509
|
+
failed?: string;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Extracts a trace into action steps: agent activity only (system/user
|
|
513
|
+
* context is noise for the "what did the agent do" view), adjacent
|
|
514
|
+
* duplicates collapsed (the collector replays each turn twice).
|
|
515
|
+
*/
|
|
516
|
+
export declare function traceToActions(trace: TraceData): ActionStep[];
|
|
517
|
+
/**
|
|
518
|
+
* Convert a run into a sverka `RunEvent` stream. Per-step `at`/`durationMs`
|
|
519
|
+
* are synthetic — allocated from the real `executionTimeMs` total.
|
|
520
|
+
*/
|
|
521
|
+
export declare function traceToRunEvents(result: RunResult, opts?: {
|
|
522
|
+
runId?: string;
|
|
523
|
+
planId?: string;
|
|
524
|
+
}): RunEvent[];
|
|
525
|
+
/**
|
|
526
|
+
* Linear-chain graph so the report's DAG view shows the action
|
|
527
|
+
* sequence. Only the fields layoutDag reads are populated.
|
|
528
|
+
*/
|
|
529
|
+
export declare function traceGraph(actions: readonly ActionStep[]): DefinitionGraph;
|
|
530
|
+
/** Render a run's trace as a standalone sverka HTML report. */
|
|
531
|
+
export declare function writeTraceReport(result: RunResult, outputPath: string, context?: ReportContext): void;
|
|
532
|
+
//#endregion
|
|
533
|
+
export type { ActionStep, AgentAdapter, AgentProcess, AgentSpawnConfig, AggregateMetrics, ArenaConfig, ArenaConfigFile, ArenaResult, CaseAnalysis, CheckResult, ComboComparison, DeterministicCheck, JudgeConfig, JudgeVerdict, ModelConfig, Observation, PluginConfig, RunMetrics, RunResult, Task, ToolCall, ToolCallMessage, ToolCallUpdateMessage, TraceData, TraceStep };
|
|
497
534
|
//# sourceMappingURL=index.d.mts.map
|
package/dist/index.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/adapters/devin.ts","../src/acp.ts","../src/runner.ts","../src/judge.ts","../src/config.ts","../src/report.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/adapters/devin.ts","../src/acp.ts","../src/runner.ts","../src/judge.ts","../src/config.ts","../src/report.ts","../src/trace-report.ts"],"mappings":";;;;;;;;;UACiB;EACf;EACA,MAAM,QAAQ,mBAAmB;;UAGlB;EACf,OAAO;EACP;EACA,SAAS;EACT;EACA,MAAM;;UAGS;EACf,IAAI,gBAAgB,oBAAoB,QAAQ;EAChD;;;UAIe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;;;;EAKA;;;;EAIA;;;EAGA,SAAS;;;UAIM;EACf;EACA;EACA;;;UAIe;EACf,OAAO;EACP,OAAO;EACP,QAAQ;EACR,SAAS;;EAET;EACA;EACA;;EAEA,QAAQ;;;UAIO;;EAEf,OAAO;;EAEP,OAAO;;EAEP;;EAEA;;EAEA;;;UAIe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA,SAAS;EACT,OAAO;;EAEP;EACA;;EAEA,cAAc;EACd;;EAEA,UAAU;;;UAIK;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA,OAAO;EACP;IACE;IACA;IACA;IACA;;;;UAKa;EACf;EACA;EACA;EACA;EACA,YAAY;EACZ,eAAe;EACf;;;UAIe;EACf;EACA,WAAW;EACX;;;UAIe;EACf;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;;UAIe;EACf;EACA;IACE;IACA;IACA;IACA;IACA;;EAEF,SAAS;EACT,YAAY;;EAEZ,UAAU;;;UAIK;EACf;EACA;EACA;;EAEA,aAAa;;EAEb;;;UAIe;;EAEf;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;;;;UCxMQ;EACR;EACA;EACA;EACA;EACA;IACE;IACA;MACE;MACA;;;;;UAMI;EACR;EACA;EACA;EACA;;;UAIQ;EACR;EACA;EACA;IACE;IACA;IACA;;EAEF,eAAe;EACf,OAAO;;;wBAMO;;wBAiMA,cAAc,YAAY;;wBAmC1B,mBAAmB,OAAO;EACxC;EACA;EACA;EACA;;;;;;;;;;;wBA0DoB,eACpB,mBACA,SAAS,iBACR;;;;;;;;;;;;qBAyGU,wBAAwB;WAC1B;EAET,MAAM,QAAQ,mBAAmB;;;;;KCzcvB,kBAAkB,QAC5B;EACE;;;KAIQ,wBAAwB,QAClC;EACE;;;;;;;;;;;;wBA2EY,YAAY,QAAQ,mBAAmB;;;;;;;;wBAgBvC,2BAA2B,IAAI,qBAC7C,IAAI,0BACJ,IAAI;;;;;;;;;;;;;;;wBAqFgB,cACpB,MAAM,cACN,mBACA,gBACA,mBACA,cAAc,QAAQ,0BACtB,gBAAgB,QAAQ,gCACxB,aAAa,6BACZ,QAAQ;;;;;;;;;;;;;wBC3KK,mBAAmB,SAAS,iBAAiB;;;;;;;;;wBA0BvC,SAAS,QAAQ,cAAc,QAAQ;;;;;;;;wBAgU7C,iBACd,SAAS,aACT,SAAS,GAAG,uBACZ,iBACC;;;;;;;;;;wBAoIa,gBACd,SAAS,aACT,OAAO,SACN;;;;qBChgBU;;;;;;;;;wBAkBG,iBACd,MAAM,MACN,mBACA,QAAQ,aACR;;;;;;;;wBA0Cc,mBACd,kBACA,kBACA,KAAK,YACJ;;;;;wBAoDmB,SACpB,KAAK,WACL,MAAM,MACN,QAAQ,aACR,mBACC,QAAQ;;;;;;;wBAmCW,aACpB,SAAS,aACT,OAAO,QACP,QAAQ,cACP,QAAQ;;;;;;;;;;;;wBAoCK,cACd,UAAU,WACV,WAAW,YACV;;;qBCnNU,mBAAmB;WAGnB;EADT,YAAA,iBACS;;wBAaG,eAAe,eAAe;cA4CxC,kBAAgB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiBpB,EAAA,KAAA;;KAGU,kBAAkB,EAAE,aAAa;;wBAG7B,aAAa,QAAQ,kBAAkB;;;;;wBAUjC,gBACpB,qBACC,QAAQ;;;;wBCzDK,aAAa,QAAQ;;;;KClChC;;UASY;EACf;EACA,MAAM;EACN;EACA;;EAEA;;;;;;;wBAuCc,eAAe,OAAO,YAAY;;;;;wBA4DlC,iBACd,QAAQ,WACR;EAAS;EAAgB;IACxB;;;;;wBA6Da,WAAW,kBAAkB,eAAe;;wBAsB5C,iBACd,QAAQ,WACR,oBACA,UAAU"}
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1,199 @@
|
|
|
1
1
|
import { C as sanitizeEnv, S as runAcpSession, _ as countLlmCalls, a as resolveAdapter, b as transcriptDir, c as pluginCombinations, d as buildJudgePrompt, f as compareCombos, g as DevinAdapter, h as parseJudgeResponse, i as loadArenaConfig, l as runArena, m as judgeRun, n as ArenaError, o as aggregateResults, p as judgeAllRuns, r as defineConfig, s as computeAnalysis, t as renderReport, u as DEFAULT_JUDGE_PROMPT, v as extractUsageTokens, x as createPermissionHandler, y as installPlugins } from "./report-zoaX4hqr.mjs";
|
|
2
|
-
|
|
2
|
+
import { createHtmlRenderer } from "@sverka/reporter";
|
|
3
|
+
//#region src/trace-report.ts
|
|
4
|
+
function weight(kind) {
|
|
5
|
+
if (kind === "tool") return 4;
|
|
6
|
+
if (kind === "think") return 2;
|
|
7
|
+
return 1;
|
|
8
|
+
}
|
|
9
|
+
/** The argument that best summarizes a call (command line, path…). */
|
|
10
|
+
function keyArgument(call) {
|
|
11
|
+
const v = call.arguments["command"] ?? call.arguments["file_path"] ?? call.arguments["command_text"] ?? "";
|
|
12
|
+
return typeof v === "string" ? v.replace(/\s+/g, " ") : "";
|
|
13
|
+
}
|
|
14
|
+
/** Short, readable label for a tool call — function + key argument. */
|
|
15
|
+
function toolLabel(s) {
|
|
16
|
+
const calls = s.toolCalls ?? [];
|
|
17
|
+
const names = calls.map((c) => c.functionName).join(", ");
|
|
18
|
+
const arg = calls.length > 0 ? keyArgument(calls[0]) : "";
|
|
19
|
+
const base = arg ? `${names}: ${arg}` : names;
|
|
20
|
+
return base.length > 60 ? `${base.slice(0, 59)}…` : base;
|
|
21
|
+
}
|
|
22
|
+
function classify(s) {
|
|
23
|
+
if (s.toolCalls && s.toolCalls.length > 0) return "tool";
|
|
24
|
+
if (s.isLlmCall) return "think";
|
|
25
|
+
return "message";
|
|
26
|
+
}
|
|
27
|
+
function toLabel(s) {
|
|
28
|
+
if (s.toolCalls && s.toolCalls.length > 0) return toolLabel(s);
|
|
29
|
+
const oneLine = s.message.replace(/\s+/g, " ").trim();
|
|
30
|
+
return oneLine.length > 60 ? `${oneLine.slice(0, 59)}…` : oneLine;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Extracts a trace into action steps: agent activity only (system/user
|
|
34
|
+
* context is noise for the "what did the agent do" view), adjacent
|
|
35
|
+
* duplicates collapsed (the collector replays each turn twice).
|
|
36
|
+
*/
|
|
37
|
+
function traceToActions(trace) {
|
|
38
|
+
const out = [];
|
|
39
|
+
let prevKey;
|
|
40
|
+
let n = 0;
|
|
41
|
+
for (const s of trace.steps) {
|
|
42
|
+
if (s.source !== "agent") continue;
|
|
43
|
+
const callIds = (s.toolCalls ?? []).map((c) => c.toolCallId || c.functionName).join(",");
|
|
44
|
+
const key = `${s.isLlmCall}|${s.message}|${callIds}`;
|
|
45
|
+
if (key === prevKey) continue;
|
|
46
|
+
prevKey = key;
|
|
47
|
+
const kind = classify(s);
|
|
48
|
+
const label = toLabel(s);
|
|
49
|
+
const outputs = (s.observations ?? []).map((o) => o.content);
|
|
50
|
+
const detail = [s.message, ...outputs].filter(Boolean).join("\n\n");
|
|
51
|
+
out.push({
|
|
52
|
+
stepId: `s${String(++n).padStart(2, "0")} ${kind}: ${label}`,
|
|
53
|
+
kind,
|
|
54
|
+
label,
|
|
55
|
+
detail
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Evidence for a failed run: failing deterministic checks or the run
|
|
62
|
+
* error become a synthetic tail step — never pinned on an arbitrary
|
|
63
|
+
* agent action.
|
|
64
|
+
*/
|
|
65
|
+
function failureStep(result, n) {
|
|
66
|
+
if (result.success) return void 0;
|
|
67
|
+
const failedChecks = result.checkResults.filter((c) => !c.passed);
|
|
68
|
+
if (failedChecks.length === 0 && !result.error) return void 0;
|
|
69
|
+
const detail = [...failedChecks.map((c) => `${c.checkId}: ${c.output}`), result.error ?? ""].filter(Boolean).join("\n");
|
|
70
|
+
return {
|
|
71
|
+
stepId: `s${String(n + 1).padStart(2, "0")} checks: deterministic`,
|
|
72
|
+
kind: "tool",
|
|
73
|
+
label: "deterministic checks",
|
|
74
|
+
detail,
|
|
75
|
+
failed: result.error ?? `${failedChecks.length} check(s) failed`
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Convert a run into a sverka `RunEvent` stream. Per-step `at`/`durationMs`
|
|
80
|
+
* are synthetic — allocated from the real `executionTimeMs` total.
|
|
81
|
+
*/
|
|
82
|
+
function traceToRunEvents(result, opts) {
|
|
83
|
+
const actions = traceToActions(result.trace);
|
|
84
|
+
const failed = failureStep(result, actions.length);
|
|
85
|
+
const steps = failed ? [...actions, failed] : actions;
|
|
86
|
+
const totalMs = Math.max(result.metrics.executionTimeMs || 0, 1);
|
|
87
|
+
const totalWeight = steps.reduce((a, s) => a + weight(s.kind), 0) || 1;
|
|
88
|
+
const runId = opts?.runId ?? `arena-${result.taskId}`;
|
|
89
|
+
const events = [{
|
|
90
|
+
type: "run-started",
|
|
91
|
+
runId,
|
|
92
|
+
planId: opts?.planId ?? result.taskId,
|
|
93
|
+
at: 0
|
|
94
|
+
}];
|
|
95
|
+
let t = 0;
|
|
96
|
+
steps.forEach((a, i) => {
|
|
97
|
+
const dur = i === steps.length - 1 ? Math.max(totalMs - t, 0) : Math.round(totalMs * weight(a.kind) / totalWeight);
|
|
98
|
+
events.push({
|
|
99
|
+
type: "step-started",
|
|
100
|
+
stepId: a.stepId,
|
|
101
|
+
at: t
|
|
102
|
+
});
|
|
103
|
+
events.push(a.failed ? {
|
|
104
|
+
type: "step-failed",
|
|
105
|
+
stepId: a.stepId,
|
|
106
|
+
error: a.failed,
|
|
107
|
+
durationMs: dur,
|
|
108
|
+
stdout: a.detail,
|
|
109
|
+
at: t + dur
|
|
110
|
+
} : {
|
|
111
|
+
type: "step-succeeded",
|
|
112
|
+
stepId: a.stepId,
|
|
113
|
+
durationMs: dur,
|
|
114
|
+
stdout: a.detail,
|
|
115
|
+
at: t + dur
|
|
116
|
+
});
|
|
117
|
+
t += dur;
|
|
118
|
+
});
|
|
119
|
+
events.push({
|
|
120
|
+
type: "run-completed",
|
|
121
|
+
runId,
|
|
122
|
+
status: result.success ? "success" : "failure",
|
|
123
|
+
durationMs: t,
|
|
124
|
+
at: t
|
|
125
|
+
});
|
|
126
|
+
return events;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Linear-chain graph so the report's DAG view shows the action
|
|
130
|
+
* sequence. Only the fields layoutDag reads are populated.
|
|
131
|
+
*/
|
|
132
|
+
function traceGraph(actions) {
|
|
133
|
+
let prev = "";
|
|
134
|
+
return { project: {
|
|
135
|
+
id: "arena-trace",
|
|
136
|
+
pipelines: [{
|
|
137
|
+
id: "agent-run",
|
|
138
|
+
inputs: {},
|
|
139
|
+
entries: [],
|
|
140
|
+
steps: actions.map((a) => {
|
|
141
|
+
const dependencies = prev ? [{
|
|
142
|
+
kind: "control",
|
|
143
|
+
producer: prev
|
|
144
|
+
}] : [];
|
|
145
|
+
prev = a.stepId;
|
|
146
|
+
return {
|
|
147
|
+
id: a.stepId,
|
|
148
|
+
dependencies
|
|
149
|
+
};
|
|
150
|
+
}),
|
|
151
|
+
outputs: []
|
|
152
|
+
}]
|
|
153
|
+
} };
|
|
154
|
+
}
|
|
155
|
+
/** Render a run's trace as a standalone sverka HTML report. */
|
|
156
|
+
function writeTraceReport(result, outputPath, context) {
|
|
157
|
+
const actions = traceToActions(result.trace);
|
|
158
|
+
const combo = result.pluginIds.length ? result.pluginIds.join("+") : "no-plugins";
|
|
159
|
+
const meta = [
|
|
160
|
+
{
|
|
161
|
+
label: "task",
|
|
162
|
+
value: result.taskId
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
label: "model",
|
|
166
|
+
value: result.modelId
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
label: "plugins",
|
|
170
|
+
value: combo
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
label: "success",
|
|
174
|
+
value: String(result.success)
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
label: "timing",
|
|
178
|
+
value: "durations estimated — ACP traces have no per-step timestamps; total is real"
|
|
179
|
+
},
|
|
180
|
+
...context?.meta ?? []
|
|
181
|
+
];
|
|
182
|
+
const renderer = createHtmlRenderer({
|
|
183
|
+
outputPath,
|
|
184
|
+
graph: traceGraph(actions),
|
|
185
|
+
context: {
|
|
186
|
+
title: `arena: ${result.taskId} — ${combo}`,
|
|
187
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
188
|
+
command: "sverka-arena run",
|
|
189
|
+
...context,
|
|
190
|
+
meta
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
for (const e of traceToRunEvents(result)) renderer.onEvent(e);
|
|
194
|
+
renderer.flush();
|
|
195
|
+
}
|
|
196
|
+
//#endregion
|
|
197
|
+
export { ArenaError, DEFAULT_JUDGE_PROMPT, DevinAdapter, aggregateResults, buildJudgePrompt, compareCombos, computeAnalysis, countLlmCalls, createPermissionHandler, defineConfig, extractUsageTokens, installPlugins, judgeAllRuns, judgeRun, loadArenaConfig, parseJudgeResponse, pluginCombinations, renderReport, resolveAdapter, runAcpSession, runArena, sanitizeEnv, traceGraph, traceToActions, traceToRunEvents, transcriptDir, writeTraceReport };
|
|
198
|
+
|
|
199
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../src/trace-report.ts"],"sourcesContent":["/**\n * Arena trace → sverka report. Spec 52.\n *\n * The ACP collector stamps every trace step with the same timestamp\n * (session replay), so real per-step timing does not exist. We still\n * know the true wall-clock total (`metrics.executionTimeMs`), so the\n * report synthesizes per-step durations by allocating the real total\n * across steps weighted by kind — tool calls, LLM thinking, plain\n * messages. Honest totals, estimated breakdown — marked in context.\n */\nimport type { RunEvent } from \"@sverka/runtime\";\nimport type { DefinitionGraph } from \"@sverka/workflow\";\nimport type { ReportContext } from \"@sverka/reporter\";\nimport { createHtmlRenderer } from \"@sverka/reporter\";\nimport type { RunResult, TraceData, TraceStep, ToolCall } from \"./types.js\";\n\n/** Relative duration weight per step kind — shares of the real total. */\ntype StepKind = \"tool\" | \"think\" | \"message\";\n\nfunction weight(kind: StepKind): number {\n if (kind === \"tool\") return 4;\n if (kind === \"think\") return 2;\n return 1;\n}\n\n/** A trace step drives one Gantt bar. */\nexport interface ActionStep {\n stepId: string;\n kind: StepKind;\n label: string;\n detail: string;\n /** Set when this step should render as failed (run-level evidence). */\n failed?: string;\n}\n\n/** The argument that best summarizes a call (command line, path…). */\nfunction keyArgument(call: ToolCall): string {\n const v =\n call.arguments[\"command\"] ??\n call.arguments[\"file_path\"] ??\n call.arguments[\"command_text\"] ??\n \"\";\n return typeof v === \"string\" ? v.replace(/\\s+/g, \" \") : \"\";\n}\n\n/** Short, readable label for a tool call — function + key argument. */\nfunction toolLabel(s: TraceStep): string {\n const calls = s.toolCalls ?? [];\n const names = calls.map((c) => c.functionName).join(\", \");\n const arg = calls.length > 0 ? keyArgument(calls[0]!) : \"\";\n const base = arg ? `${names}: ${arg}` : names;\n return base.length > 60 ? `${base.slice(0, 59)}…` : base;\n}\n\nfunction classify(s: TraceStep): StepKind {\n if (s.toolCalls && s.toolCalls.length > 0) return \"tool\";\n if (s.isLlmCall) return \"think\";\n return \"message\";\n}\n\nfunction toLabel(s: TraceStep): string {\n if (s.toolCalls && s.toolCalls.length > 0) return toolLabel(s);\n const oneLine = s.message.replace(/\\s+/g, \" \").trim();\n return oneLine.length > 60 ? `${oneLine.slice(0, 59)}…` : oneLine;\n}\n\n/**\n * Extracts a trace into action steps: agent activity only (system/user\n * context is noise for the \"what did the agent do\" view), adjacent\n * duplicates collapsed (the collector replays each turn twice).\n */\nexport function traceToActions(trace: TraceData): ActionStep[] {\n const out: ActionStep[] = [];\n let prevKey: string | undefined;\n let n = 0;\n for (const s of trace.steps) {\n if (s.source !== \"agent\") continue;\n // Dedupe replayed steps — the collector re-emits a turn once per\n // tool call (same message) and again per observation, so identity\n // must include the call ids, not just the message text.\n const callIds = (s.toolCalls ?? [])\n .map((c) => c.toolCallId || c.functionName)\n .join(\",\");\n const key = `${s.isLlmCall}|${s.message}|${callIds}`;\n if (key === prevKey) continue;\n prevKey = key;\n const kind = classify(s);\n const label = toLabel(s);\n // For tool steps the observations are the tool's own output — pair\n // them under the step that made the call (the message is the\n // preceding reasoning, kept on top for context).\n const outputs = (s.observations ?? []).map((o) => o.content);\n const detail = [s.message, ...outputs].filter(Boolean).join(\"\\n\\n\");\n out.push({\n stepId: `s${String(++n).padStart(2, \"0\")} ${kind}: ${label}`,\n kind,\n label,\n detail,\n });\n }\n return out;\n}\n\n/**\n * Evidence for a failed run: failing deterministic checks or the run\n * error become a synthetic tail step — never pinned on an arbitrary\n * agent action.\n */\nfunction failureStep(result: RunResult, n: number): ActionStep | undefined {\n if (result.success) return undefined;\n const failedChecks = result.checkResults.filter((c) => !c.passed);\n if (failedChecks.length === 0 && !result.error) return undefined;\n const detail = [\n ...failedChecks.map((c) => `${c.checkId}: ${c.output}`),\n result.error ?? \"\",\n ]\n .filter(Boolean)\n .join(\"\\n\");\n return {\n stepId: `s${String(n + 1).padStart(2, \"0\")} checks: deterministic`,\n kind: \"tool\",\n label: \"deterministic checks\",\n detail,\n failed: result.error ?? `${failedChecks.length} check(s) failed`,\n };\n}\n\n/**\n * Convert a run into a sverka `RunEvent` stream. Per-step `at`/`durationMs`\n * are synthetic — allocated from the real `executionTimeMs` total.\n */\nexport function traceToRunEvents(\n result: RunResult,\n opts?: { runId?: string; planId?: string },\n): RunEvent[] {\n const actions = traceToActions(result.trace);\n const failed = failureStep(result, actions.length);\n const steps = failed ? [...actions, failed] : actions;\n const totalMs = Math.max(result.metrics.executionTimeMs || 0, 1);\n const totalWeight = steps.reduce((a, s) => a + weight(s.kind), 0) || 1;\n const runId = opts?.runId ?? `arena-${result.taskId}`;\n\n const events: RunEvent[] = [\n {\n type: \"run-started\",\n runId,\n planId: opts?.planId ?? result.taskId,\n at: 0,\n },\n ];\n\n let t = 0;\n steps.forEach((a, i) => {\n // Weighted share of the REAL total — the last step absorbs the\n // rounding remainder so durations always sum to executionTimeMs.\n const dur =\n i === steps.length - 1\n ? Math.max(totalMs - t, 0)\n : Math.round((totalMs * weight(a.kind)) / totalWeight);\n events.push({ type: \"step-started\", stepId: a.stepId, at: t });\n events.push(\n a.failed\n ? {\n type: \"step-failed\",\n stepId: a.stepId,\n error: a.failed,\n durationMs: dur,\n stdout: a.detail,\n at: t + dur,\n }\n : {\n type: \"step-succeeded\",\n stepId: a.stepId,\n durationMs: dur,\n stdout: a.detail,\n at: t + dur,\n },\n );\n t += dur;\n });\n\n events.push({\n type: \"run-completed\",\n runId,\n status: result.success ? \"success\" : \"failure\",\n durationMs: t,\n at: t,\n });\n return events;\n}\n\n/**\n * Linear-chain graph so the report's DAG view shows the action\n * sequence. Only the fields layoutDag reads are populated.\n */\nexport function traceGraph(actions: readonly ActionStep[]): DefinitionGraph {\n let prev = \"\";\n const steps = actions.map((a) => {\n const dependencies = prev\n ? [{ kind: \"control\" as const, producer: prev }]\n : [];\n prev = a.stepId;\n return { id: a.stepId, dependencies };\n });\n // Minimal graph — layoutDag only reads project.pipelines[].steps\n // ({id, dependencies}).\n return {\n project: {\n id: \"arena-trace\",\n pipelines: [\n { id: \"agent-run\", inputs: {}, entries: [], steps, outputs: [] },\n ],\n },\n } as unknown as DefinitionGraph;\n}\n\n/** Render a run's trace as a standalone sverka HTML report. */\nexport function writeTraceReport(\n result: RunResult,\n outputPath: string,\n context?: ReportContext,\n): void {\n const actions = traceToActions(result.trace);\n const combo = result.pluginIds.length\n ? result.pluginIds.join(\"+\")\n : \"no-plugins\";\n // Caller meta extends the defaults — the synthetic-timing row must\n // survive a custom context.\n const meta = [\n { label: \"task\", value: result.taskId },\n { label: \"model\", value: result.modelId },\n { label: \"plugins\", value: combo },\n { label: \"success\", value: String(result.success) },\n {\n label: \"timing\",\n value:\n \"durations estimated — ACP traces have no per-step timestamps; total is real\",\n },\n ...(context?.meta ?? []),\n ];\n const renderer = createHtmlRenderer({\n outputPath,\n graph: traceGraph(actions),\n context: {\n title: `arena: ${result.taskId} — ${combo}`,\n generatedAt: new Date().toISOString(),\n command: \"sverka-arena run\",\n ...context,\n meta,\n },\n });\n for (const e of traceToRunEvents(result)) renderer.onEvent(e);\n renderer.flush();\n}\n"],"mappings":";;;AAmBA,SAAS,OAAO,MAAwB;CACtC,IAAI,SAAS,QAAQ,OAAO;CAC5B,IAAI,SAAS,SAAS,OAAO;CAC7B,OAAO;AACT;;AAaA,SAAS,YAAY,MAAwB;CAC3C,MAAM,IACJ,KAAK,UAAU,cACf,KAAK,UAAU,gBACf,KAAK,UAAU,mBACf;CACF,OAAO,OAAO,MAAM,WAAW,EAAE,QAAQ,QAAQ,GAAG,IAAI;AAC1D;;AAGA,SAAS,UAAU,GAAsB;CACvC,MAAM,QAAQ,EAAE,aAAa,CAAC;CAC9B,MAAM,QAAQ,MAAM,KAAK,MAAM,EAAE,YAAY,CAAC,CAAC,KAAK,IAAI;CACxD,MAAM,MAAM,MAAM,SAAS,IAAI,YAAY,MAAM,EAAG,IAAI;CACxD,MAAM,OAAO,MAAM,GAAG,MAAM,IAAI,QAAQ;CACxC,OAAO,KAAK,SAAS,KAAK,GAAG,KAAK,MAAM,GAAG,EAAE,EAAE,KAAK;AACtD;AAEA,SAAS,SAAS,GAAwB;CACxC,IAAI,EAAE,aAAa,EAAE,UAAU,SAAS,GAAG,OAAO;CAClD,IAAI,EAAE,WAAW,OAAO;CACxB,OAAO;AACT;AAEA,SAAS,QAAQ,GAAsB;CACrC,IAAI,EAAE,aAAa,EAAE,UAAU,SAAS,GAAG,OAAO,UAAU,CAAC;CAC7D,MAAM,UAAU,EAAE,QAAQ,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CACpD,OAAO,QAAQ,SAAS,KAAK,GAAG,QAAQ,MAAM,GAAG,EAAE,EAAE,KAAK;AAC5D;;;;;;AAOA,SAAgB,eAAe,OAAgC;CAC7D,MAAM,MAAoB,CAAC;CAC3B,IAAI;CACJ,IAAI,IAAI;CACR,KAAK,MAAM,KAAK,MAAM,OAAO;EAC3B,IAAI,EAAE,WAAW,SAAS;EAI1B,MAAM,WAAW,EAAE,aAAa,CAAC,EAAA,CAC9B,KAAK,MAAM,EAAE,cAAc,EAAE,YAAY,CAAC,CAC1C,KAAK,GAAG;EACX,MAAM,MAAM,GAAG,EAAE,UAAU,GAAG,EAAE,QAAQ,GAAG;EAC3C,IAAI,QAAQ,SAAS;EACrB,UAAU;EACV,MAAM,OAAO,SAAS,CAAC;EACvB,MAAM,QAAQ,QAAQ,CAAC;EAIvB,MAAM,WAAW,EAAE,gBAAgB,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,OAAO;EAC3D,MAAM,SAAS,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,MAAM;EAClE,IAAI,KAAK;GACP,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE,GAAG,KAAK,IAAI;GACrD;GACA;GACA;EACF,CAAC;CACH;CACA,OAAO;AACT;;;;;;AAOA,SAAS,YAAY,QAAmB,GAAmC;CACzE,IAAI,OAAO,SAAS,OAAO,KAAA;CAC3B,MAAM,eAAe,OAAO,aAAa,QAAQ,MAAM,CAAC,EAAE,MAAM;CAChE,IAAI,aAAa,WAAW,KAAK,CAAC,OAAO,OAAO,OAAO,KAAA;CACvD,MAAM,SAAS,CACb,GAAG,aAAa,KAAK,MAAM,GAAG,EAAE,QAAQ,IAAI,EAAE,QAAQ,GACtD,OAAO,SAAS,EAClB,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;CACZ,OAAO;EACL,QAAQ,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE;EAC3C,MAAM;EACN,OAAO;EACP;EACA,QAAQ,OAAO,SAAS,GAAG,aAAa,OAAO;CACjD;AACF;;;;;AAMA,SAAgB,iBACd,QACA,MACY;CACZ,MAAM,UAAU,eAAe,OAAO,KAAK;CAC3C,MAAM,SAAS,YAAY,QAAQ,QAAQ,MAAM;CACjD,MAAM,QAAQ,SAAS,CAAC,GAAG,SAAS,MAAM,IAAI;CAC9C,MAAM,UAAU,KAAK,IAAI,OAAO,QAAQ,mBAAmB,GAAG,CAAC;CAC/D,MAAM,cAAc,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,EAAE,IAAI,GAAG,CAAC,KAAK;CACrE,MAAM,QAAQ,MAAM,SAAS,SAAS,OAAO;CAE7C,MAAM,SAAqB,CACzB;EACE,MAAM;EACN;EACA,QAAQ,MAAM,UAAU,OAAO;EAC/B,IAAI;CACN,CACF;CAEA,IAAI,IAAI;CACR,MAAM,SAAS,GAAG,MAAM;EAGtB,MAAM,MACJ,MAAM,MAAM,SAAS,IACjB,KAAK,IAAI,UAAU,GAAG,CAAC,IACvB,KAAK,MAAO,UAAU,OAAO,EAAE,IAAI,IAAK,WAAW;EACzD,OAAO,KAAK;GAAE,MAAM;GAAgB,QAAQ,EAAE;GAAQ,IAAI;EAAE,CAAC;EAC7D,OAAO,KACL,EAAE,SACE;GACE,MAAM;GACN,QAAQ,EAAE;GACV,OAAO,EAAE;GACT,YAAY;GACZ,QAAQ,EAAE;GACV,IAAI,IAAI;EACV,IACA;GACE,MAAM;GACN,QAAQ,EAAE;GACV,YAAY;GACZ,QAAQ,EAAE;GACV,IAAI,IAAI;EACV,CACN;EACA,KAAK;CACP,CAAC;CAED,OAAO,KAAK;EACV,MAAM;EACN;EACA,QAAQ,OAAO,UAAU,YAAY;EACrC,YAAY;EACZ,IAAI;CACN,CAAC;CACD,OAAO;AACT;;;;;AAMA,SAAgB,WAAW,SAAiD;CAC1E,IAAI,OAAO;CAUX,OAAO,EACL,SAAS;EACP,IAAI;EACJ,WAAW,CACT;GAAE,IAAI;GAAa,QAAQ,CAAC;GAAG,SAAS,CAAC;GAAG,OAbpC,QAAQ,KAAK,MAAM;IAC/B,MAAM,eAAe,OACjB,CAAC;KAAE,MAAM;KAAoB,UAAU;IAAK,CAAC,IAC7C,CAAC;IACL,OAAO,EAAE;IACT,OAAO;KAAE,IAAI,EAAE;KAAQ;IAAa;GACtC,CAOsD;GAAG,SAAS,CAAC;EAAE,CACjE;CACF,EACF;AACF;;AAGA,SAAgB,iBACd,QACA,YACA,SACM;CACN,MAAM,UAAU,eAAe,OAAO,KAAK;CAC3C,MAAM,QAAQ,OAAO,UAAU,SAC3B,OAAO,UAAU,KAAK,GAAG,IACzB;CAGJ,MAAM,OAAO;EACX;GAAE,OAAO;GAAQ,OAAO,OAAO;EAAO;EACtC;GAAE,OAAO;GAAS,OAAO,OAAO;EAAQ;EACxC;GAAE,OAAO;GAAW,OAAO;EAAM;EACjC;GAAE,OAAO;GAAW,OAAO,OAAO,OAAO,OAAO;EAAE;EAClD;GACE,OAAO;GACP,OACE;EACJ;EACA,GAAI,SAAS,QAAQ,CAAC;CACxB;CACA,MAAM,WAAW,mBAAmB;EAClC;EACA,OAAO,WAAW,OAAO;EACzB,SAAS;GACP,OAAO,UAAU,OAAO,OAAO,KAAK;GACpC,8BAAa,IAAI,KAAK,EAAA,CAAE,YAAY;GACpC,SAAS;GACT,GAAG;GACH;EACF;CACF,CAAC;CACD,KAAK,MAAM,KAAK,iBAAiB,MAAM,GAAG,SAAS,QAAQ,CAAC;CAC5D,SAAS,MAAM;AACjB"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sverka/arena",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "Sverka agent arena
|
|
3
|
+
"version": "0.1.21",
|
|
4
|
+
"description": "Sverka agent arena — run agents against verification workflows",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"sideEffects": false,
|
|
7
7
|
"repository": {
|
|
@@ -35,7 +35,10 @@
|
|
|
35
35
|
},
|
|
36
36
|
"dependencies": {
|
|
37
37
|
"@agentclientprotocol/sdk": "^1.5.0",
|
|
38
|
-
"zod": "^4.6.5"
|
|
38
|
+
"zod": "^4.6.5",
|
|
39
|
+
"@sverka/runtime": "^0.1.21",
|
|
40
|
+
"@sverka/reporter": "^0.1.21",
|
|
41
|
+
"@sverka/workflow": "^0.1.21"
|
|
39
42
|
},
|
|
40
43
|
"devDependencies": {
|
|
41
44
|
"vitest": "^5.0.0",
|