@sverka/benchmark 0.0.0 → 0.1.2

package/dist/bin.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/bin.mjs

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+import { c as BENCHMARK_TASKS, f as runBenchmark, l as DEFAULT_AGENTS, p as writeReport } from "./src-PvGPQO8c.mjs";
+import { join } from "node:path";
+import process from "node:process";
+//#region src/bin.ts
+async function main() {
+	const model = process.env.DEVIN_MODEL ?? "glm-5-2";
+	const outputDir = process.argv[2] ?? process.cwd();
+	process.stderr.write(`Sverka Benchmark Arena\n`);
+	process.stderr.write(`Model: ${model}\n`);
+	process.stderr.write(`Tasks: ${BENCHMARK_TASKS.length}\n`);
+	process.stderr.write(`Agents: ${DEFAULT_AGENTS.length}\n\n`);
+	const result = await runBenchmark({
+		tasks: [...BENCHMARK_TASKS],
+		agents: [...DEFAULT_AGENTS],
+		model,
+		outputDir
+	});
+	const filename = `benchmark-${result.timestamp.replace(/[:.]/g, "-")}.json`;
+	const outputPath = join(outputDir, filename);
+	await writeReport(result, outputPath);
+	process.stderr.write(`\nReport written to: ${outputPath}\n`);
+	process.stderr.write(`\nSummary:\n`);
+	process.stderr.write(`  raw-shell: ${result.summary["raw-shell"].successCount}/${result.summary["raw-shell"].totalTasks} pass, avg ${result.summary["raw-shell"].avgTotalTokens} tokens\n`);
+	process.stderr.write(`  sverka:    ${result.summary.sverka.successCount}/${result.summary.sverka.totalTasks} pass, avg ${result.summary.sverka.avgTotalTokens} tokens\n`);
+	return 0;
+}
+main().then(process.exit).catch((err) => {
+	process.stderr.write(`Fatal: ${err}\n`);
+	process.exit(1);
+});
+//#endregion
+export {};
+
+//# sourceMappingURL=bin.mjs.map

package/dist/bin.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\nimport process from \"node:process\";\nimport { join } from \"node:path\";\nimport { runBenchmark, writeReport, BENCHMARK_TASKS, DEFAULT_AGENTS } from \"./index.js\";\n\nasync function main(): Promise<number> {\n  const model = process.env.DEVIN_MODEL ?? \"glm-5-2\";\n  const outputDir = process.argv[2] ?? process.cwd();\n\n  process.stderr.write(`Sverka Benchmark Arena\\n`);\n  process.stderr.write(`Model: ${model}\\n`);\n  process.stderr.write(`Tasks: ${BENCHMARK_TASKS.length}\\n`);\n  process.stderr.write(`Agents: ${DEFAULT_AGENTS.length}\\n\\n`);\n\n  const result = await runBenchmark({\n    tasks: [...BENCHMARK_TASKS],\n    agents: [...DEFAULT_AGENTS],\n    model,\n    outputDir,\n  });\n\n  const filename = `benchmark-${result.timestamp.replace(/[:.]/g, \"-\")}.json`;\n  const outputPath = join(outputDir, filename);\n  await writeReport(result, outputPath);\n\n  process.stderr.write(`\\nReport written to: ${outputPath}\\n`);\n  process.stderr.write(`\\nSummary:\\n`);\n  process.stderr.write(`  raw-shell: ${result.summary[\"raw-shell\"].successCount}/${result.summary[\"raw-shell\"].totalTasks} pass, avg ${result.summary[\"raw-shell\"].avgTotalTokens} tokens\\n`);\n  process.stderr.write(`  sverka:    ${result.summary.sverka.successCount}/${result.summary.sverka.totalTasks} pass, avg ${result.summary.sverka.avgTotalTokens} tokens\\n`);\n\n  return 0;\n}\n\nmain().then(process.exit).catch((err) => {\n  process.stderr.write(`Fatal: ${err}\\n`);\n  process.exit(1);\n});\n"],"mappings":";;;;;AAKA,eAAe,OAAwB;CACrC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,YAAY,QAAQ,KAAK,MAAM,QAAQ,IAAI;CAEjD,QAAQ,OAAO,MAAM,0BAA0B;CAC/C,QAAQ,OAAO,MAAM,UAAU,MAAM,GAAG;CACxC,QAAQ,OAAO,MAAM,UAAU,gBAAgB,OAAO,GAAG;CACzD,QAAQ,OAAO,MAAM,WAAW,eAAe,OAAO,KAAK;CAE3D,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,CAAC,GAAG,eAAe;EAC1B,QAAQ,CAAC,GAAG,cAAc;EAC1B;EACA;CACF,CAAC;CAED,MAAM,WAAW,aAAa,OAAO,UAAU,QAAQ,SAAS,GAAG,EAAE;CACrE,MAAM,aAAa,KAAK,WAAW,QAAQ;CAC3C,MAAM,YAAY,QAAQ,UAAU;CAEpC,QAAQ,OAAO,MAAM,wBAAwB,WAAW,GAAG;CAC3D,QAAQ,OAAO,MAAM,cAAc;CACnC,QAAQ,OAAO,MAAM,gBAAgB,OAAO,QAAQ,YAAY,CAAC,aAAa,GAAG,OAAO,QAAQ,YAAY,CAAC,WAAW,aAAa,OAAO,QAAQ,YAAY,CAAC,eAAe,UAAU;CAC1L,QAAQ,OAAO,MAAM,gBAAgB,OAAO,QAAQ,OAAO,aAAa,GAAG,OAAO,QAAQ,OAAO,WAAW,aAAa,OAAO,QAAQ,OAAO,eAAe,UAAU;CAExK,OAAO;AACT;AAEA,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,QAAQ;CACvC,QAAQ,OAAO,MAAM,UAAU,IAAI,GAAG;CACtC,QAAQ,KAAK,CAAC;AAChB,CAAC"}

package/dist/index.d.mts

@@ -0,0 +1,164 @@
+import { Usage } from "@agentclientprotocol/sdk";
+//#region src/types.d.ts
+/** A benchmark task scenario. */
+interface Task {
+  id: string;
+  name: string;
+  prompt: string;
+  timeoutMs?: number;
+}
+/** Agent configuration — determines workspace setup. */
+interface AgentConfig {
+  id: string;
+  name: string;
+  type: "raw-shell" | "sverka";
+}
+/** Metrics collected from ACP events + timing. */
+interface RunMetrics {
+  inputTokens: number;
+  outputTokens: number;
+  thoughtTokens: number;
+  totalTokens: number;
+  toolCallCount: number;
+  executionTimeMs: number;
+  stopReason: string;
+}
+/** Result of a single task+agent run. */
+interface RunResult {
+  taskId: string;
+  agentId: string;
+  agentType: string;
+  metrics: RunMetrics;
+  success: boolean;
+  error?: string;
+}
+/** Aggregate metrics for one agent type across all tasks. */
+interface AggregateMetrics {
+  totalTasks: number;
+  successCount: number;
+  avgInputTokens: number;
+  avgOutputTokens: number;
+  avgTotalTokens: number;
+  avgToolCalls: number;
+  avgExecutionTimeMs: number;
+}
+/** Full benchmark output. */
+interface BenchmarkResult {
+  timestamp: string;
+  model: string;
+  results: RunResult[];
+  summary: {
+    "raw-shell": AggregateMetrics;
+    sverka: AggregateMetrics;
+  };
+}
+/** Arena configuration. */
+interface BenchmarkConfig {
+  tasks: Task[];
+  agents: AgentConfig[];
+  model?: string;
+  outputDir?: string;
+}
+//#endregion
+//#region src/arena.d.ts
+/** Extract RunMetrics from a PromptResponse + collected event data. */
+export declare function extractMetrics(promptResult: {
+  stopReason: string;
+  usage?: Usage | null;
+}, toolCallCount: number, startTime: number): RunMetrics;
+/** Aggregate metrics for one agent type across multiple RunResults. */
+export declare function aggregateMetrics(results: RunResult[], agentType: string): AggregateMetrics;
+/** Run the full benchmark: each task × each agent, sequentially. */
+export declare function runBenchmark(config: BenchmarkConfig): Promise<BenchmarkResult>;
+/** Write benchmark results to a JSON file. */
+export declare function writeReport(result: BenchmarkResult, outputPath: string): Promise<void>;
+//#endregion
+//#region src/tasks.d.ts
+/** 5 task scenarios, ordered simple → complex. */
+export declare const BENCHMARK_TASKS: readonly Task[];
+/** Default agent configs: raw-shell (no sverka) vs sverka (CLI + skill). */
+export declare const DEFAULT_AGENTS: readonly AgentConfig[];
+//#endregion
+//#region src/collector.d.ts
+/**
+ * Transcript collector — reads Devin CLI transcripts and transforms them
+ * to trace data for the benchmark trace viewer.
+ *
+ * Transcript source: ~/.local/share/devin/cli/transcripts/<session-id>.json
+ * Each transcript has steps[] with source, message, timestamp, and extra.telemetry.
+ * Agent steps with telemetry.operation === "inference" are LLM calls.
+ */
+interface TranscriptStep {
+  step_id: number;
+  source: "system" | "agent" | "user";
+  message: string;
+  timestamp: string;
+  extra?: {
+    generation_model?: string | null;
+    telemetry?: {
+      source?: string;
+      operation?: string;
+    };
+  };
+}
+interface TranscriptMetrics {
+  total_prompt_tokens: number;
+  total_completion_tokens: number;
+  total_cached_tokens: number;
+  total_steps: number;
+}
+interface Transcript {
+  session_id: string;
+  schema_version: string;
+  agent: {
+    name: string;
+    version: string;
+    model_name: string;
+  };
+  final_metrics: TranscriptMetrics;
+  steps: TranscriptStep[];
+}
+interface TraceStep {
+  stepId: number;
+  source: "system" | "agent" | "user";
+  timestamp: string;
+  message: string;
+  isLlmCall: boolean;
+}
+interface TraceAgentData {
+  sessionId: string;
+  model: string;
+  finalMetrics: {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalCachedTokens: number;
+    totalSteps: number;
+  };
+  llmCallCount: number;
+  steps: TraceStep[];
+}
+interface TraceData {
+  taskId: string;
+  prompt: string;
+  agents: {
+    "raw-shell"?: TraceAgentData;
+    sverka?: TraceAgentData;
+  };
+}
+/** Default transcript directory. */
+export declare function transcriptDir(): string;
+/** Read a transcript JSON file by session ID. */
+export declare function readTranscript(sessionId: string, dir?: string): Transcript;
+/** Transform a raw transcript step to a trace step. */
+export declare function transformStep(step: TranscriptStep): TraceStep;
+/** Count LLM inference calls in a transcript. */
+export declare function countLlmCalls(transcript: Transcript): number;
+/** Transform a full transcript to agent trace data. */
+export declare function transformTranscript(transcript: Transcript, agentType: "raw-shell" | "sverka"): TraceAgentData;
+/** Build a combined trace data object for a task from two transcripts. */
+export declare function buildTraceData(taskId: string, prompt: string, rawShellSessionId: string, sverkaSessionId: string, dir?: string): TraceData;
+/** Write trace data to the benchmark traces directory. */
+export declare function writeTraceData(trace: TraceData, outputDir: string): Promise<string>;
+//#endregion
+export type { AgentConfig, AggregateMetrics, BenchmarkConfig, BenchmarkResult, RunMetrics, RunResult, Task, TraceAgentData, TraceData, TraceStep, Transcript, TranscriptMetrics, TranscriptStep };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"mappings":";;;UACiB;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA,SAAS;EACT;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA,SAAS;EACT;IACE,aAAa;IACb,QAAQ;;;;UAKK;EACf,OAAO;EACP,QAAQ;EACR;EACA;;;;;wBC1Cc,eACd;EAAgB;EAAoB,QAAQ;GAC5C,uBACA,oBACC;;wBAUa,iBACd,SAAS,aACT,oBACC;;wBA2BmB,aACpB,QAAQ,kBACP,QAAQ;;wBA6HW,YACpB,QAAQ,iBACR,qBACC;;;;qBChMU,0BAA0B;;qBAuC1B,yBAAyB;;;;;;;;;;;UC1BrB;EACf;EACA;EACA;EACA;EACA;IACE;IACA;MACE;MACA;;;;UAKW;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;;EAEF,eAAe;EACf,OAAO;;UAKQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;IACA;;EAEF;EACA,OAAO;;UAGQ;EACf;EACA;EACA;IACE,cAAc;IACd,SAAS;;;;wBAOG;;wBAKA,eACd,mBACA,eACC;;wBASa,cAAc,MAAM,iBAAiB;;wBAcrC,cAAc,YAAY;;wBAS1B,oBACd,YAAY,YACZ,oCACC;;wBAgBa,eACd,gBACA,gBACA,2BACA,yBACA,eACC;;wBAcmB,eACpB,OAAO,WACP,oBACC"}

package/dist/index.mjs

@@ -0,0 +1,2 @@
+import { a as transformStep, c as BENCHMARK_TASKS, d as extractMetrics, f as runBenchmark, i as transcriptDir, l as DEFAULT_AGENTS, n as countLlmCalls, o as transformTranscript, p as writeReport, r as readTranscript, s as writeTraceData, t as buildTraceData, u as aggregateMetrics } from "./src-PvGPQO8c.mjs";
+export { BENCHMARK_TASKS, DEFAULT_AGENTS, aggregateMetrics, buildTraceData, countLlmCalls, extractMetrics, readTranscript, runBenchmark, transcriptDir, transformStep, transformTranscript, writeReport, writeTraceData };

package/dist/src-PvGPQO8c.mjs

@@ -0,0 +1,279 @@
+import { spawn } from "node:child_process";
+import { cp, mkdir, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { extractUsageTokens, runAcpSession, sanitizeEnv } from "@sverka/arena";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+//#region src/arena.ts
+/** Extract RunMetrics from a PromptResponse + collected event data. */
+function extractMetrics(promptResult, toolCallCount, startTime) {
+	return {
+		...extractUsageTokens(promptResult.usage),
+		toolCallCount,
+		executionTimeMs: Date.now() - startTime,
+		stopReason: promptResult.stopReason
+	};
+}
+/** Aggregate metrics for one agent type across multiple RunResults. */
+function aggregateMetrics(results, agentType) {
+	const filtered = results.filter((r) => r.agentType === agentType);
+	const n = filtered.length;
+	if (n === 0) return {
+		totalTasks: 0,
+		successCount: 0,
+		avgInputTokens: 0,
+		avgOutputTokens: 0,
+		avgTotalTokens: 0,
+		avgToolCalls: 0,
+		avgExecutionTimeMs: 0
+	};
+	const sum = (f) => filtered.reduce((a, r) => a + f(r), 0);
+	return {
+		totalTasks: n,
+		successCount: filtered.filter((r) => r.success).length,
+		avgInputTokens: Math.round(sum((r) => r.metrics.inputTokens) / n),
+		avgOutputTokens: Math.round(sum((r) => r.metrics.outputTokens) / n),
+		avgTotalTokens: Math.round(sum((r) => r.metrics.totalTokens) / n),
+		avgToolCalls: Math.round(sum((r) => r.metrics.toolCallCount) / n * 100) / 100,
+		avgExecutionTimeMs: Math.round(sum((r) => r.metrics.executionTimeMs) / n)
+	};
+}
+/** Run the full benchmark: each task × each agent, sequentially. */
+async function runBenchmark(config) {
+	const model = config.model ?? "glm-5-2";
+	const results = [];
+	for (const task of config.tasks) for (const agent of config.agents) {
+		process.stderr.write(`[${task.id}/${agent.id}] Running...\n`);
+		const result = await runTask(task, agent, model);
+		results.push(result);
+		process.stderr.write(`[${task.id}/${agent.id}] ${result.success ? "PASS" : "FAIL"} — ${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ${result.metrics.executionTimeMs}ms\n`);
+	}
+	return {
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		model,
+		results,
+		summary: {
+			"raw-shell": aggregateMetrics(results, "raw-shell"),
+			sverka: aggregateMetrics(results, "sverka")
+		}
+	};
+}
+/** Run a single task with one agent. */
+async function runTask(task, agent, model) {
+	const workspace = await setupWorkspace(agent);
+	const startTime = Date.now();
+	let proc = null;
+	try {
+		proc = spawnAcpAgent(workspace, model);
+		let toolCallCount = 0;
+		const metrics = extractMetrics(await runAcpSession(proc, workspace, task.prompt, task.timeoutMs ?? 12e4, () => {
+			toolCallCount++;
+		}), toolCallCount, startTime);
+		return {
+			taskId: task.id,
+			agentId: agent.id,
+			agentType: agent.type,
+			metrics,
+			success: metrics.stopReason === "end_turn"
+		};
+	} catch (error) {
+		const metrics = extractMetrics({
+			stopReason: "cancelled",
+			usage: null
+		}, 0, startTime);
+		return {
+			taskId: task.id,
+			agentId: agent.id,
+			agentType: agent.type,
+			metrics,
+			success: false,
+			error: error instanceof Error ? error.message : String(error)
+		};
+	} finally {
+		proc?.kill("SIGTERM");
+	}
+}
+/** Set up workspace for an agent.
+*
+*  Both agents work in the actual Sverka repo so they have the real project
+*  to operate on. The only difference is that the "sverka" agent gets the
+*  Sverka skill installed in .agents/skills/sverka/.
+*
+*  We do NOT create a temp copy — copying node_modules + dist is too slow
+*  and the agents need the real build artifacts to run `sverka` commands.
+*/
+async function setupWorkspace(agent) {
+	const here = dirname(fileURLToPath(import.meta.url));
+	const workspace = resolve(here, "..", "..", "..");
+	if (agent.type === "sverka") {
+		const skillDir = join(workspace, ".agents", "skills", "sverka");
+		const skillFile = join(skillDir, "SKILL.md");
+		try {
+			await mkdir(skillDir, { recursive: true });
+			const repoSkillPath = resolveRepoFile(".agents/skills/sverka/SKILL.md");
+			if (repoSkillPath && resolve(repoSkillPath) !== resolve(skillFile)) await cp(repoSkillPath, skillFile, { recursive: true });
+		} catch {}
+	}
+	return workspace;
+}
+/** Spawn `devin acp` as a subprocess in the given workspace. */
+function spawnAcpAgent(workspace, model) {
+	const env = sanitizeEnv({
+		model: {
+			id: model,
+			name: model
+		},
+		workspace,
+		plugins: []
+	});
+	return spawn("devin", [
+		"acp",
+		"--model",
+		model
+	], {
+		cwd: workspace,
+		stdio: [
+			"pipe",
+			"pipe",
+			"inherit"
+		],
+		env
+	});
+}
+/** Write benchmark results to a JSON file. */
+async function writeReport(result, outputPath) {
+	const json = JSON.stringify(result, null, 2);
+	await writeFile(outputPath, json, "utf-8");
+}
+/** Resolve a file path relative to the repo root. */
+function resolveRepoFile(relPath) {
+	try {
+		const here = dirname(fileURLToPath(import.meta.url));
+		const repoRoot = resolve(here, "..", "..", "..");
+		return join(repoRoot, relPath);
+	} catch {
+		return null;
+	}
+}
+//#endregion
+//#region src/tasks.ts
+/** 5 task scenarios, ordered simple → complex. */
+const BENCHMARK_TASKS = [
+	{
+		id: "run-checks",
+		name: "run-checks",
+		prompt: "Run all quality checks (lint, typecheck, test, build) for this TypeScript project and report the results. If any check fails, report which one failed and why.",
+		timeoutMs: 12e4
+	},
+	{
+		id: "discover",
+		name: "discover",
+		prompt: "Discover what checks are available in this project by reading package.json scripts and looking for config files. List each check with its command.",
+		timeoutMs: 6e4
+	},
+	{
+		id: "create-config",
+		name: "create-config",
+		prompt: "Create a CI workflow configuration file for this project that runs lint, typecheck, test, and build in dependency order. The config should define steps with dependencies: lint and typecheck run first (parallel), test depends on both, build depends on test.",
+		timeoutMs: 12e4
+	},
+	{
+		id: "compile-github",
+		name: "compile-github",
+		prompt: "Compile a CI workflow to GitHub Actions YAML format for this project. The workflow should run lint, typecheck, test, and build on push events. Output the complete YAML.",
+		timeoutMs: 12e4
+	},
+	{
+		id: "multi-step",
+		name: "multi-step",
+		prompt: "Create a multi-step pipeline with dependencies: lint and typecheck run in parallel, test depends on both, build depends on test. Define the pipeline as code and validate it works correctly.",
+		timeoutMs: 12e4
+	}
+];
+/** Default agent configs: raw-shell (no sverka) vs sverka (CLI + skill). */
+const DEFAULT_AGENTS = [{
+	id: "raw-shell",
+	name: "Raw Shell Agent",
+	type: "raw-shell"
+}, {
+	id: "sverka",
+	name: "Sverka CLI Agent",
+	type: "sverka"
+}];
+//#endregion
+//#region src/collector.ts
+/**
+* Transcript collector — reads Devin CLI transcripts and transforms them
+* to trace data for the benchmark trace viewer.
+*
+* Transcript source: ~/.local/share/devin/cli/transcripts/<session-id>.json
+* Each transcript has steps[] with source, message, timestamp, and extra.telemetry.
+* Agent steps with telemetry.operation === "inference" are LLM calls.
+*/
+/** Default transcript directory. */
+function transcriptDir() {
+	return join(homedir(), ".local", "share", "devin", "cli", "transcripts");
+}
+/** Read a transcript JSON file by session ID. */
+function readTranscript(sessionId, dir) {
+	const path = join(dir ?? transcriptDir(), `${sessionId}.json`);
+	if (!existsSync(path)) throw new Error(`Transcript not found: ${path}`);
+	return JSON.parse(readFileSync(path, "utf-8"));
+}
+/** Transform a raw transcript step to a trace step. */
+function transformStep(step) {
+	const isLlmCall = step.source === "agent" && step.extra?.telemetry?.operation === "inference";
+	return {
+		stepId: step.step_id,
+		source: step.source,
+		timestamp: step.timestamp,
+		message: step.message,
+		isLlmCall
+	};
+}
+/** Count LLM inference calls in a transcript. */
+function countLlmCalls(transcript) {
+	return transcript.steps.filter((s) => s.source === "agent" && s.extra?.telemetry?.operation === "inference").length;
+}
+/** Transform a full transcript to agent trace data. */
+function transformTranscript(transcript, agentType) {
+	return {
+		sessionId: transcript.session_id,
+		model: transcript.agent.model_name,
+		finalMetrics: {
+			totalPromptTokens: transcript.final_metrics.total_prompt_tokens,
+			totalCompletionTokens: transcript.final_metrics.total_completion_tokens,
+			totalCachedTokens: transcript.final_metrics.total_cached_tokens,
+			totalSteps: transcript.final_metrics.total_steps
+		},
+		llmCallCount: countLlmCalls(transcript),
+		steps: transcript.steps.map(transformStep)
+	};
+}
+/** Build a combined trace data object for a task from two transcripts. */
+function buildTraceData(taskId, prompt, rawShellSessionId, sverkaSessionId, dir) {
+	const rawTranscript = readTranscript(rawShellSessionId, dir);
+	const sverkaTranscript = readTranscript(sverkaSessionId, dir);
+	return {
+		taskId,
+		prompt,
+		agents: {
+			"raw-shell": transformTranscript(rawTranscript, "raw-shell"),
+			sverka: transformTranscript(sverkaTranscript, "sverka")
+		}
+	};
+}
+/** Write trace data to the benchmark traces directory. */
+async function writeTraceData(trace, outputDir) {
+	if (trace.taskId.includes("..") || trace.taskId.includes("/") || trace.taskId.includes("\\")) throw new Error(`Invalid taskId for trace output: ${trace.taskId}`);
+	const taskDir = join(outputDir, trace.taskId);
+	await mkdir(taskDir, { recursive: true });
+	const path = join(taskDir, "trace.json");
+	await writeFile(path, JSON.stringify(trace, null, 2), "utf-8");
+	return path;
+}
+//#endregion
+export { transformStep as a, BENCHMARK_TASKS as c, extractMetrics as d, runBenchmark as f, transcriptDir as i, DEFAULT_AGENTS as l, countLlmCalls as n, transformTranscript as o, writeReport as p, readTranscript as r, writeTraceData as s, buildTraceData as t, aggregateMetrics as u };
+
+//# sourceMappingURL=src-PvGPQO8c.mjs.map

package/dist/src-PvGPQO8c.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"src-PvGPQO8c.mjs","names":[],"sources":["../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"sourcesContent":["import { spawn, type ChildProcess } from \"node:child_process\";\nimport { cp, mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { dirname, resolve } from \"node:path\";\n\nimport type { Usage } from \"@agentclientprotocol/sdk\";\n\nimport { sanitizeEnv, runAcpSession, extractUsageTokens } from \"@sverka/arena\";\n\nimport type {\n  Task,\n  AgentConfig,\n  RunMetrics,\n  RunResult,\n  AggregateMetrics,\n  BenchmarkResult,\n  BenchmarkConfig,\n} from \"./types.js\";\n\n/** Extract RunMetrics from a PromptResponse + collected event data. */\nexport function extractMetrics(\n  promptResult: { stopReason: string; usage?: Usage | null },\n  toolCallCount: number,\n  startTime: number,\n): RunMetrics {\n  return {\n    ...extractUsageTokens(promptResult.usage),\n    toolCallCount,\n    executionTimeMs: Date.now() - startTime,\n    stopReason: promptResult.stopReason,\n  };\n}\n\n/** Aggregate metrics for one agent type across multiple RunResults. */\nexport function aggregateMetrics(\n  results: RunResult[],\n  agentType: string,\n): AggregateMetrics {\n  const filtered = results.filter((r) => r.agentType === agentType);\n  const n = filtered.length;\n  if (n === 0) {\n    return {\n      totalTasks: 0,\n      successCount: 0,\n      avgInputTokens: 0,\n      avgOutputTokens: 0,\n      avgTotalTokens: 0,\n      avgToolCalls: 0,\n      avgExecutionTimeMs: 0,\n    };\n  }\n  const sum = (f: (r: RunResult) => number) => filtered.reduce((a, r) => a + f(r), 0);\n  return {\n    totalTasks: n,\n    successCount: filtered.filter((r) => r.success).length,\n    avgInputTokens: Math.round(sum((r) => r.metrics.inputTokens) / n),\n    avgOutputTokens: Math.round(sum((r) => r.metrics.outputTokens) / n),\n    avgTotalTokens: Math.round(sum((r) => r.metrics.totalTokens) / n),\n    avgToolCalls: Math.round((sum((r) => r.metrics.toolCallCount) / n) * 100) / 100,\n    avgExecutionTimeMs: Math.round(sum((r) => r.metrics.executionTimeMs) / n),\n  };\n}\n\n/** Run the full benchmark: each task × each agent, sequentially. */\nexport async function runBenchmark(\n  config: BenchmarkConfig,\n): Promise<BenchmarkResult> {\n  const model = config.model ?? \"glm-5-2\";\n  const results: RunResult[] = [];\n\n  for (const task of config.tasks) {\n    for (const agent of config.agents) {\n      process.stderr.write(`[${task.id}/${agent.id}] Running...\\n`);\n      const result = await runTask(task, agent, model);\n      results.push(result);\n      process.stderr.write(\n        `[${task.id}/${agent.id}] ${result.success ? \"PASS\" : \"FAIL\"} — ` +\n          `${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ` +\n          `${result.metrics.executionTimeMs}ms\\n`,\n      );\n    }\n  }\n\n  return {\n    timestamp: new Date().toISOString(),\n    model,\n    results,\n    summary: {\n      \"raw-shell\": aggregateMetrics(results, \"raw-shell\"),\n      sverka: aggregateMetrics(results, \"sverka\"),\n    },\n  };\n}\n\n/** Run a single task with one agent. */\nasync function runTask(\n  task: Task,\n  agent: AgentConfig,\n  model: string,\n): Promise<RunResult> {\n  const workspace = await setupWorkspace(agent);\n  const startTime = Date.now();\n  let proc: ChildProcess | null = null;\n\n  try {\n    proc = spawnAcpAgent(workspace, model);\n    let toolCallCount = 0;\n    const promptResult = await runAcpSession(\n      proc,\n      workspace,\n      task.prompt,\n      task.timeoutMs ?? 120000,\n      () => {\n        toolCallCount++;\n      },\n    );\n    const metrics = extractMetrics(promptResult, toolCallCount, startTime);\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: metrics.stopReason === \"end_turn\",\n    };\n  } catch (error) {\n    const metrics = extractMetrics(\n      { stopReason: \"cancelled\", usage: null },\n      0,\n      startTime,\n    );\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: false,\n      error: error instanceof Error ? error.message : String(error),\n    };\n  } finally {\n    proc?.kill(\"SIGTERM\");\n  }\n}\n\n/** Set up workspace for an agent.\n *\n *  Both agents work in the actual Sverka repo so they have the real project\n *  to operate on. The only difference is that the \"sverka\" agent gets the\n *  Sverka skill installed in .agents/skills/sverka/.\n *\n *  We do NOT create a temp copy — copying node_modules + dist is too slow\n *  and the agents need the real build artifacts to run `sverka` commands.\n */\nasync function setupWorkspace(agent: AgentConfig): Promise<string> {\n  const here = dirname(fileURLToPath(import.meta.url));\n  const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n  const workspace = repoRoot;\n\n  if (agent.type === \"sverka\") {\n    // Skill is already in the repo at .agents/skills/sverka/SKILL.md\n    // No need to copy — just ensure the directory exists\n    const skillDir = join(workspace, \".agents\", \"skills\", \"sverka\");\n    const skillFile = join(skillDir, \"SKILL.md\");\n    try {\n      await mkdir(skillDir, { recursive: true });\n      const repoSkillPath = resolveRepoFile(\".agents/skills/sverka/SKILL.md\");\n      if (repoSkillPath && resolve(repoSkillPath) !== resolve(skillFile)) {\n        await cp(repoSkillPath, skillFile, { recursive: true });\n      }\n    } catch {\n      // Skill already exists — that's fine\n    }\n  }\n\n  return workspace;\n}\n\n/** Spawn `devin acp` as a subprocess in the given workspace. */\nfunction spawnAcpAgent(workspace: string, model: string): ChildProcess {\n  const env = sanitizeEnv({\n    model: { id: model, name: model },\n    workspace,\n    plugins: [],\n  });\n  return spawn(\"devin\", [\"acp\", \"--model\", model], { // NOSONAR — PATH needed for devin binary\n    cwd: workspace,\n    stdio: [\"pipe\", \"pipe\", \"inherit\"],\n    env,\n  });\n}\n\n/** Write benchmark results to a JSON file. */\nexport async function writeReport(\n  result: BenchmarkResult,\n  outputPath: string,\n): Promise<void> {\n  const json = JSON.stringify(result, null, 2);\n  await writeFile(outputPath, json, \"utf-8\");\n}\n\n/** Resolve a file path relative to the repo root. */\nfunction resolveRepoFile(relPath: string): string | null {\n  try {\n    const here = dirname(fileURLToPath(import.meta.url));\n    // From packages/benchmark/src/ or packages/benchmark/dist/ go up to repo root\n    const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n    return join(repoRoot, relPath);\n  } catch {\n    return null;\n  }\n}\n","import type { Task, AgentConfig } from \"./types.js\";\n\n/** 5 task scenarios, ordered simple → complex. */\nexport const BENCHMARK_TASKS: readonly Task[] = [\n  {\n    id: \"run-checks\",\n    name: \"run-checks\",\n    prompt:\n      \"Run all quality checks (lint, typecheck, test, build) for this TypeScript project and report the results. If any check fails, report which one failed and why.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"discover\",\n    name: \"discover\",\n    prompt:\n      \"Discover what checks are available in this project by reading package.json scripts and looking for config files. List each check with its command.\",\n    timeoutMs: 60000,\n  },\n  {\n    id: \"create-config\",\n    name: \"create-config\",\n    prompt:\n      \"Create a CI workflow configuration file for this project that runs lint, typecheck, test, and build in dependency order. The config should define steps with dependencies: lint and typecheck run first (parallel), test depends on both, build depends on test.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"compile-github\",\n    name: \"compile-github\",\n    prompt:\n      \"Compile a CI workflow to GitHub Actions YAML format for this project. The workflow should run lint, typecheck, test, and build on push events. Output the complete YAML.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"multi-step\",\n    name: \"multi-step\",\n    prompt:\n      \"Create a multi-step pipeline with dependencies: lint and typecheck run in parallel, test depends on both, build depends on test. Define the pipeline as code and validate it works correctly.\",\n    timeoutMs: 120000,\n  },\n] as const;\n\n/** Default agent configs: raw-shell (no sverka) vs sverka (CLI + skill). */\nexport const DEFAULT_AGENTS: readonly AgentConfig[] = [\n  {\n    id: \"raw-shell\",\n    name: \"Raw Shell Agent\",\n    type: \"raw-shell\",\n  },\n  {\n    id: \"sverka\",\n    name: \"Sverka CLI Agent\",\n    type: \"sverka\",\n  },\n] as const;\n","/**\n * Transcript collector — reads Devin CLI transcripts and transforms them\n * to trace data for the benchmark trace viewer.\n *\n * Transcript source: ~/.local/share/devin/cli/transcripts/<session-id>.json\n * Each transcript has steps[] with source, message, timestamp, and extra.telemetry.\n * Agent steps with telemetry.operation === \"inference\" are LLM calls.\n */\n\nimport { readFileSync, existsSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport { homedir } from \"node:os\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\n\n// ─── Transcript source types (raw JSON from Devin CLI) ───────────────\n\nexport interface TranscriptStep {\n  step_id: number;\n  source: \"system\" | \"agent\" | \"user\";\n  message: string;\n  timestamp: string;\n  extra?: {\n    generation_model?: string | null;\n    telemetry?: {\n      source?: string;\n      operation?: string;\n    };\n  };\n}\n\nexport interface TranscriptMetrics {\n  total_prompt_tokens: number;\n  total_completion_tokens: number;\n  total_cached_tokens: number;\n  total_steps: number;\n}\n\nexport interface Transcript {\n  session_id: string;\n  schema_version: string;\n  agent: {\n    name: string;\n    version: string;\n    model_name: string;\n  };\n  final_metrics: TranscriptMetrics;\n  steps: TranscriptStep[];\n}\n\n// ─── Trace data types (transformed for viewer) ───────────────────────\n\nexport interface TraceStep {\n  stepId: number;\n  source: \"system\" | \"agent\" | \"user\";\n  timestamp: string;\n  message: string;\n  isLlmCall: boolean;\n}\n\nexport interface TraceAgentData {\n  sessionId: string;\n  model: string;\n  finalMetrics: {\n    totalPromptTokens: number;\n    totalCompletionTokens: number;\n    totalCachedTokens: number;\n    totalSteps: number;\n  };\n  llmCallCount: number;\n  steps: TraceStep[];\n}\n\nexport interface TraceData {\n  taskId: string;\n  prompt: string;\n  agents: {\n    \"raw-shell\"?: TraceAgentData;\n    sverka?: TraceAgentData;\n  };\n}\n\n// ─── Collector functions ─────────────────────────────────────────────\n\n/** Default transcript directory. */\nexport function transcriptDir(): string {\n  return join(homedir(), \".local\", \"share\", \"devin\", \"cli\", \"transcripts\");\n}\n\n/** Read a transcript JSON file by session ID. */\nexport function readTranscript(\n  sessionId: string,\n  dir?: string,\n): Transcript {\n  const path = join(dir ?? transcriptDir(), `${sessionId}.json`);\n  if (!existsSync(path)) {\n    throw new Error(`Transcript not found: ${path}`);\n  }\n  return JSON.parse(readFileSync(path, \"utf-8\")) as Transcript;\n}\n\n/** Transform a raw transcript step to a trace step. */\nexport function transformStep(step: TranscriptStep): TraceStep {\n  const isLlmCall =\n    step.source === \"agent\" &&\n    step.extra?.telemetry?.operation === \"inference\";\n  return {\n    stepId: step.step_id,\n    source: step.source,\n    timestamp: step.timestamp,\n    message: step.message,\n    isLlmCall,\n  };\n}\n\n/** Count LLM inference calls in a transcript. */\nexport function countLlmCalls(transcript: Transcript): number {\n  return transcript.steps.filter(\n    (s) =>\n      s.source === \"agent\" &&\n      s.extra?.telemetry?.operation === \"inference\",\n  ).length;\n}\n\n/** Transform a full transcript to agent trace data. */\nexport function transformTranscript(\n  transcript: Transcript,\n  agentType: \"raw-shell\" | \"sverka\",\n): TraceAgentData {\n  return {\n    sessionId: transcript.session_id,\n    model: transcript.agent.model_name,\n    finalMetrics: {\n      totalPromptTokens: transcript.final_metrics.total_prompt_tokens,\n      totalCompletionTokens: transcript.final_metrics.total_completion_tokens,\n      totalCachedTokens: transcript.final_metrics.total_cached_tokens,\n      totalSteps: transcript.final_metrics.total_steps,\n    },\n    llmCallCount: countLlmCalls(transcript),\n    steps: transcript.steps.map(transformStep),\n  };\n}\n\n/** Build a combined trace data object for a task from two transcripts. */\nexport function buildTraceData(\n  taskId: string,\n  prompt: string,\n  rawShellSessionId: string,\n  sverkaSessionId: string,\n  dir?: string,\n): TraceData {\n  const rawTranscript = readTranscript(rawShellSessionId, dir);\n  const sverkaTranscript = readTranscript(sverkaSessionId, dir);\n  return {\n    taskId,\n    prompt,\n    agents: {\n      \"raw-shell\": transformTranscript(rawTranscript, \"raw-shell\"),\n      sverka: transformTranscript(sverkaTranscript, \"sverka\"),\n    },\n  };\n}\n\n/** Write trace data to the benchmark traces directory. */\nexport async function writeTraceData(\n  trace: TraceData,\n  outputDir: string,\n): Promise<string> {\n  // Reject path traversal in taskId to prevent writing outside outputDir\n  if (trace.taskId.includes(\"..\") || trace.taskId.includes(\"/\") || trace.taskId.includes(\"\\\\\")) {\n    throw new Error(`Invalid taskId for trace output: ${trace.taskId}`);\n  }\n  const taskDir = join(outputDir, trace.taskId);\n  await mkdir(taskDir, { recursive: true });\n  const path = join(taskDir, \"trace.json\");\n  await writeFile(path, JSON.stringify(trace, null, 2), \"utf-8\");\n  return path;\n}\n"],"mappings":";;;;;;;;;AAqBA,SAAgB,eACd,cACA,eACA,WACY;CACZ,OAAO;EACL,GAAG,mBAAmB,aAAa,KAAK;EACxC;EACA,iBAAiB,KAAK,IAAI,IAAI;EAC9B,YAAY,aAAa;CAC3B;AACF;;AAGA,SAAgB,iBACd,SACA,WACkB;CAClB,MAAM,WAAW,QAAQ,QAAQ,MAAM,EAAE,cAAc,SAAS;CAChE,MAAM,IAAI,SAAS;CACnB,IAAI,MAAM,GACR,OAAO;EACL,YAAY;EACZ,cAAc;EACd,gBAAgB;EAChB,iBAAiB;EACjB,gBAAgB;EAChB,cAAc;EACd,oBAAoB;CACtB;CAEF,MAAM,OAAO,MAAgC,SAAS,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC;CAClF,OAAO;EACL,YAAY;EACZ,cAAc,SAAS,QAAQ,MAAM,EAAE,OAAO,CAAC,CAAC;EAChD,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,iBAAiB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,YAAY,IAAI,CAAC;EAClE,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,cAAc,KAAK,MAAO,KAAK,MAAM,EAAE,QAAQ,aAAa,IAAI,IAAK,GAAG,IAAI;EAC5E,oBAAoB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,eAAe,IAAI,CAAC;CAC1E;AACF;;AAGA,eAAsB,aACpB,QAC0B;CAC1B,MAAM,QAAQ,OAAO,SAAS;CAC9B,MAAM,UAAuB,CAAC;CAE9B,KAAK,MAAM,QAAQ,OAAO,OACxB,KAAK,MAAM,SAAS,OAAO,QAAQ;EACjC,QAAQ,OAAO,MAAM,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,eAAe;EAC5D,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO,KAAK;EAC/C,QAAQ,KAAK,MAAM;EACnB,QAAQ,OAAO,MACb,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,IAAI,OAAO,UAAU,SAAS,OAAO,KACxD,OAAO,QAAQ,YAAY,WAAW,OAAO,QAAQ,cAAc,UACnE,OAAO,QAAQ,gBAAgB,KACtC;CACF;CAGF,OAAO;EACL,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;EAClC;EACA;EACA,SAAS;GACP,aAAa,iBAAiB,SAAS,WAAW;GAClD,QAAQ,iBAAiB,SAAS,QAAQ;EAC5C;CACF;AACF;;AAGA,eAAe,QACb,MACA,OACA,OACoB;CACpB,MAAM,YAAY,MAAM,eAAe,KAAK;CAC5C,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI,OAA4B;CAEhC,IAAI;EACF,OAAO,cAAc,WAAW,KAAK;EACrC,IAAI,gBAAgB;EAUpB,MAAM,UAAU,eAAe,MATJ,cACzB,MACA,WACA,KAAK,QACL,KAAK,aAAa,YACZ;GACJ;EACF,CACF,GAC6C,eAAe,SAAS;EACrE,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS,QAAQ,eAAe;EAClC;CACF,SAAS,OAAO;EACd,MAAM,UAAU,eACd;GAAE,YAAY;GAAa,OAAO;EAAK,GACvC,GACA,SACF;EACA,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS;GACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9D;CACF,UAAU;EACR,MAAM,KAAK,SAAS;CACtB;AACF;;;;;;;;;;AAWA,eAAe,eAAe,OAAqC;CACjE,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;CAEnD,MAAM,YADW,QAAQ,MAAM,MAAM,MAAM,IAClB;CAEzB,IAAI,MAAM,SAAS,UAAU;EAG3B,MAAM,WAAW,KAAK,WAAW,WAAW,UAAU,QAAQ;EAC9D,MAAM,YAAY,KAAK,UAAU,UAAU;EAC3C,IAAI;GACF,MAAM,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;GACzC,MAAM,gBAAgB,gBAAgB,gCAAgC;GACtE,IAAI,iBAAiB,QAAQ,aAAa,MAAM,QAAQ,SAAS,GAC/D,MAAM,GAAG,eAAe,WAAW,EAAE,WAAW,KAAK,CAAC;EAE1D,QAAQ,CAER;CACF;CAEA,OAAO;AACT;;AAGA,SAAS,cAAc,WAAmB,OAA6B;CACrE,MAAM,MAAM,YAAY;EACtB,OAAO;GAAE,IAAI;GAAO,MAAM;EAAM;EAChC;EACA,SAAS,CAAC;CACZ,CAAC;CACD,OAAO,MAAM,SAAS;EAAC;EAAO;EAAW;CAAK,GAAG;EAC/C,KAAK;EACL,OAAO;GAAC;GAAQ;GAAQ;EAAS;EACjC;CACF,CAAC;AACH;;AAGA,eAAsB,YACpB,QACA,YACe;CACf,MAAM,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;CAC3C,MAAM,UAAU,YAAY,MAAM,OAAO;AAC3C;;AAGA,SAAS,gBAAgB,SAAgC;CACvD,IAAI;EACF,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;EAEnD,MAAM,WAAW,QAAQ,MAAM,MAAM,MAAM,IAAI;EAC/C,OAAO,KAAK,UAAU,OAAO;CAC/B,QAAQ;EACN,OAAO;CACT;AACF;;;;AC/MA,MAAa,kBAAmC;CAC9C;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;AACF;;AAGA,MAAa,iBAAyC,CACpD;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,GACA;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,CACF;;;;;;;;;;;;AC+BA,SAAgB,gBAAwB;CACtC,OAAO,KAAK,QAAQ,GAAG,UAAU,SAAS,SAAS,OAAO,aAAa;AACzE;;AAGA,SAAgB,eACd,WACA,KACY;CACZ,MAAM,OAAO,KAAK,OAAO,cAAc,GAAG,GAAG,UAAU,MAAM;CAC7D,IAAI,CAAC,WAAW,IAAI,GAClB,MAAM,IAAI,MAAM,yBAAyB,MAAM;CAEjD,OAAO,KAAK,MAAM,aAAa,MAAM,OAAO,CAAC;AAC/C;;AAGA,SAAgB,cAAc,MAAiC;CAC7D,MAAM,YACJ,KAAK,WAAW,WAChB,KAAK,OAAO,WAAW,cAAc;CACvC,OAAO;EACL,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,WAAW,KAAK;EAChB,SAAS,KAAK;EACd;CACF;AACF;;AAGA,SAAgB,cAAc,YAAgC;CAC5D,OAAO,WAAW,MAAM,QACrB,MACC,EAAE,WAAW,WACb,EAAE,OAAO,WAAW,cAAc,WACtC,CAAC,CAAC;AACJ;;AAGA,SAAgB,oBACd,YACA,WACgB;CAChB,OAAO;EACL,WAAW,WAAW;EACtB,OAAO,WAAW,MAAM;EACxB,cAAc;GACZ,mBAAmB,WAAW,cAAc;GAC5C,uBAAuB,WAAW,cAAc;GAChD,mBAAmB,WAAW,cAAc;GAC5C,YAAY,WAAW,cAAc;EACvC;EACA,cAAc,cAAc,UAAU;EACtC,OAAO,WAAW,MAAM,IAAI,aAAa;CAC3C;AACF;;AAGA,SAAgB,eACd,QACA,QACA,mBACA,iBACA,KACW;CACX,MAAM,gBAAgB,eAAe,mBAAmB,GAAG;CAC3D,MAAM,mBAAmB,eAAe,iBAAiB,GAAG;CAC5D,OAAO;EACL;EACA;EACA,QAAQ;GACN,aAAa,oBAAoB,eAAe,WAAW;GAC3D,QAAQ,oBAAoB,kBAAkB,QAAQ;EACxD;CACF;AACF;;AAGA,eAAsB,eACpB,OACA,WACiB;CAEjB,IAAI,MAAM,OAAO,SAAS,IAAI,KAAK,MAAM,OAAO,SAAS,GAAG,KAAK,MAAM,OAAO,SAAS,IAAI,GACzF,MAAM,IAAI,MAAM,oCAAoC,MAAM,QAAQ;CAEpE,MAAM,UAAU,KAAK,WAAW,MAAM,MAAM;CAC5C,MAAM,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;CACxC,MAAM,OAAO,KAAK,SAAS,YAAY;CACvC,MAAM,UAAU,MAAM,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,OAAO;CAC7D,OAAO;AACT"}

package/package.json

@@ -1,20 +1,52 @@
 {
-  "bugs": {
-    "url": "https://github.com/sverka-dev/sverka/issues"
-  },
-  "description": "Placeholder for @sverka/benchmark published by @nx-devkit/prepare-for-release.",
-  "homepage": "https://sverka.dev",
-  "license": "MIT",
   "name": "@sverka/benchmark",
-  "publishConfig": {
-    "access": "public",
-    "registry": "https://registry.npmjs.org/"
-  },
+  "version": "0.1.2",
+  "description": "Sverka benchmark arena — two-agent comparison harness",
+  "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/sverka-dev/sverka",
     "directory": "packages/benchmark"
   },
+  "homepage": "https://sverka.dev",
+  "bugs": {
+    "url": "https://github.com/sverka-dev/sverka/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
   "type": "module",
-  "version": "0.0.0"
-}
+  "engines": {
+    "node": ">=22.5.0"
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "bin": {
+    "sverka-benchmark": "./dist/bin.mjs"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest run --passWithNoTests",
+    "lint": "eslint src",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@agentclientprotocol/sdk": "^1.4.0",
+    "@sverka/arena": "^0.1.2"
+  },
+  "devDependencies": {
+    "tsdown": "^0.23.0",
+    "typescript": "^5.8.0",
+    "vitest": "^5.0.0"
+  }
+}