@sverka/arena 0.0.0 → 0.1.1

package/dist/bin.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/bin.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import process from "node:process";
+//#region src/bin.ts
+async function main() {
+	process.stderr.write("arena CLI — not yet implemented\n");
+	return 1;
+}
+const code = await main();
+process.exit(code);
+//#endregion
+export {};
+
+//# sourceMappingURL=bin.mjs.map

package/dist/bin.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\nimport process from \"node:process\";\n// Stub — will be implemented in next phase\nasync function main(): Promise<number> {\n  process.stderr.write(\"arena CLI — not yet implemented\\n\");\n  return 1;\n}\n\nconst code = await main();\nprocess.exit(code);\n"],"mappings":";;;AAGA,eAAe,OAAwB;CACrC,QAAQ,OAAO,MAAM,mCAAmC;CACxD,OAAO;AACT;AAEA,MAAM,OAAO,MAAM,KAAK;AACxB,QAAQ,KAAK,IAAI"}

package/dist/index.d.mts

@@ -0,0 +1,426 @@
+import { ChildProcess } from "node:child_process";
+import * as acp from "@agentclientprotocol/sdk";
+import { PromptResponse, SessionUpdate, Usage } from "@agentclientprotocol/sdk";
+//#region src/types.d.ts
+/** Agent adapter — pluggable interface for different agent CLIs */
+interface AgentAdapter {
+  id: string;
+  spawn(config: AgentSpawnConfig): AgentProcess;
+}
+interface AgentSpawnConfig {
+  model: ModelConfig;
+  workspace: string;
+  plugins: PluginConfig[];
+  permissionMode?: "auto" | "dangerous" | "accept-edits";
+  env?: Record<string, string>;
+}
+interface AgentProcess {
+  run(prompt: string, timeoutMs: number): Promise<RunResult>;
+  kill(): void;
+}
+/** Model configuration */
+interface ModelConfig {
+  id: string;
+  name: string;
+  envVar?: string;
+}
+/** Plugin/skill configuration — the variable we're testing */
+interface PluginConfig {
+  id: string;
+  name: string;
+  path: string;
+  enabled: boolean;
+  installPath?: string;
+}
+/** Task to benchmark */
+interface Task {
+  id: string;
+  name: string;
+  prompt: string;
+  timeoutMs?: number;
+  /** Natural-language criteria the judge uses to evaluate the output */
+  successCriteria?: string;
+  /** Optional expected output for deterministic comparison */
+  expectedOutput?: string;
+  /** Fixture project to copy into the workspace for this task.
+   * Path relative to the arena package root (e.g. "fixtures/ts-fix-test").
+   * If omitted, the runner uses an empty temp directory. */
+  fixture?: string;
+  /** Optional deterministic checks to run after the agent finishes.
+   * Each check is a shell command; exit 0 = pass, non-zero = fail. */
+  checks?: DeterministicCheck[];
+}
+/** A deterministic check — shell command that exits 0 on success. */
+interface DeterministicCheck {
+  id: string;
+  command: string;
+  description: string;
+}
+/** Full arena configuration — defines the matrix */
+interface ArenaConfig {
+  tasks: Task[];
+  agent: AgentAdapter;
+  models: ModelConfig[];
+  plugins: PluginConfig[];
+  workspace?: string;
+  repetitions?: number;
+  outputDir: string;
+  /** Judge configuration for blind evaluation (optional) */
+  judge?: JudgeConfig;
+}
+/** Judge configuration — blind LLM evaluation of run outputs */
+interface JudgeConfig {
+  /** Model to use for judging (can differ from agent models) */
+  model: ModelConfig;
+  /** Agent adapter for the judge (can differ from the agent under test) */
+  agent: AgentAdapter;
+  /** Number of judge repetitions per run (for confidence intervals) */
+  repetitions?: number;
+  /** Whether to show the judge which plugins were active (default: false = blind) */
+  revealPlugins?: boolean;
+  /** Custom system prompt for the judge */
+  systemPrompt?: string;
+}
+/** Judge evaluation of a single run */
+interface JudgeVerdict {
+  /** Run being judged (index into ArenaResult.results) */
+  runIndex: number;
+  /** Score 0-100 */
+  score: number;
+  /** Whether the judge considers this a pass (score >= threshold) */
+  passed: boolean;
+  /** Judge's reasoning */
+  reasoning: string;
+  /** Specific issues found */
+  issues: string[];
+  /** Which run this verdict applies to (for pairing) */
+  taskId: string;
+  modelId: string;
+  pluginIds: string[];
+}
+/** Result of a single matrix cell run */
+interface RunResult {
+  taskId: string;
+  modelId: string;
+  pluginIds: string[];
+  metrics: RunMetrics;
+  trace: TraceData;
+  /** The agent's final output (text response to the prompt) */
+  output: string;
+  success: boolean;
+  /** Results of deterministic checks (empty if none configured) */
+  checkResults: CheckResult[];
+  error?: string;
+  /** Judge verdicts for this run (empty if no judge configured) */
+  verdicts: JudgeVerdict[];
+}
+/** Result of a single deterministic check */
+interface CheckResult {
+  checkId: string;
+  passed: boolean;
+  output: string;
+  exitCode: number;
+}
+/** Metrics collected per run */
+interface RunMetrics {
+  inputTokens: number;
+  outputTokens: number;
+  thoughtTokens: number;
+  totalTokens: number;
+  toolCallCount: number;
+  llmCallCount: number;
+  executionTimeMs: number;
+  stopReason: string;
+}
+/** Full trace data for visualization */
+interface TraceData {
+  sessionId: string;
+  model: string;
+  steps: TraceStep[];
+  finalMetrics: {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalCachedTokens: number;
+    totalSteps: number;
+  };
+}
+/** A single step in the trace */
+interface TraceStep {
+  stepId: number;
+  timestamp: string;
+  source: "system" | "agent" | "user";
+  message: string;
+  toolCalls?: ToolCall[];
+  observations?: Observation[];
+  isLlmCall: boolean;
+}
+/** A tool call within a step */
+interface ToolCall {
+  functionName: string;
+  arguments: Record<string, unknown>;
+  toolCallId: string;
+}
+/** Observation result from a tool call */
+interface Observation {
+  sourceCallId: string;
+  content: string;
+}
+/** Aggregate metrics for a group of runs */
+interface AggregateMetrics {
+  totalRuns: number;
+  successCount: number;
+  avgInputTokens: number;
+  avgOutputTokens: number;
+  avgTotalTokens: number;
+  avgToolCalls: number;
+  avgLlmCalls: number;
+  avgExecutionTimeMs: number;
+  /** Average judge score (0-100) across judged runs */
+  avgJudgeScore: number;
+  /** Number of runs that passed judge evaluation */
+  judgePassCount: number;
+  /** Label for this aggregate group (e.g. "glm-5-2/sverka=on") */
+  label: string;
+}
+/** Full arena result — all cells in the matrix */
+interface ArenaResult {
+  timestamp: string;
+  config: {
+    models: string[];
+    plugins: string[];
+    tasks: string[];
+    repetitions: number;
+    judgeModel?: string;
+  };
+  results: RunResult[];
+  aggregates: AggregateMetrics[];
+  /** Per-task analysis comparing plugin on vs off */
+  analysis: CaseAnalysis[];
+}
+/** Analysis of a single case (task) across plugin combinations */
+interface CaseAnalysis {
+  taskId: string;
+  taskName: string;
+  prompt: string;
+  /** Comparison between plugin combos for this task */
+  comparisons: ComboComparison[];
+  /** AI-generated summary of findings for this case */
+  summary?: string;
+}
+/** Comparison between two plugin combos for a task */
+interface ComboComparison {
+  /** Plugin combo label (e.g. "no-plugins" vs "sverka") */
+  baseline: string;
+  candidate: string;
+  /** Metric deltas (candidate - baseline) */
+  deltaTokens: number;
+  deltaToolCalls: number;
+  deltaLlmCalls: number;
+  deltaTimeMs: number;
+  deltaJudgeScore: number;
+  /** Whether the candidate is better than baseline */
+  candidateBetter: boolean;
+}
+//#endregion
+//#region src/adapters/devin.d.ts
+/** A single step in the raw Devin CLI transcript. */
+interface TranscriptStep {
+  step_id: number;
+  source: "system" | "agent" | "user";
+  message: string;
+  timestamp: string;
+  extra?: {
+    generation_model?: string | null;
+    telemetry?: {
+      source?: string;
+      operation?: string;
+    } | null;
+  } | null;
+}
+/** Final metrics block in a Devin CLI transcript. */
+interface TranscriptMetrics {
+  total_prompt_tokens: number;
+  total_completion_tokens: number;
+  total_cached_tokens: number;
+  total_steps: number;
+}
+/** Full raw transcript written by the Devin CLI. */
+interface Transcript {
+  session_id: string;
+  schema_version: string;
+  agent: {
+    name: string;
+    version: string;
+    model_name: string;
+  };
+  final_metrics: TranscriptMetrics;
+  steps: TranscriptStep[];
+}
+/** Default transcript directory: ~/.local/share/devin/cli/transcripts */
+export declare function transcriptDir(): string;
+/** Count LLM inference calls in a transcript (agent steps with telemetry.operation === "inference"). */
+export declare function countLlmCalls(transcript: Transcript): number;
+/** Extract token counts from a Usage object, defaulting to 0. */
+export declare function extractUsageTokens(usage: Usage | null | undefined): {
+  inputTokens: number;
+  outputTokens: number;
+  thoughtTokens: number;
+  totalTokens: number;
+};
+/**
+ * Install enabled plugins and remove disabled ones from the workspace.
+ *
+ * CRITICAL: This function first removes the entire `.agents/skills/`
+ * directory to ensure no inherited skills from the host contaminate the
+ * benchmark. Only explicitly enabled plugins are installed.
+ *
+ * Each enabled plugin's `path` is copied to `workspace/.agents/skills/<id>/`.
+ */
+export declare function installPlugins(workspace: string, plugins: PluginConfig[]): Promise<void>;
+/**
+ * Devin ACP adapter — spawns `devin acp` as a subprocess.
+ *
+ * Usage:
+ * ```ts
+ * const adapter = new DevinAdapter();
+ * const proc = adapter.spawn({ model, workspace, plugins });
+ * const result = await proc.run("Fix the bug", 120_000);
+ * proc.kill();
+ * ```
+ */
+export declare class DevinAdapter implements AgentAdapter {
+  readonly id = "devin";
+  spawn(config: AgentSpawnConfig): AgentProcess;
+}
+//#endregion
+//#region src/acp.d.ts
+/** A `session/update` notification whose `sessionUpdate` is `"tool_call"`. */
+type ToolCallMessage = Extract<SessionUpdate, {
+  sessionUpdate: "tool_call";
+}>;
+/** A `session/update` notification whose `sessionUpdate` is `"tool_call_update"`. */
+type ToolCallUpdateMessage = Extract<SessionUpdate, {
+  sessionUpdate: "tool_call_update";
+}>;
+/**
+ * Build a sanitized environment for an agent subprocess.
+ *
+ * Strips out host contamination: Gas City, Beads, MCP servers, project
+ * skills, and other variables that would leak the host's configuration
+ * into the benchmark. The agent must only see what we explicitly install.
+ *
+ * Sets `DEVIN_PERMISSION_MODE` and `DEVIN_MODEL` from the spawn config,
+ * then merges any explicit `config.env` overrides (which win).
+ */
+export declare function sanitizeEnv(config: AgentSpawnConfig): Record<string, string>;
+/**
+ * Create a permission-request handler that auto-approves tool calls.
+ *
+ * Selects the first `allow_once` or `allow_always` option, falling back to
+ * the first available option. This is the same handler used by both the
+ * Devin adapter and the benchmark runner.
+ */
+export declare function createPermissionHandler(): acp.ClientRequestHandler<acp.RequestPermissionRequest, acp.RequestPermissionResponse>;
+/**
+ * Run a full ACP session against a spawned agent subprocess.
+ *
+ * Connects to the agent, initializes the protocol, creates a session in
+ * `workspace`, sends `prompt`, and collects update events until the session
+ * stops or `timeoutMs` elapses.
+ *
+ * The `onToolCall` and `onToolUpdate` callbacks let callers collect tool
+ * calls and observations without duplicating the session loop logic. The
+ * optional `onSession` callback reports the session ID once created. All
+ * callbacks are optional.
+ *
+ * @returns The final {@link PromptResponse} from the agent.
+ */
+export declare function runAcpSession(proc: ChildProcess, workspace: string, prompt: string, timeoutMs: number, onToolCall?: (update: ToolCallMessage) => void, onToolUpdate?: (update: ToolCallUpdateMessage) => void, onSession?: (sessionId: string) => void): Promise<PromptResponse>;
+//#endregion
+//#region src/runner.d.ts
+/**
+ * Generate all plugin on/off combinations.
+ *
+ * For N plugins this produces 2^N combinations, each a full copy of the
+ * plugins array with `enabled` set appropriately.
+ *
+ * - 0 plugins → `[[]]` (one empty combination)
+ * - 1 plugin  → `[[off], [on]]`
+ * - 2 plugins → `[[off,off], [on,off], [off,on], [on,on]]`
+ */
+export declare function pluginCombinations(plugins: PluginConfig[]): PluginConfig[][];
+/**
+ * Run the full arena matrix: task × model × plugin combo × repetitions.
+ *
+ * Each cell spawns the configured agent adapter, installs the plugin
+ * combination, runs the prompt, collects trace + metrics, and kills the
+ * agent. Results are written to `config.outputDir/results.json` and the
+ * full {@link ArenaResult} is returned.
+ */
+export declare function runArena(config: ArenaConfig): Promise<ArenaResult>;
+/**
+ * Aggregate metrics for a group of runs selected by `filter`.
+ *
+ * Computes averages across all matching runs. Returns zeroed metrics when
+ * no runs match. Includes judge-related fields (`avgJudgeScore`,
+ * `judgePassCount`) and a `label` identifying the aggregate group.
+ */
+export declare function aggregateResults(results: RunResult[], filter: (r: RunResult) => boolean, label?: string): AggregateMetrics;
+/**
+ * Compute per-task analysis comparing baseline (no plugins) vs each
+ * candidate plugin combo.
+ *
+ * For each task, groups runs by plugin combo, creates an averaged
+ * representative run per group, then compares the baseline (empty
+ * `pluginIds`) against every non-empty candidate combo using
+ * {@link compareCombos} from `judge.ts`.
+ */
+export declare function computeAnalysis(results: RunResult[], tasks: Task[]): CaseAnalysis[];
+//#endregion
+//#region src/judge.d.ts
+/** Default system prompt for the judge. */
+export declare const DEFAULT_JUDGE_PROMPT: string;
+/**
+ * Build the judge prompt for a single run.
+ *
+ * Includes the task prompt, success criteria, and the agent's output. Does
+ * NOT include which plugins were active unless `config.revealPlugins` is
+ * true. Asks the judge for a JSON response with score, passed, reasoning,
+ * and issues.
+ */
+export declare function buildJudgePrompt(task: Task, runOutput: string, config: JudgeConfig, pluginIds?: string[]): string;
+/**
+ * Parse a judge response into a {@link JudgeVerdict}.
+ *
+ * Extracts JSON from the response, handling markdown code fences
+ * (` ```json ... ``` `). Falls back to `score=0, passed=false` when the
+ * response cannot be parsed.
+ */
+export declare function parseJudgeResponse(response: string, runIndex: number, run: RunResult): JudgeVerdict;
+/**
+ * Judge a single run — spawn the judge agent, send the prompt, collect the
+ * response, parse it into a {@link JudgeVerdict}, and kill the agent.
+ */
+export declare function judgeRun(run: RunResult, task: Task, config: JudgeConfig, runIndex: number): Promise<JudgeVerdict>;
+/**
+ * Judge all runs in a list, pairing each run with its matching task.
+ *
+ * Supports `config.repetitions` — the judge is run N times per run for
+ * confidence. All verdicts are returned (one per repetition).
+ */
+export declare function judgeAllRuns(results: RunResult[], tasks: Task[], config: JudgeConfig): Promise<JudgeVerdict[]>;
+/**
+ * Compare two plugin combos for a task (for {@link ComboComparison}).
+ *
+ * Computes deltas as `candidate - baseline`. Determines whether the
+ * candidate is better:
+ *   - Lower tokens / toolCalls / llmCalls / time = better
+ *   - Higher judgeScore = better
+ *
+ * The judge score for each run is the average of its verdicts' scores
+ * (0 when there are no verdicts).
+ */
+export declare function compareCombos(baseline: RunResult, candidate: RunResult): ComboComparison;
+//#endregion
+export type { AgentAdapter, AgentProcess, AgentSpawnConfig, AggregateMetrics, ArenaConfig, ArenaResult, CaseAnalysis, CheckResult, ComboComparison, DeterministicCheck, JudgeConfig, JudgeVerdict, ModelConfig, Observation, PluginConfig, RunMetrics, RunResult, Task, ToolCall, ToolCallMessage, ToolCallUpdateMessage, TraceData, TraceStep };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/adapters/devin.ts","../src/acp.ts","../src/runner.ts","../src/judge.ts"],"mappings":";;;;;UACiB;EACf;EACA,MAAM,QAAQ,mBAAmB;;UAGlB;EACf,OAAO;EACP;EACA,SAAS;EACT;EACA,MAAM;;UAGS;EACf,IAAI,gBAAgB,oBAAoB,QAAQ;EAChD;;;UAIe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;;;EAIA;;;EAGA,SAAS;;;UAIM;EACf;EACA;EACA;;;UAIe;EACf,OAAO;EACP,OAAO;EACP,QAAQ;EACR,SAAS;EACT;EACA;EACA;;EAEA,QAAQ;;;UAIO;;EAEf,OAAO;;EAEP,OAAO;;EAEP;;EAEA;;EAEA;;;UAIe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA,SAAS;EACT,OAAO;;EAEP;EACA;;EAEA,cAAc;EACd;;EAEA,UAAU;;;UAIK;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA,OAAO;EACP;IACE;IACA;IACA;IACA;;;;UAKa;EACf;EACA;EACA;EACA;EACA,YAAY;EACZ,eAAe;EACf;;;UAIe;EACf;EACA,WAAW;EACX;;;UAIe;EACf;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;;UAIe;EACf;EACA;IACE;IACA;IACA;IACA;IACA;;EAEF,SAAS;EACT,YAAY;;EAEZ,UAAU;;;UAIK;EACf;EACA;EACA;;EAEA,aAAa;;EAEb;;;UAIe;;EAEf;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;;;;UCvMQ;EACR;EACA;EACA;EACA;EACA;IACE;IACA;MACE;MACA;;;;;UAMI;EACR;EACA;EACA;EACA;;;UAIQ;EACR;EACA;EACA;IACE;IACA;IACA;;EAEF,eAAe;EACf,OAAO;;;wBAMO;;wBAeA,cAAc,YAAY;;wBAqC1B,mBAAmB,OAAO;EACxC;EACA;EACA;EACA;;;;;;;;;;;wBA0DoB,eACpB,mBACA,SAAS,iBACR;;;;;;;;;;;;qBAoCU,wBAAwB;WAC1B;EAET,MAAM,QAAQ,mBAAmB;;;;;KC/MvB,kBAAkB,QAAQ;EAAiB;;;KAG3C,wBAAwB,QAClC;EACE;;;;;;;;;;;;wBAqEY,YAAY,QAAQ,mBAAmB;;;;;;;;wBAgBvC,2BAA2B,IAAI,qBAC7C,IAAI,0BACJ,IAAI;;;;;;;;;;;;;;;wBAqFgB,cACpB,MAAM,cACN,mBACA,gBACA,mBACA,cAAc,QAAQ,0BACtB,gBAAgB,QAAQ,gCACxB,aAAa,6BACZ,QAAQ;;;;;;;;;;;;;wBClKK,mBAAmB,SAAS,iBAAiB;;;;;;;;;wBAgCvC,SAAS,QAAQ,cAAc,QAAQ;;;;;;;;wBAkQ7C,iBACd,SAAS,aACT,SAAS,GAAG,uBACZ,iBACC;;;;;;;;;;wBAoIa,gBACd,SAAS,aACT,OAAO,SACN;;;;qBCxcU;;;;;;;;;wBAkBG,iBACd,MAAM,MACN,mBACA,QAAQ,aACR;;;;;;;;wBAgDc,mBACd,kBACA,kBACA,KAAK,YACJ;;;;;wBAsDmB,SACpB,KAAK,WACL,MAAM,MACN,QAAQ,aACR,mBACC,QAAQ;;;;;;;wBAoCW,aACpB,SAAS,aACT,OAAO,QACP,QAAQ,cACP,QAAQ;;;;;;;;;;;;wBAoCK,cACd,UAAU,WACV,WAAW,YACV"}