@sverka/benchmark 0.2.7 → 0.2.9

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { c as BENCHMARK_TASKS, f as runBenchmark, l as DEFAULT_AGENTS, p as writeReport } from "./src-PvGPQO8c.mjs";
+import { c as BENCHMARK_TASKS, f as runBenchmark, l as DEFAULT_AGENTS, p as writeReport } from "./src-D-nX2E21.mjs";
 import { join } from "node:path";
 import process from "node:process";
 //#region src/bin.ts

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"mappings":";;;UACiB;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA,SAAS;EACT;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA,SAAS;EACT;IACE,aAAa;IACb,QAAQ;;;;UAKK;EACf,OAAO;EACP,QAAQ;EACR;EACA;;;;;wBC1Cc,eACd;EAAgB;EAAoB,QAAQ;GAC5C,uBACA,oBACC;;wBAUa,iBACd,SAAS,aACT,oBACC;;wBA6BmB,aACpB,QAAQ,kBACP,QAAQ;;wBA8HW,YACpB,QAAQ,iBACR,qBACC;;;;qBCnMU,0BAA0B;;qBAuC1B,yBAAyB;;;;;;;;;;;UC1BrB;EACf;EACA;EACA;EACA;EACA;IACE;IACA;MACE;MACA;;;;UAKW;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;;EAEF,eAAe;EACf,OAAO;;UAKQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;IACA;;EAEF;EACA,OAAO;;UAGQ;EACf;EACA;EACA;IACE,cAAc;IACd,SAAS;;;;wBAOG;;wBAKA,eAAe,mBAAmB,eAAe;;wBASjD,cAAc,MAAM,iBAAiB;;wBAarC,cAAc,YAAY;;wBAQ1B,oBACd,YAAY,YACZ,oCACC;;wBAgBa,eACd,gBACA,gBACA,2BACA,yBACA,eACC;;wBAcmB,eACpB,OAAO,WACP,oBACC"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"mappings":";;;UACiB;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA,SAAS;EACT;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA,SAAS;EACT;IACE,aAAa;IACb,QAAQ;;;;UAKK;EACf,OAAO;EACP,QAAQ;EACR;EACA;;;;;wBCtCc,eACd;EAAgB;EAAoB,QAAQ;GAC5C,uBACA,oBACC;;wBAUa,iBACd,SAAS,aACT,oBACC;;wBA6BmB,aACpB,QAAQ,kBACP,QAAQ;;wBAgIW,YACpB,QAAQ,iBACR,qBACC;;;;qBCzMU,0BAA0B;;qBAuC1B,yBAAyB;;;;;;;;;;;UC1BrB;EACf;EACA;EACA;EACA;EACA;IACE;IACA;MACE;MACA;;;;UAKW;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;;EAEF,eAAe;EACf,OAAO;;UAKQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;IACE;IACA;IACA;IACA;;EAEF;EACA,OAAO;;UAGQ;EACf;EACA;EACA;IACE,cAAc;IACd,SAAS;;;;wBAOG;;wBAKA,eAAe,mBAAmB,eAAe;;wBASjD,cAAc,MAAM,iBAAiB;;wBAarC,cAAc,YAAY;;wBAQ1B,oBACd,YAAY,YACZ,oCACC;;wBAgBa,eACd,gBACA,gBACA,2BACA,yBACA,eACC;;wBAcmB,eACpB,OAAO,WACP,oBACC"}

package/dist/index.mjs

@@ -1,2 +1,2 @@
-import { a as transformStep, c as BENCHMARK_TASKS, d as extractMetrics, f as runBenchmark, i as transcriptDir, l as DEFAULT_AGENTS, n as countLlmCalls, o as transformTranscript, p as writeReport, r as readTranscript, s as writeTraceData, t as buildTraceData, u as aggregateMetrics } from "./src-PvGPQO8c.mjs";
+import { a as transformStep, c as BENCHMARK_TASKS, d as extractMetrics, f as runBenchmark, i as transcriptDir, l as DEFAULT_AGENTS, n as countLlmCalls, o as transformTranscript, p as writeReport, r as readTranscript, s as writeTraceData, t as buildTraceData, u as aggregateMetrics } from "./src-D-nX2E21.mjs";
 export { BENCHMARK_TASKS, DEFAULT_AGENTS, aggregateMetrics, buildTraceData, countLlmCalls, extractMetrics, readTranscript, runBenchmark, transcriptDir, transformStep, transformTranscript, writeReport, writeTraceData };

package/dist/{src-PvGPQO8c.mjs → src-D-nX2E21.mjs}

@@ -119,27 +119,27 @@ async function setupWorkspace(agent) {
 }
 /** Spawn `devin acp` as a subprocess in the given workspace. */
 function spawnAcpAgent(workspace, model) {
-	const env = sanitizeEnv({
-		model: {
-			id: model,
-			name: model
-		},
-		workspace,
-		plugins: []
-	});
-	return spawn("devin", [
-		"acp",
-		"--model",
-		model
-	], {
+	const opts = {
 		cwd: workspace,
 		stdio: [
 			"pipe",
 			"pipe",
 			"inherit"
 		],
-		env
-	});
+		env: sanitizeEnv({
+			model: {
+				id: model,
+				name: model
+			},
+			workspace,
+			plugins: []
+		})
+	};
+	return spawn("devin", [
+		"acp",
+		"--model",
+		model
+	], opts);
 }
 /** Write benchmark results to a JSON file. */
 async function writeReport(result, outputPath) {
@@ -276,4 +276,4 @@ async function writeTraceData(trace, outputDir) {
 //#endregion
 export { transformStep as a, BENCHMARK_TASKS as c, extractMetrics as d, runBenchmark as f, transcriptDir as i, DEFAULT_AGENTS as l, countLlmCalls as n, transformTranscript as o, writeReport as p, readTranscript as r, writeTraceData as s, buildTraceData as t, aggregateMetrics as u };
 
-//# sourceMappingURL=src-PvGPQO8c.mjs.map
+//# sourceMappingURL=src-D-nX2E21.mjs.map

package/dist/src-D-nX2E21.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"src-D-nX2E21.mjs","names":[],"sources":["../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"sourcesContent":["import {\n  spawn,\n  type ChildProcess,\n  type SpawnOptions,\n} from \"node:child_process\";\nimport { cp, mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { dirname, resolve } from \"node:path\";\n\nimport type { Usage } from \"@agentclientprotocol/sdk\";\n\nimport { sanitizeEnv, runAcpSession, extractUsageTokens } from \"@sverka/arena\";\n\nimport type {\n  Task,\n  AgentConfig,\n  RunMetrics,\n  RunResult,\n  AggregateMetrics,\n  BenchmarkResult,\n  BenchmarkConfig,\n} from \"./types.js\";\n\n/** Extract RunMetrics from a PromptResponse + collected event data. */\nexport function extractMetrics(\n  promptResult: { stopReason: string; usage?: Usage | null },\n  toolCallCount: number,\n  startTime: number,\n): RunMetrics {\n  return {\n    ...extractUsageTokens(promptResult.usage),\n    toolCallCount,\n    executionTimeMs: Date.now() - startTime,\n    stopReason: promptResult.stopReason,\n  };\n}\n\n/** Aggregate metrics for one agent type across multiple RunResults. */\nexport function aggregateMetrics(\n  results: RunResult[],\n  agentType: string,\n): AggregateMetrics {\n  const filtered = results.filter((r) => r.agentType === agentType);\n  const n = filtered.length;\n  if (n === 0) {\n    return {\n      totalTasks: 0,\n      successCount: 0,\n      avgInputTokens: 0,\n      avgOutputTokens: 0,\n      avgTotalTokens: 0,\n      avgToolCalls: 0,\n      avgExecutionTimeMs: 0,\n    };\n  }\n  const sum = (f: (r: RunResult) => number) =>\n    filtered.reduce((a, r) => a + f(r), 0);\n  return {\n    totalTasks: n,\n    successCount: filtered.filter((r) => r.success).length,\n    avgInputTokens: Math.round(sum((r) => r.metrics.inputTokens) / n),\n    avgOutputTokens: Math.round(sum((r) => r.metrics.outputTokens) / n),\n    avgTotalTokens: Math.round(sum((r) => r.metrics.totalTokens) / n),\n    avgToolCalls:\n      Math.round((sum((r) => r.metrics.toolCallCount) / n) * 100) / 100,\n    avgExecutionTimeMs: Math.round(sum((r) => r.metrics.executionTimeMs) / n),\n  };\n}\n\n/** Run the full benchmark: each task × each agent, sequentially. */\nexport async function runBenchmark(\n  config: BenchmarkConfig,\n): Promise<BenchmarkResult> {\n  const model = config.model ?? \"glm-5-2\";\n  const results: RunResult[] = [];\n\n  for (const task of config.tasks) {\n    for (const agent of config.agents) {\n      process.stderr.write(`[${task.id}/${agent.id}] Running...\\n`);\n      const result = await runTask(task, agent, model);\n      results.push(result);\n      process.stderr.write(\n        `[${task.id}/${agent.id}] ${result.success ? \"PASS\" : \"FAIL\"} — ` +\n          `${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ` +\n          `${result.metrics.executionTimeMs}ms\\n`,\n      );\n    }\n  }\n\n  return {\n    timestamp: new Date().toISOString(),\n    model,\n    results,\n    summary: {\n      \"raw-shell\": aggregateMetrics(results, \"raw-shell\"),\n      sverka: aggregateMetrics(results, \"sverka\"),\n    },\n  };\n}\n\n/** Run a single task with one agent. */\nasync function runTask(\n  task: Task,\n  agent: AgentConfig,\n  model: string,\n): Promise<RunResult> {\n  const workspace = await setupWorkspace(agent);\n  const startTime = Date.now();\n  let proc: ChildProcess | null = null;\n\n  try {\n    proc = spawnAcpAgent(workspace, model);\n    let toolCallCount = 0;\n    const promptResult = await runAcpSession(\n      proc,\n      workspace,\n      task.prompt,\n      task.timeoutMs ?? 120000,\n      () => {\n        toolCallCount++;\n      },\n    );\n    const metrics = extractMetrics(promptResult, toolCallCount, startTime);\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: metrics.stopReason === \"end_turn\",\n    };\n  } catch (error) {\n    const metrics = extractMetrics(\n      { stopReason: \"cancelled\", usage: null },\n      0,\n      startTime,\n    );\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: false,\n      error: error instanceof Error ? error.message : String(error),\n    };\n  } finally {\n    proc?.kill(\"SIGTERM\");\n  }\n}\n\n/** Set up workspace for an agent.\n *\n *  Both agents work in the actual Sverka repo so they have the real project\n *  to operate on. The only difference is that the \"sverka\" agent gets the\n *  Sverka skill installed in .agents/skills/sverka/.\n *\n *  We do NOT create a temp copy — copying node_modules + dist is too slow\n *  and the agents need the real build artifacts to run `sverka` commands.\n */\nasync function setupWorkspace(agent: AgentConfig): Promise<string> {\n  const here = dirname(fileURLToPath(import.meta.url));\n  const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n  const workspace = repoRoot;\n\n  if (agent.type === \"sverka\") {\n    // Skill is already in the repo at .agents/skills/sverka/SKILL.md\n    // No need to copy — just ensure the directory exists\n    const skillDir = join(workspace, \".agents\", \"skills\", \"sverka\");\n    const skillFile = join(skillDir, \"SKILL.md\");\n    try {\n      await mkdir(skillDir, { recursive: true });\n      const repoSkillPath = resolveRepoFile(\".agents/skills/sverka/SKILL.md\");\n      if (repoSkillPath && resolve(repoSkillPath) !== resolve(skillFile)) {\n        await cp(repoSkillPath, skillFile, { recursive: true });\n      }\n    } catch {\n      // Skill already exists — that's fine\n    }\n  }\n\n  return workspace;\n}\n\n/** Spawn `devin acp` as a subprocess in the given workspace. */\nfunction spawnAcpAgent(workspace: string, model: string): ChildProcess {\n  const env = sanitizeEnv({\n    model: { id: model, name: model },\n    workspace,\n    plugins: [],\n  });\n  const opts: SpawnOptions = {\n    cwd: workspace,\n    stdio: [\"pipe\", \"pipe\", \"inherit\"],\n    env,\n  };\n  // NOSONAR suppresses only issues on the marker's own line — it must\n  // trail the spawn call, not sit inside the options literal.\n  return spawn(\"devin\", [\"acp\", \"--model\", model], opts); // NOSONAR — PATH needed for devin binary\n}\n\n/** Write benchmark results to a JSON file. */\nexport async function writeReport(\n  result: BenchmarkResult,\n  outputPath: string,\n): Promise<void> {\n  const json = JSON.stringify(result, null, 2);\n  await writeFile(outputPath, json, \"utf-8\");\n}\n\n/** Resolve a file path relative to the repo root. */\nfunction resolveRepoFile(relPath: string): string | null {\n  try {\n    const here = dirname(fileURLToPath(import.meta.url));\n    // From packages/benchmark/src/ or packages/benchmark/dist/ go up to repo root\n    const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n    return join(repoRoot, relPath);\n  } catch {\n    return null;\n  }\n}\n","import type { Task, AgentConfig } from \"./types.js\";\n\n/** 5 task scenarios, ordered simple → complex. */\nexport const BENCHMARK_TASKS: readonly Task[] = [\n  {\n    id: \"run-checks\",\n    name: \"run-checks\",\n    prompt:\n      \"Run all quality checks (lint, typecheck, test, build) for this TypeScript project and report the results. If any check fails, report which one failed and why.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"discover\",\n    name: \"discover\",\n    prompt:\n      \"Discover what checks are available in this project by reading package.json scripts and looking for config files. List each check with its command.\",\n    timeoutMs: 60000,\n  },\n  {\n    id: \"create-config\",\n    name: \"create-config\",\n    prompt:\n      \"Create a CI workflow configuration file for this project that runs lint, typecheck, test, and build in dependency order. The config should define steps with dependencies: lint and typecheck run first (parallel), test depends on both, build depends on test.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"compile-github\",\n    name: \"compile-github\",\n    prompt:\n      \"Compile a CI workflow to GitHub Actions YAML format for this project. The workflow should run lint, typecheck, test, and build on push events. Output the complete YAML.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"multi-step\",\n    name: \"multi-step\",\n    prompt:\n      \"Create a multi-step pipeline with dependencies: lint and typecheck run in parallel, test depends on both, build depends on test. Define the pipeline as code and validate it works correctly.\",\n    timeoutMs: 120000,\n  },\n] as const;\n\n/** Default agent configs: raw-shell (no sverka) vs sverka (CLI + skill). */\nexport const DEFAULT_AGENTS: readonly AgentConfig[] = [\n  {\n    id: \"raw-shell\",\n    name: \"Raw Shell Agent\",\n    type: \"raw-shell\",\n  },\n  {\n    id: \"sverka\",\n    name: \"Sverka CLI Agent\",\n    type: \"sverka\",\n  },\n] as const;\n","/**\n * Transcript collector — reads Devin CLI transcripts and transforms them\n * to trace data for the benchmark trace viewer.\n *\n * Transcript source: ~/.local/share/devin/cli/transcripts/<session-id>.json\n * Each transcript has steps[] with source, message, timestamp, and extra.telemetry.\n * Agent steps with telemetry.operation === \"inference\" are LLM calls.\n */\n\nimport { readFileSync, existsSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport { homedir } from \"node:os\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\n\n// ─── Transcript source types (raw JSON from Devin CLI) ───────────────\n\nexport interface TranscriptStep {\n  step_id: number;\n  source: \"system\" | \"agent\" | \"user\";\n  message: string;\n  timestamp: string;\n  extra?: {\n    generation_model?: string | null;\n    telemetry?: {\n      source?: string;\n      operation?: string;\n    };\n  };\n}\n\nexport interface TranscriptMetrics {\n  total_prompt_tokens: number;\n  total_completion_tokens: number;\n  total_cached_tokens: number;\n  total_steps: number;\n}\n\nexport interface Transcript {\n  session_id: string;\n  schema_version: string;\n  agent: {\n    name: string;\n    version: string;\n    model_name: string;\n  };\n  final_metrics: TranscriptMetrics;\n  steps: TranscriptStep[];\n}\n\n// ─── Trace data types (transformed for viewer) ───────────────────────\n\nexport interface TraceStep {\n  stepId: number;\n  source: \"system\" | \"agent\" | \"user\";\n  timestamp: string;\n  message: string;\n  isLlmCall: boolean;\n}\n\nexport interface TraceAgentData {\n  sessionId: string;\n  model: string;\n  finalMetrics: {\n    totalPromptTokens: number;\n    totalCompletionTokens: number;\n    totalCachedTokens: number;\n    totalSteps: number;\n  };\n  llmCallCount: number;\n  steps: TraceStep[];\n}\n\nexport interface TraceData {\n  taskId: string;\n  prompt: string;\n  agents: {\n    \"raw-shell\"?: TraceAgentData;\n    sverka?: TraceAgentData;\n  };\n}\n\n// ─── Collector functions ─────────────────────────────────────────────\n\n/** Default transcript directory. */\nexport function transcriptDir(): string {\n  return join(homedir(), \".local\", \"share\", \"devin\", \"cli\", \"transcripts\");\n}\n\n/** Read a transcript JSON file by session ID. */\nexport function readTranscript(sessionId: string, dir?: string): Transcript {\n  const path = join(dir ?? transcriptDir(), `${sessionId}.json`);\n  if (!existsSync(path)) {\n    throw new Error(`Transcript not found: ${path}`);\n  }\n  return JSON.parse(readFileSync(path, \"utf-8\")) as Transcript;\n}\n\n/** Transform a raw transcript step to a trace step. */\nexport function transformStep(step: TranscriptStep): TraceStep {\n  const isLlmCall =\n    step.source === \"agent\" && step.extra?.telemetry?.operation === \"inference\";\n  return {\n    stepId: step.step_id,\n    source: step.source,\n    timestamp: step.timestamp,\n    message: step.message,\n    isLlmCall,\n  };\n}\n\n/** Count LLM inference calls in a transcript. */\nexport function countLlmCalls(transcript: Transcript): number {\n  return transcript.steps.filter(\n    (s) =>\n      s.source === \"agent\" && s.extra?.telemetry?.operation === \"inference\",\n  ).length;\n}\n\n/** Transform a full transcript to agent trace data. */\nexport function transformTranscript(\n  transcript: Transcript,\n  agentType: \"raw-shell\" | \"sverka\",\n): TraceAgentData {\n  return {\n    sessionId: transcript.session_id,\n    model: transcript.agent.model_name,\n    finalMetrics: {\n      totalPromptTokens: transcript.final_metrics.total_prompt_tokens,\n      totalCompletionTokens: transcript.final_metrics.total_completion_tokens,\n      totalCachedTokens: transcript.final_metrics.total_cached_tokens,\n      totalSteps: transcript.final_metrics.total_steps,\n    },\n    llmCallCount: countLlmCalls(transcript),\n    steps: transcript.steps.map(transformStep),\n  };\n}\n\n/** Build a combined trace data object for a task from two transcripts. */\nexport function buildTraceData(\n  taskId: string,\n  prompt: string,\n  rawShellSessionId: string,\n  sverkaSessionId: string,\n  dir?: string,\n): TraceData {\n  const rawTranscript = readTranscript(rawShellSessionId, dir);\n  const sverkaTranscript = readTranscript(sverkaSessionId, dir);\n  return {\n    taskId,\n    prompt,\n    agents: {\n      \"raw-shell\": transformTranscript(rawTranscript, \"raw-shell\"),\n      sverka: transformTranscript(sverkaTranscript, \"sverka\"),\n    },\n  };\n}\n\n/** Write trace data to the benchmark traces directory. */\nexport async function writeTraceData(\n  trace: TraceData,\n  outputDir: string,\n): Promise<string> {\n  // Reject path traversal in taskId to prevent writing outside outputDir\n  if (\n    trace.taskId.includes(\"..\") ||\n    trace.taskId.includes(\"/\") ||\n    trace.taskId.includes(\"\\\\\")\n  ) {\n    throw new Error(`Invalid taskId for trace output: ${trace.taskId}`);\n  }\n  const taskDir = join(outputDir, trace.taskId);\n  await mkdir(taskDir, { recursive: true });\n  const path = join(taskDir, \"trace.json\");\n  await writeFile(path, JSON.stringify(trace, null, 2), \"utf-8\");\n  return path;\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAgB,eACd,cACA,eACA,WACY;CACZ,OAAO;EACL,GAAG,mBAAmB,aAAa,KAAK;EACxC;EACA,iBAAiB,KAAK,IAAI,IAAI;EAC9B,YAAY,aAAa;CAC3B;AACF;;AAGA,SAAgB,iBACd,SACA,WACkB;CAClB,MAAM,WAAW,QAAQ,QAAQ,MAAM,EAAE,cAAc,SAAS;CAChE,MAAM,IAAI,SAAS;CACnB,IAAI,MAAM,GACR,OAAO;EACL,YAAY;EACZ,cAAc;EACd,gBAAgB;EAChB,iBAAiB;EACjB,gBAAgB;EAChB,cAAc;EACd,oBAAoB;CACtB;CAEF,MAAM,OAAO,MACX,SAAS,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC;CACvC,OAAO;EACL,YAAY;EACZ,cAAc,SAAS,QAAQ,MAAM,EAAE,OAAO,CAAC,CAAC;EAChD,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,iBAAiB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,YAAY,IAAI,CAAC;EAClE,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,cACE,KAAK,MAAO,KAAK,MAAM,EAAE,QAAQ,aAAa,IAAI,IAAK,GAAG,IAAI;EAChE,oBAAoB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,eAAe,IAAI,CAAC;CAC1E;AACF;;AAGA,eAAsB,aACpB,QAC0B;CAC1B,MAAM,QAAQ,OAAO,SAAS;CAC9B,MAAM,UAAuB,CAAC;CAE9B,KAAK,MAAM,QAAQ,OAAO,OACxB,KAAK,MAAM,SAAS,OAAO,QAAQ;EACjC,QAAQ,OAAO,MAAM,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,eAAe;EAC5D,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO,KAAK;EAC/C,QAAQ,KAAK,MAAM;EACnB,QAAQ,OAAO,MACb,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,IAAI,OAAO,UAAU,SAAS,OAAO,KACxD,OAAO,QAAQ,YAAY,WAAW,OAAO,QAAQ,cAAc,UACnE,OAAO,QAAQ,gBAAgB,KACtC;CACF;CAGF,OAAO;EACL,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;EAClC;EACA;EACA,SAAS;GACP,aAAa,iBAAiB,SAAS,WAAW;GAClD,QAAQ,iBAAiB,SAAS,QAAQ;EAC5C;CACF;AACF;;AAGA,eAAe,QACb,MACA,OACA,OACoB;CACpB,MAAM,YAAY,MAAM,eAAe,KAAK;CAC5C,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI,OAA4B;CAEhC,IAAI;EACF,OAAO,cAAc,WAAW,KAAK;EACrC,IAAI,gBAAgB;EAUpB,MAAM,UAAU,eAAe,MATJ,cACzB,MACA,WACA,KAAK,QACL,KAAK,aAAa,YACZ;GACJ;EACF,CACF,GAC6C,eAAe,SAAS;EACrE,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS,QAAQ,eAAe;EAClC;CACF,SAAS,OAAO;EACd,MAAM,UAAU,eACd;GAAE,YAAY;GAAa,OAAO;EAAK,GACvC,GACA,SACF;EACA,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS;GACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9D;CACF,UAAU;EACR,MAAM,KAAK,SAAS;CACtB;AACF;;;;;;;;;;AAWA,eAAe,eAAe,OAAqC;CACjE,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;CAEnD,MAAM,YADW,QAAQ,MAAM,MAAM,MAAM,IAClB;CAEzB,IAAI,MAAM,SAAS,UAAU;EAG3B,MAAM,WAAW,KAAK,WAAW,WAAW,UAAU,QAAQ;EAC9D,MAAM,YAAY,KAAK,UAAU,UAAU;EAC3C,IAAI;GACF,MAAM,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;GACzC,MAAM,gBAAgB,gBAAgB,gCAAgC;GACtE,IAAI,iBAAiB,QAAQ,aAAa,MAAM,QAAQ,SAAS,GAC/D,MAAM,GAAG,eAAe,WAAW,EAAE,WAAW,KAAK,CAAC;EAE1D,QAAQ,CAER;CACF;CAEA,OAAO;AACT;;AAGA,SAAS,cAAc,WAAmB,OAA6B;CAMrE,MAAM,OAAqB;EACzB,KAAK;EACL,OAAO;GAAC;GAAQ;GAAQ;EAAS;EACjC,KARU,YAAY;GACtB,OAAO;IAAE,IAAI;IAAO,MAAM;GAAM;GAChC;GACA,SAAS,CAAC;EACZ,CAII;CACJ;CAGA,OAAO,MAAM,SAAS;EAAC;EAAO;EAAW;CAAK,GAAG,IAAI;AACvD;;AAGA,eAAsB,YACpB,QACA,YACe;CACf,MAAM,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;CAC3C,MAAM,UAAU,YAAY,MAAM,OAAO;AAC3C;;AAGA,SAAS,gBAAgB,SAAgC;CACvD,IAAI;EACF,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;EAEnD,MAAM,WAAW,QAAQ,MAAM,MAAM,MAAM,IAAI;EAC/C,OAAO,KAAK,UAAU,OAAO;CAC/B,QAAQ;EACN,OAAO;CACT;AACF;;;;ACxNA,MAAa,kBAAmC;CAC9C;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;AACF;;AAGA,MAAa,iBAAyC,CACpD;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,GACA;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,CACF;;;;;;;;;;;;AC+BA,SAAgB,gBAAwB;CACtC,OAAO,KAAK,QAAQ,GAAG,UAAU,SAAS,SAAS,OAAO,aAAa;AACzE;;AAGA,SAAgB,eAAe,WAAmB,KAA0B;CAC1E,MAAM,OAAO,KAAK,OAAO,cAAc,GAAG,GAAG,UAAU,MAAM;CAC7D,IAAI,CAAC,WAAW,IAAI,GAClB,MAAM,IAAI,MAAM,yBAAyB,MAAM;CAEjD,OAAO,KAAK,MAAM,aAAa,MAAM,OAAO,CAAC;AAC/C;;AAGA,SAAgB,cAAc,MAAiC;CAC7D,MAAM,YACJ,KAAK,WAAW,WAAW,KAAK,OAAO,WAAW,cAAc;CAClE,OAAO;EACL,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,WAAW,KAAK;EAChB,SAAS,KAAK;EACd;CACF;AACF;;AAGA,SAAgB,cAAc,YAAgC;CAC5D,OAAO,WAAW,MAAM,QACrB,MACC,EAAE,WAAW,WAAW,EAAE,OAAO,WAAW,cAAc,WAC9D,CAAC,CAAC;AACJ;;AAGA,SAAgB,oBACd,YACA,WACgB;CAChB,OAAO;EACL,WAAW,WAAW;EACtB,OAAO,WAAW,MAAM;EACxB,cAAc;GACZ,mBAAmB,WAAW,cAAc;GAC5C,uBAAuB,WAAW,cAAc;GAChD,mBAAmB,WAAW,cAAc;GAC5C,YAAY,WAAW,cAAc;EACvC;EACA,cAAc,cAAc,UAAU;EACtC,OAAO,WAAW,MAAM,IAAI,aAAa;CAC3C;AACF;;AAGA,SAAgB,eACd,QACA,QACA,mBACA,iBACA,KACW;CACX,MAAM,gBAAgB,eAAe,mBAAmB,GAAG;CAC3D,MAAM,mBAAmB,eAAe,iBAAiB,GAAG;CAC5D,OAAO;EACL;EACA;EACA,QAAQ;GACN,aAAa,oBAAoB,eAAe,WAAW;GAC3D,QAAQ,oBAAoB,kBAAkB,QAAQ;EACxD;CACF;AACF;;AAGA,eAAsB,eACpB,OACA,WACiB;CAEjB,IACE,MAAM,OAAO,SAAS,IAAI,KAC1B,MAAM,OAAO,SAAS,GAAG,KACzB,MAAM,OAAO,SAAS,IAAI,GAE1B,MAAM,IAAI,MAAM,oCAAoC,MAAM,QAAQ;CAEpE,MAAM,UAAU,KAAK,WAAW,MAAM,MAAM;CAC5C,MAAM,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;CACxC,MAAM,OAAO,KAAK,SAAS,YAAY;CACvC,MAAM,UAAU,MAAM,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,OAAO;CAC7D,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sverka/benchmark",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "Sverka benchmark arena — two-agent comparison harness",
   "license": "MIT",
   "sideEffects": false,
@@ -43,7 +43,7 @@
   },
   "dependencies": {
     "@agentclientprotocol/sdk": "^1.5.0",
-    "@sverka/arena": "^0.2.7"
+    "@sverka/arena": "^0.2.9"
   },
   "devDependencies": {
     "tsdown": "^0.23.0",

package/dist/src-PvGPQO8c.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"src-PvGPQO8c.mjs","names":[],"sources":["../src/arena.ts","../src/tasks.ts","../src/collector.ts"],"sourcesContent":["import { spawn, type ChildProcess } from \"node:child_process\";\nimport { cp, mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { dirname, resolve } from \"node:path\";\n\nimport type { Usage } from \"@agentclientprotocol/sdk\";\n\nimport { sanitizeEnv, runAcpSession, extractUsageTokens } from \"@sverka/arena\";\n\nimport type {\n  Task,\n  AgentConfig,\n  RunMetrics,\n  RunResult,\n  AggregateMetrics,\n  BenchmarkResult,\n  BenchmarkConfig,\n} from \"./types.js\";\n\n/** Extract RunMetrics from a PromptResponse + collected event data. */\nexport function extractMetrics(\n  promptResult: { stopReason: string; usage?: Usage | null },\n  toolCallCount: number,\n  startTime: number,\n): RunMetrics {\n  return {\n    ...extractUsageTokens(promptResult.usage),\n    toolCallCount,\n    executionTimeMs: Date.now() - startTime,\n    stopReason: promptResult.stopReason,\n  };\n}\n\n/** Aggregate metrics for one agent type across multiple RunResults. */\nexport function aggregateMetrics(\n  results: RunResult[],\n  agentType: string,\n): AggregateMetrics {\n  const filtered = results.filter((r) => r.agentType === agentType);\n  const n = filtered.length;\n  if (n === 0) {\n    return {\n      totalTasks: 0,\n      successCount: 0,\n      avgInputTokens: 0,\n      avgOutputTokens: 0,\n      avgTotalTokens: 0,\n      avgToolCalls: 0,\n      avgExecutionTimeMs: 0,\n    };\n  }\n  const sum = (f: (r: RunResult) => number) =>\n    filtered.reduce((a, r) => a + f(r), 0);\n  return {\n    totalTasks: n,\n    successCount: filtered.filter((r) => r.success).length,\n    avgInputTokens: Math.round(sum((r) => r.metrics.inputTokens) / n),\n    avgOutputTokens: Math.round(sum((r) => r.metrics.outputTokens) / n),\n    avgTotalTokens: Math.round(sum((r) => r.metrics.totalTokens) / n),\n    avgToolCalls:\n      Math.round((sum((r) => r.metrics.toolCallCount) / n) * 100) / 100,\n    avgExecutionTimeMs: Math.round(sum((r) => r.metrics.executionTimeMs) / n),\n  };\n}\n\n/** Run the full benchmark: each task × each agent, sequentially. */\nexport async function runBenchmark(\n  config: BenchmarkConfig,\n): Promise<BenchmarkResult> {\n  const model = config.model ?? \"glm-5-2\";\n  const results: RunResult[] = [];\n\n  for (const task of config.tasks) {\n    for (const agent of config.agents) {\n      process.stderr.write(`[${task.id}/${agent.id}] Running...\\n`);\n      const result = await runTask(task, agent, model);\n      results.push(result);\n      process.stderr.write(\n        `[${task.id}/${agent.id}] ${result.success ? \"PASS\" : \"FAIL\"} — ` +\n          `${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ` +\n          `${result.metrics.executionTimeMs}ms\\n`,\n      );\n    }\n  }\n\n  return {\n    timestamp: new Date().toISOString(),\n    model,\n    results,\n    summary: {\n      \"raw-shell\": aggregateMetrics(results, \"raw-shell\"),\n      sverka: aggregateMetrics(results, \"sverka\"),\n    },\n  };\n}\n\n/** Run a single task with one agent. */\nasync function runTask(\n  task: Task,\n  agent: AgentConfig,\n  model: string,\n): Promise<RunResult> {\n  const workspace = await setupWorkspace(agent);\n  const startTime = Date.now();\n  let proc: ChildProcess | null = null;\n\n  try {\n    proc = spawnAcpAgent(workspace, model);\n    let toolCallCount = 0;\n    const promptResult = await runAcpSession(\n      proc,\n      workspace,\n      task.prompt,\n      task.timeoutMs ?? 120000,\n      () => {\n        toolCallCount++;\n      },\n    );\n    const metrics = extractMetrics(promptResult, toolCallCount, startTime);\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: metrics.stopReason === \"end_turn\",\n    };\n  } catch (error) {\n    const metrics = extractMetrics(\n      { stopReason: \"cancelled\", usage: null },\n      0,\n      startTime,\n    );\n    return {\n      taskId: task.id,\n      agentId: agent.id,\n      agentType: agent.type,\n      metrics,\n      success: false,\n      error: error instanceof Error ? error.message : String(error),\n    };\n  } finally {\n    proc?.kill(\"SIGTERM\");\n  }\n}\n\n/** Set up workspace for an agent.\n *\n *  Both agents work in the actual Sverka repo so they have the real project\n *  to operate on. The only difference is that the \"sverka\" agent gets the\n *  Sverka skill installed in .agents/skills/sverka/.\n *\n *  We do NOT create a temp copy — copying node_modules + dist is too slow\n *  and the agents need the real build artifacts to run `sverka` commands.\n */\nasync function setupWorkspace(agent: AgentConfig): Promise<string> {\n  const here = dirname(fileURLToPath(import.meta.url));\n  const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n  const workspace = repoRoot;\n\n  if (agent.type === \"sverka\") {\n    // Skill is already in the repo at .agents/skills/sverka/SKILL.md\n    // No need to copy — just ensure the directory exists\n    const skillDir = join(workspace, \".agents\", \"skills\", \"sverka\");\n    const skillFile = join(skillDir, \"SKILL.md\");\n    try {\n      await mkdir(skillDir, { recursive: true });\n      const repoSkillPath = resolveRepoFile(\".agents/skills/sverka/SKILL.md\");\n      if (repoSkillPath && resolve(repoSkillPath) !== resolve(skillFile)) {\n        await cp(repoSkillPath, skillFile, { recursive: true });\n      }\n    } catch {\n      // Skill already exists — that's fine\n    }\n  }\n\n  return workspace;\n}\n\n/** Spawn `devin acp` as a subprocess in the given workspace. */\nfunction spawnAcpAgent(workspace: string, model: string): ChildProcess {\n  const env = sanitizeEnv({\n    model: { id: model, name: model },\n    workspace,\n    plugins: [],\n  });\n  return spawn(\"devin\", [\"acp\", \"--model\", model], {\n    // NOSONAR — PATH needed for devin binary\n    cwd: workspace,\n    stdio: [\"pipe\", \"pipe\", \"inherit\"],\n    env,\n  });\n}\n\n/** Write benchmark results to a JSON file. */\nexport async function writeReport(\n  result: BenchmarkResult,\n  outputPath: string,\n): Promise<void> {\n  const json = JSON.stringify(result, null, 2);\n  await writeFile(outputPath, json, \"utf-8\");\n}\n\n/** Resolve a file path relative to the repo root. */\nfunction resolveRepoFile(relPath: string): string | null {\n  try {\n    const here = dirname(fileURLToPath(import.meta.url));\n    // From packages/benchmark/src/ or packages/benchmark/dist/ go up to repo root\n    const repoRoot = resolve(here, \"..\", \"..\", \"..\");\n    return join(repoRoot, relPath);\n  } catch {\n    return null;\n  }\n}\n","import type { Task, AgentConfig } from \"./types.js\";\n\n/** 5 task scenarios, ordered simple → complex. */\nexport const BENCHMARK_TASKS: readonly Task[] = [\n  {\n    id: \"run-checks\",\n    name: \"run-checks\",\n    prompt:\n      \"Run all quality checks (lint, typecheck, test, build) for this TypeScript project and report the results. If any check fails, report which one failed and why.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"discover\",\n    name: \"discover\",\n    prompt:\n      \"Discover what checks are available in this project by reading package.json scripts and looking for config files. List each check with its command.\",\n    timeoutMs: 60000,\n  },\n  {\n    id: \"create-config\",\n    name: \"create-config\",\n    prompt:\n      \"Create a CI workflow configuration file for this project that runs lint, typecheck, test, and build in dependency order. The config should define steps with dependencies: lint and typecheck run first (parallel), test depends on both, build depends on test.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"compile-github\",\n    name: \"compile-github\",\n    prompt:\n      \"Compile a CI workflow to GitHub Actions YAML format for this project. The workflow should run lint, typecheck, test, and build on push events. Output the complete YAML.\",\n    timeoutMs: 120000,\n  },\n  {\n    id: \"multi-step\",\n    name: \"multi-step\",\n    prompt:\n      \"Create a multi-step pipeline with dependencies: lint and typecheck run in parallel, test depends on both, build depends on test. Define the pipeline as code and validate it works correctly.\",\n    timeoutMs: 120000,\n  },\n] as const;\n\n/** Default agent configs: raw-shell (no sverka) vs sverka (CLI + skill). */\nexport const DEFAULT_AGENTS: readonly AgentConfig[] = [\n  {\n    id: \"raw-shell\",\n    name: \"Raw Shell Agent\",\n    type: \"raw-shell\",\n  },\n  {\n    id: \"sverka\",\n    name: \"Sverka CLI Agent\",\n    type: \"sverka\",\n  },\n] as const;\n","/**\n * Transcript collector — reads Devin CLI transcripts and transforms them\n * to trace data for the benchmark trace viewer.\n *\n * Transcript source: ~/.local/share/devin/cli/transcripts/<session-id>.json\n * Each transcript has steps[] with source, message, timestamp, and extra.telemetry.\n * Agent steps with telemetry.operation === \"inference\" are LLM calls.\n */\n\nimport { readFileSync, existsSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport { homedir } from \"node:os\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\n\n// ─── Transcript source types (raw JSON from Devin CLI) ───────────────\n\nexport interface TranscriptStep {\n  step_id: number;\n  source: \"system\" | \"agent\" | \"user\";\n  message: string;\n  timestamp: string;\n  extra?: {\n    generation_model?: string | null;\n    telemetry?: {\n      source?: string;\n      operation?: string;\n    };\n  };\n}\n\nexport interface TranscriptMetrics {\n  total_prompt_tokens: number;\n  total_completion_tokens: number;\n  total_cached_tokens: number;\n  total_steps: number;\n}\n\nexport interface Transcript {\n  session_id: string;\n  schema_version: string;\n  agent: {\n    name: string;\n    version: string;\n    model_name: string;\n  };\n  final_metrics: TranscriptMetrics;\n  steps: TranscriptStep[];\n}\n\n// ─── Trace data types (transformed for viewer) ───────────────────────\n\nexport interface TraceStep {\n  stepId: number;\n  source: \"system\" | \"agent\" | \"user\";\n  timestamp: string;\n  message: string;\n  isLlmCall: boolean;\n}\n\nexport interface TraceAgentData {\n  sessionId: string;\n  model: string;\n  finalMetrics: {\n    totalPromptTokens: number;\n    totalCompletionTokens: number;\n    totalCachedTokens: number;\n    totalSteps: number;\n  };\n  llmCallCount: number;\n  steps: TraceStep[];\n}\n\nexport interface TraceData {\n  taskId: string;\n  prompt: string;\n  agents: {\n    \"raw-shell\"?: TraceAgentData;\n    sverka?: TraceAgentData;\n  };\n}\n\n// ─── Collector functions ─────────────────────────────────────────────\n\n/** Default transcript directory. */\nexport function transcriptDir(): string {\n  return join(homedir(), \".local\", \"share\", \"devin\", \"cli\", \"transcripts\");\n}\n\n/** Read a transcript JSON file by session ID. */\nexport function readTranscript(sessionId: string, dir?: string): Transcript {\n  const path = join(dir ?? transcriptDir(), `${sessionId}.json`);\n  if (!existsSync(path)) {\n    throw new Error(`Transcript not found: ${path}`);\n  }\n  return JSON.parse(readFileSync(path, \"utf-8\")) as Transcript;\n}\n\n/** Transform a raw transcript step to a trace step. */\nexport function transformStep(step: TranscriptStep): TraceStep {\n  const isLlmCall =\n    step.source === \"agent\" && step.extra?.telemetry?.operation === \"inference\";\n  return {\n    stepId: step.step_id,\n    source: step.source,\n    timestamp: step.timestamp,\n    message: step.message,\n    isLlmCall,\n  };\n}\n\n/** Count LLM inference calls in a transcript. */\nexport function countLlmCalls(transcript: Transcript): number {\n  return transcript.steps.filter(\n    (s) =>\n      s.source === \"agent\" && s.extra?.telemetry?.operation === \"inference\",\n  ).length;\n}\n\n/** Transform a full transcript to agent trace data. */\nexport function transformTranscript(\n  transcript: Transcript,\n  agentType: \"raw-shell\" | \"sverka\",\n): TraceAgentData {\n  return {\n    sessionId: transcript.session_id,\n    model: transcript.agent.model_name,\n    finalMetrics: {\n      totalPromptTokens: transcript.final_metrics.total_prompt_tokens,\n      totalCompletionTokens: transcript.final_metrics.total_completion_tokens,\n      totalCachedTokens: transcript.final_metrics.total_cached_tokens,\n      totalSteps: transcript.final_metrics.total_steps,\n    },\n    llmCallCount: countLlmCalls(transcript),\n    steps: transcript.steps.map(transformStep),\n  };\n}\n\n/** Build a combined trace data object for a task from two transcripts. */\nexport function buildTraceData(\n  taskId: string,\n  prompt: string,\n  rawShellSessionId: string,\n  sverkaSessionId: string,\n  dir?: string,\n): TraceData {\n  const rawTranscript = readTranscript(rawShellSessionId, dir);\n  const sverkaTranscript = readTranscript(sverkaSessionId, dir);\n  return {\n    taskId,\n    prompt,\n    agents: {\n      \"raw-shell\": transformTranscript(rawTranscript, \"raw-shell\"),\n      sverka: transformTranscript(sverkaTranscript, \"sverka\"),\n    },\n  };\n}\n\n/** Write trace data to the benchmark traces directory. */\nexport async function writeTraceData(\n  trace: TraceData,\n  outputDir: string,\n): Promise<string> {\n  // Reject path traversal in taskId to prevent writing outside outputDir\n  if (\n    trace.taskId.includes(\"..\") ||\n    trace.taskId.includes(\"/\") ||\n    trace.taskId.includes(\"\\\\\")\n  ) {\n    throw new Error(`Invalid taskId for trace output: ${trace.taskId}`);\n  }\n  const taskDir = join(outputDir, trace.taskId);\n  await mkdir(taskDir, { recursive: true });\n  const path = join(taskDir, \"trace.json\");\n  await writeFile(path, JSON.stringify(trace, null, 2), \"utf-8\");\n  return path;\n}\n"],"mappings":";;;;;;;;;AAqBA,SAAgB,eACd,cACA,eACA,WACY;CACZ,OAAO;EACL,GAAG,mBAAmB,aAAa,KAAK;EACxC;EACA,iBAAiB,KAAK,IAAI,IAAI;EAC9B,YAAY,aAAa;CAC3B;AACF;;AAGA,SAAgB,iBACd,SACA,WACkB;CAClB,MAAM,WAAW,QAAQ,QAAQ,MAAM,EAAE,cAAc,SAAS;CAChE,MAAM,IAAI,SAAS;CACnB,IAAI,MAAM,GACR,OAAO;EACL,YAAY;EACZ,cAAc;EACd,gBAAgB;EAChB,iBAAiB;EACjB,gBAAgB;EAChB,cAAc;EACd,oBAAoB;CACtB;CAEF,MAAM,OAAO,MACX,SAAS,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC;CACvC,OAAO;EACL,YAAY;EACZ,cAAc,SAAS,QAAQ,MAAM,EAAE,OAAO,CAAC,CAAC;EAChD,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,iBAAiB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,YAAY,IAAI,CAAC;EAClE,gBAAgB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,WAAW,IAAI,CAAC;EAChE,cACE,KAAK,MAAO,KAAK,MAAM,EAAE,QAAQ,aAAa,IAAI,IAAK,GAAG,IAAI;EAChE,oBAAoB,KAAK,MAAM,KAAK,MAAM,EAAE,QAAQ,eAAe,IAAI,CAAC;CAC1E;AACF;;AAGA,eAAsB,aACpB,QAC0B;CAC1B,MAAM,QAAQ,OAAO,SAAS;CAC9B,MAAM,UAAuB,CAAC;CAE9B,KAAK,MAAM,QAAQ,OAAO,OACxB,KAAK,MAAM,SAAS,OAAO,QAAQ;EACjC,QAAQ,OAAO,MAAM,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,eAAe;EAC5D,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO,KAAK;EAC/C,QAAQ,KAAK,MAAM;EACnB,QAAQ,OAAO,MACb,IAAI,KAAK,GAAG,GAAG,MAAM,GAAG,IAAI,OAAO,UAAU,SAAS,OAAO,KACxD,OAAO,QAAQ,YAAY,WAAW,OAAO,QAAQ,cAAc,UACnE,OAAO,QAAQ,gBAAgB,KACtC;CACF;CAGF,OAAO;EACL,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;EAClC;EACA;EACA,SAAS;GACP,aAAa,iBAAiB,SAAS,WAAW;GAClD,QAAQ,iBAAiB,SAAS,QAAQ;EAC5C;CACF;AACF;;AAGA,eAAe,QACb,MACA,OACA,OACoB;CACpB,MAAM,YAAY,MAAM,eAAe,KAAK;CAC5C,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI,OAA4B;CAEhC,IAAI;EACF,OAAO,cAAc,WAAW,KAAK;EACrC,IAAI,gBAAgB;EAUpB,MAAM,UAAU,eAAe,MATJ,cACzB,MACA,WACA,KAAK,QACL,KAAK,aAAa,YACZ;GACJ;EACF,CACF,GAC6C,eAAe,SAAS;EACrE,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS,QAAQ,eAAe;EAClC;CACF,SAAS,OAAO;EACd,MAAM,UAAU,eACd;GAAE,YAAY;GAAa,OAAO;EAAK,GACvC,GACA,SACF;EACA,OAAO;GACL,QAAQ,KAAK;GACb,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA,SAAS;GACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9D;CACF,UAAU;EACR,MAAM,KAAK,SAAS;CACtB;AACF;;;;;;;;;;AAWA,eAAe,eAAe,OAAqC;CACjE,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;CAEnD,MAAM,YADW,QAAQ,MAAM,MAAM,MAAM,IAClB;CAEzB,IAAI,MAAM,SAAS,UAAU;EAG3B,MAAM,WAAW,KAAK,WAAW,WAAW,UAAU,QAAQ;EAC9D,MAAM,YAAY,KAAK,UAAU,UAAU;EAC3C,IAAI;GACF,MAAM,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;GACzC,MAAM,gBAAgB,gBAAgB,gCAAgC;GACtE,IAAI,iBAAiB,QAAQ,aAAa,MAAM,QAAQ,SAAS,GAC/D,MAAM,GAAG,eAAe,WAAW,EAAE,WAAW,KAAK,CAAC;EAE1D,QAAQ,CAER;CACF;CAEA,OAAO;AACT;;AAGA,SAAS,cAAc,WAAmB,OAA6B;CACrE,MAAM,MAAM,YAAY;EACtB,OAAO;GAAE,IAAI;GAAO,MAAM;EAAM;EAChC;EACA,SAAS,CAAC;CACZ,CAAC;CACD,OAAO,MAAM,SAAS;EAAC;EAAO;EAAW;CAAK,GAAG;EAE/C,KAAK;EACL,OAAO;GAAC;GAAQ;GAAQ;EAAS;EACjC;CACF,CAAC;AACH;;AAGA,eAAsB,YACpB,QACA,YACe;CACf,MAAM,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;CAC3C,MAAM,UAAU,YAAY,MAAM,OAAO;AAC3C;;AAGA,SAAS,gBAAgB,SAAgC;CACvD,IAAI;EACF,MAAM,OAAO,QAAQ,cAAc,YAAY,GAAG,CAAC;EAEnD,MAAM,WAAW,QAAQ,MAAM,MAAM,MAAM,IAAI;EAC/C,OAAO,KAAK,UAAU,OAAO;CAC/B,QAAQ;EACN,OAAO;CACT;AACF;;;;AClNA,MAAa,kBAAmC;CAC9C;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;CACA;EACE,IAAI;EACJ,MAAM;EACN,QACE;EACF,WAAW;CACb;AACF;;AAGA,MAAa,iBAAyC,CACpD;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,GACA;CACE,IAAI;CACJ,MAAM;CACN,MAAM;AACR,CACF;;;;;;;;;;;;AC+BA,SAAgB,gBAAwB;CACtC,OAAO,KAAK,QAAQ,GAAG,UAAU,SAAS,SAAS,OAAO,aAAa;AACzE;;AAGA,SAAgB,eAAe,WAAmB,KAA0B;CAC1E,MAAM,OAAO,KAAK,OAAO,cAAc,GAAG,GAAG,UAAU,MAAM;CAC7D,IAAI,CAAC,WAAW,IAAI,GAClB,MAAM,IAAI,MAAM,yBAAyB,MAAM;CAEjD,OAAO,KAAK,MAAM,aAAa,MAAM,OAAO,CAAC;AAC/C;;AAGA,SAAgB,cAAc,MAAiC;CAC7D,MAAM,YACJ,KAAK,WAAW,WAAW,KAAK,OAAO,WAAW,cAAc;CAClE,OAAO;EACL,QAAQ,KAAK;EACb,QAAQ,KAAK;EACb,WAAW,KAAK;EAChB,SAAS,KAAK;EACd;CACF;AACF;;AAGA,SAAgB,cAAc,YAAgC;CAC5D,OAAO,WAAW,MAAM,QACrB,MACC,EAAE,WAAW,WAAW,EAAE,OAAO,WAAW,cAAc,WAC9D,CAAC,CAAC;AACJ;;AAGA,SAAgB,oBACd,YACA,WACgB;CAChB,OAAO;EACL,WAAW,WAAW;EACtB,OAAO,WAAW,MAAM;EACxB,cAAc;GACZ,mBAAmB,WAAW,cAAc;GAC5C,uBAAuB,WAAW,cAAc;GAChD,mBAAmB,WAAW,cAAc;GAC5C,YAAY,WAAW,cAAc;EACvC;EACA,cAAc,cAAc,UAAU;EACtC,OAAO,WAAW,MAAM,IAAI,aAAa;CAC3C;AACF;;AAGA,SAAgB,eACd,QACA,QACA,mBACA,iBACA,KACW;CACX,MAAM,gBAAgB,eAAe,mBAAmB,GAAG;CAC3D,MAAM,mBAAmB,eAAe,iBAAiB,GAAG;CAC5D,OAAO;EACL;EACA;EACA,QAAQ;GACN,aAAa,oBAAoB,eAAe,WAAW;GAC3D,QAAQ,oBAAoB,kBAAkB,QAAQ;EACxD;CACF;AACF;;AAGA,eAAsB,eACpB,OACA,WACiB;CAEjB,IACE,MAAM,OAAO,SAAS,IAAI,KAC1B,MAAM,OAAO,SAAS,GAAG,KACzB,MAAM,OAAO,SAAS,IAAI,GAE1B,MAAM,IAAI,MAAM,oCAAoC,MAAM,QAAQ;CAEpE,MAAM,UAAU,KAAK,WAAW,MAAM,MAAM;CAC5C,MAAM,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;CACxC,MAAM,OAAO,KAAK,SAAS,YAAY;CACvC,MAAM,UAAU,MAAM,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,OAAO;CAC7D,OAAO;AACT"}