contextos-agents 1.6.1 → 2.0.0-beta.2

package/benchmarks/gemini-issues.js

@@ -1,533 +0,0 @@
-#!/usr/bin/env node
-
-/*
- * A reproducible, paired benchmark for ContextOS skills.
- *
- * It discovers closed JavaScript GitHub Issues linked to merged pull requests,
- * checks out the parent of each merge commit, and gives the same task to Gemini
- * twice: once with a neutral instruction and once with ContextOS skills. The
- * coding loop is intentionally controller-driven: the model may read files and
- * return a unified diff, but it never receives shell access.
- */
-
-'use strict';
-
-const crypto = require('crypto');
-const fs = require('fs');
-const https = require('https');
-const os = require('os');
-const path = require('path');
-const { spawn } = require('child_process');
-
-// The benchmark belongs to the package but operates on the project from which
-// it is invoked, including that project's installed .agents directory.
-const ROOT = process.cwd();
-const DEFAULT_COUNT = 20;
-const DEFAULT_MAX_ITERATIONS = 6;
-const DEFAULT_MODEL = 'gemini-2.5-flash';
-const DEFAULT_QUERY = 'is:issue is:closed language:JavaScript label:bug';
-const MAX_FILE_BYTES = 24 * 1024;
-const MAX_OUTPUT_BYTES = 16 * 1024;
-const DEFAULT_SKILL_FILES = [
-  '.agents/AGENTS.md',
-  '.agents/core/skills/engineering-workflow/SKILL.md',
-  '.agents/core/skills/system-design/SKILL.md',
-  '.agents/core/skills/node/SKILL.md',
-  '.agents/core/skills/security/SKILL.md',
-  '.agents/core/skills/typescript/SKILL.md',
-];
-
-function parseArgs(argv) {
-  const options = {
-    count: DEFAULT_COUNT,
-    maxIterations: DEFAULT_MAX_ITERATIONS,
-    model: DEFAULT_MODEL,
-    query: DEFAULT_QUERY,
-    output: path.join(ROOT, 'benchmarks', 'results'),
-    tasks: null,
-    allowCommands: false,
-    dryRun: false,
-  };
-
-  for (let index = 0; index < argv.length; index++) {
-    const arg = argv[index];
-    const next = () => {
-      const value = argv[++index];
-      if (!value || value.startsWith('--')) throw new Error(`${arg} requires a value`);
-      return value;
-    };
-    if (arg === '--count') options.count = Number(next());
-    else if (arg === '--max-iterations') options.maxIterations = Number(next());
-    else if (arg === '--model') options.model = next();
-    else if (arg === '--query') options.query = next();
-    else if (arg === '--output') options.output = path.resolve(next());
-    else if (arg === '--tasks') options.tasks = path.resolve(next());
-    else if (arg === '--allow-commands') options.allowCommands = true;
-    else if (arg === '--dry-run') options.dryRun = true;
-    else if (arg === '--help' || arg === '-h') options.help = true;
-    else throw new Error(`Unknown option: ${arg}`);
-  }
-
-  if (!Number.isInteger(options.count) || options.count < 1 || options.count > 100) {
-    throw new Error('--count must be an integer from 1 to 100');
-  }
-  if (!Number.isInteger(options.maxIterations) || options.maxIterations < 1 || options.maxIterations > 12) {
-    throw new Error('--max-iterations must be an integer from 1 to 12');
-  }
-  return options;
-}
-
-function printHelp() {
-  console.log(`ContextOS Gemini Issue Benchmark
-
-Usage:
-  npm run benchmark -- [options]
-
-Options:
-  --count <n>             Issues to discover (default: ${DEFAULT_COUNT})
-  --tasks <file>          Re-run a saved, reproducible task manifest
-  --query <github-query>  GitHub issue search query
-  --model <name>          Gemini model (default: ${DEFAULT_MODEL})
-  --max-iterations <n>    Maximum model turns per run (default: ${DEFAULT_MAX_ITERATIONS})
-  --output <directory>    Report directory (default: benchmarks/results)
-  --allow-commands        Required before cloning repos or running setup/tests
-  --dry-run               Discover and validate tasks, but do not call Gemini or clone
-
-Environment:
-  GEMINI_API_KEY          Required for benchmark runs
-  GITHUB_TOKEN            Strongly recommended; discovery of 20 linked issues exceeds anonymous API limits
-
-The benchmark writes an immutable task manifest containing issue URLs and base
-commits before it executes. Re-run that manifest with --tasks for comparable results.`);
-}
-
-function truncate(value, limit = MAX_OUTPUT_BYTES) {
-  const text = String(value || '');
-  return text.length <= limit ? text : `${text.slice(0, limit)}\n…[truncated]`;
-}
-
-function safeRelativePath(repository, requestedPath) {
-  if (typeof requestedPath !== 'string' || requestedPath.length === 0 || requestedPath.length > 240) return null;
-  const resolved = path.resolve(repository, requestedPath);
-  const relative = path.relative(repository, resolved);
-  if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) return null;
-  return resolved;
-}
-
-function readRequestedFiles(repository, requestedPaths) {
-  const files = {};
-  for (const requestedPath of (requestedPaths || []).slice(0, 8)) {
-    const resolved = safeRelativePath(repository, requestedPath);
-    if (!resolved || !fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) continue;
-    if (fs.statSync(resolved).size > MAX_FILE_BYTES) continue;
-    files[requestedPath] = fs.readFileSync(resolved, 'utf8');
-  }
-  return files;
-}
-
-function parseJsonObject(text) {
-  const clean = String(text || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
-  const first = clean.indexOf('{');
-  const last = clean.lastIndexOf('}');
-  if (first < 0 || last <= first) throw new Error('Model response did not contain a JSON object');
-  return JSON.parse(clean.slice(first, last + 1));
-}
-
-function parseAgentReply(text) {
-  const reply = parseJsonObject(text);
-  if (!Array.isArray(reply.read)) reply.read = [];
-  if (typeof reply.patch !== 'string') reply.patch = '';
-  if (typeof reply.ready !== 'boolean') reply.ready = false;
-  if (typeof reply.summary !== 'string') reply.summary = '';
-  if (reply.patch && !reply.patch.startsWith('diff --git ')) {
-    throw new Error('Model patch must be a unified git diff starting with "diff --git"');
-  }
-  return reply;
-}
-
-function run(command, args, options = {}) {
-  const { cwd, timeoutMs = 120_000, allowFailure = false } = options;
-  return new Promise((resolve, reject) => {
-    const child = spawn(command, args, { cwd, shell: false, windowsHide: true });
-    let stdout = '';
-    let stderr = '';
-    let timedOut = false;
-    const timer = setTimeout(() => {
-      timedOut = true;
-      child.kill('SIGTERM');
-    }, timeoutMs);
-    const append = (current, chunk) => truncate(current + chunk.toString(), MAX_OUTPUT_BYTES);
-    child.stdout.on('data', chunk => { stdout = append(stdout, chunk); });
-    child.stderr.on('data', chunk => { stderr = append(stderr, chunk); });
-    child.on('error', error => {
-      clearTimeout(timer);
-      reject(error);
-    });
-    child.on('close', code => {
-      clearTimeout(timer);
-      const result = { code, stdout, stderr, timedOut, passed: code === 0 && !timedOut };
-      if (!result.passed && !allowFailure) {
-        const error = new Error(`${command} exited with code ${code}${timedOut ? ' (timed out)' : ''}`);
-        error.result = result;
-        reject(error);
-      } else {
-        resolve(result);
-      }
-    });
-  });
-}
-
-function requestJson(url, { method = 'GET', headers = {}, body = null } = {}) {
-  return new Promise((resolve, reject) => {
-    const request = https.request(url, { method, headers }, response => {
-      let raw = '';
-      response.setEncoding('utf8');
-      response.on('data', chunk => { raw = truncate(raw + chunk, MAX_OUTPUT_BYTES); });
-      response.on('end', () => {
-        try {
-          resolve({ status: response.statusCode, body: JSON.parse(raw) });
-        } catch {
-          resolve({ status: response.statusCode, body: { message: raw } });
-        }
-      });
-    });
-    request.setTimeout(30_000, () => request.destroy(new Error('HTTP request timed out')));
-    request.on('error', reject);
-    if (body) request.write(body);
-    request.end();
-  });
-}
-
-async function githubJson(endpoint, token) {
-  const response = await requestJson(`https://api.github.com${endpoint}`, {
-    headers: {
-      Accept: 'application/vnd.github+json',
-      'X-GitHub-Api-Version': '2022-11-28',
-      'User-Agent': 'contextos-agents-benchmark',
-      ...(token ? { Authorization: `Bearer ${token}` } : {}),
-    },
-  });
-  if (response.status < 200 || response.status >= 300) {
-    throw new Error(`GitHub API ${response.status}: ${truncate(JSON.stringify(response.body), 500)}`);
-  }
-  return response.body;
-}
-
-function parseRepositoryFromApiUrl(repositoryUrl) {
-  const match = String(repositoryUrl).match(/\/repos\/([^/]+)\/([^/]+)$/);
-  if (!match) throw new Error(`Unexpected GitHub repository URL: ${repositoryUrl}`);
-  return { owner: match[1], repo: match[2] };
-}
-
-async function resolveIssueTask(issue, token) {
-  const { owner, repo } = parseRepositoryFromApiUrl(issue.repository_url);
-  const prefix = `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
-  const timeline = await githubJson(`${prefix}/issues/${issue.number}/timeline`, token);
-  const source = timeline.find(event => event.event === 'cross-referenced' && event.source?.issue?.pull_request)?.source?.issue;
-  if (!source?.number) return null;
-
-  const pull = await githubJson(`${prefix}/pulls/${source.number}`, token);
-  if (!pull.merged_at || !pull.merge_commit_sha) return null;
-  const mergeCommit = await githubJson(`${prefix}/commits/${pull.merge_commit_sha}`, token);
-  const baseSha = mergeCommit.parents?.[0]?.sha;
-  if (!baseSha) return null;
-
-  let packageJson;
-  try {
-    const content = await githubJson(`${prefix}/contents/package.json?ref=${encodeURIComponent(baseSha)}`, token);
-    packageJson = JSON.parse(Buffer.from(content.content, 'base64').toString('utf8'));
-  } catch {
-    return null;
-  }
-  if (!packageJson.scripts?.test) return null;
-
-  return {
-    id: `${owner}/${repo}#${issue.number}`,
-    repository: `${owner}/${repo}`,
-    issueNumber: issue.number,
-    issueUrl: issue.html_url,
-    title: issue.title,
-    body: issue.body || '',
-    baseSha,
-    linkedPullRequest: source.html_url,
-    setup: ['npm', 'ci', '--ignore-scripts'],
-    test: ['npm', 'test'],
-  };
-}
-
-async function discoverTasks(count, query, token) {
-  if (!token && count > 5) {
-    throw new Error('Set GITHUB_TOKEN to discover 20 tasks without GitHub API rate-limit failures.');
-  }
-  const tasks = [];
-  let page = 1;
-  while (tasks.length < count && page <= 10) {
-    const search = await githubJson(`/search/issues?q=${encodeURIComponent(query)}&per_page=100&page=${page}`, token);
-    const items = search.items || [];
-    if (items.length === 0) break;
-    for (const issue of items) {
-      if (tasks.length === count) break;
-      if (issue.pull_request) continue;
-      try {
-        const task = await resolveIssueTask(issue, token);
-        if (task) tasks.push(task);
-      } catch (error) {
-        console.warn(`[skip] ${issue.html_url}: ${error.message}`);
-      }
-    }
-    page++;
-  }
-  if (tasks.length < count) {
-    throw new Error(`Only resolved ${tasks.length}/${count} reproducible issues. Broaden --query and try again.`);
-  }
-  return tasks;
-}
-
-function writeJson(file, value) {
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
-}
-
-function readTaskManifest(file) {
-  const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
-  const tasks = Array.isArray(parsed) ? parsed : parsed.tasks;
-  if (!Array.isArray(tasks) || tasks.length === 0) throw new Error('Task manifest must contain a non-empty tasks array');
-  for (const task of tasks) {
-    if (!/^[\w.-]+\/[\w.-]+$/.test(task.repository || '') || !/^[a-f0-9]{40}$/i.test(task.baseSha || '')) {
-      throw new Error(`Invalid repository or baseSha in task: ${task.id || task.issueUrl || 'unknown'}`);
-    }
-    if (!Array.isArray(task.setup) || !Array.isArray(task.test) || !task.issueUrl) {
-      throw new Error(`Task ${task.id || task.issueUrl} lacks setup, test, or issueUrl`);
-    }
-  }
-  return tasks;
-}
-
-function loadSkillContext() {
-  return DEFAULT_SKILL_FILES
-    .filter(file => fs.existsSync(path.join(ROOT, file)))
-    .map(file => `\n===== ${file} =====\n${fs.readFileSync(path.join(ROOT, file), 'utf8')}`)
-    .join('\n');
-}
-
-function buildAgentPrompt(task, mode, initialFiles, feedback = '') {
-  const skills = mode === 'with_skills'
-    ? `\nContextOS skills are authoritative guidance for this run:\n${loadSkillContext()}`
-    : '';
-  return `You are repairing a real JavaScript repository. You have no shell or network access.\n\nIssue: ${task.title}\n${task.body}\nIssue URL: ${task.issueUrl}\n\nReturn exactly one JSON object, with no markdown fence:\n{"read":["relative/file.js"],"patch":"diff --git ... or empty string","ready":false,"summary":"short explanation"}\n\nRules:\n- Request at most 8 repository files per turn before editing.\n- Only return a valid unified git diff in patch; do not invent command output.\n- Do not change lockfiles, CI, generated files, or dependencies unless the issue requires it.\n- Set ready to true only after you have supplied all required changes.\n- The controller will apply your patch and return test output.\n\nInitial files:\n${JSON.stringify(initialFiles, null, 2)}${feedback ? `\n\nController feedback:\n${feedback}` : ''}${skills}`;
-}
-
-function extractText(payload) {
-  if (typeof payload?.output_text === 'string') return payload.output_text;
-  const chunks = [];
-  const visit = value => {
-    if (Array.isArray(value)) return value.forEach(visit);
-    if (!value || typeof value !== 'object') return;
-    if (typeof value.text === 'string' && (!value.type || /text/.test(value.type))) {
-      chunks.push(value.text);
-    }
-    for (const key of ['output', 'outputs', 'content', 'parts', 'candidates', 'message', 'delta']) {
-      if (value[key]) visit(value[key]);
-    }
-  };
-  visit(payload.output || payload.outputs || payload.candidates);
-  return chunks.join('\n');
-}
-
-async function callGemini(apiKey, model, input, systemInstruction) {
-  const started = Date.now();
-  const response = await requestJson('https://generativelanguage.googleapis.com/v1beta/interactions', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },
-    body: JSON.stringify({
-      model,
-      input,
-      system_instruction: systemInstruction,
-      generation_config: { temperature: 0 },
-    }),
-  });
-  const payload = response.body;
-  if (response.status < 200 || response.status >= 300) throw new Error(`Gemini API ${response.status}: ${truncate(JSON.stringify(payload), 800)}`);
-  const text = extractText(payload);
-  if (!text) throw new Error('Gemini returned no text output');
-  return { text, latencyMs: Date.now() - started, usage: payload.usage_metadata || payload.usageMetadata || null };
-}
-
-async function gitOutput(repository, args) {
-  const result = await run('git', args, { cwd: repository, allowFailure: true });
-  return result.stdout;
-}
-
-async function cloneTask(task, workspace) {
-  const destination = path.join(workspace, task.id.replace(/[^a-zA-Z0-9._-]/g, '-'));
-  await run('git', ['clone', '--filter=blob:none', '--no-checkout', `https://github.com/${task.repository}.git`, destination], { cwd: workspace, timeoutMs: 300_000 });
-  await run('git', ['checkout', '--detach', task.baseSha], { cwd: destination, timeoutMs: 120_000 });
-  return destination;
-}
-
-async function applyPatch(repository, patch) {
-  const patchFile = path.join(os.tmpdir(), `contextos-benchmark-${crypto.randomUUID()}.patch`);
-  fs.writeFileSync(patchFile, patch);
-  try {
-    return await run('git', ['apply', '--whitespace=nowarn', patchFile], { cwd: repository, allowFailure: true });
-  } finally {
-    fs.rmSync(patchFile, { force: true });
-  }
-}
-
-async function runTaskCommand(repository, command, timeoutMs = 300_000) {
-  if (!Array.isArray(command) || command.length === 0) throw new Error('Task command must be a non-empty argument array');
-  return run(command[0], command.slice(1), { cwd: repository, timeoutMs, allowFailure: true });
-}
-
-async function runAgent(task, mode, options, workspace) {
-  const repository = await cloneTask(task, workspace);
-  const setup = await runTaskCommand(repository, task.setup);
-  if (!setup.passed) return { task, mode, status: 'setup_failed', setup, iterations: 0, score: 0 };
-
-  const initialFiles = readRequestedFiles(repository, ['package.json', 'README.md']);
-  let feedback = '';
-  let lastTest = null;
-  let lastReply = null;
-  const turns = [];
-  for (let iteration = 1; iteration <= options.maxIterations; iteration++) {
-    const response = await callGemini(
-      options.apiKey,
-      options.model,
-      buildAgentPrompt(task, mode, initialFiles, feedback),
-      'You are a careful software engineer. Follow the controller protocol exactly.'
-    );
-    let reply;
-    try {
-      reply = parseAgentReply(response.text);
-    } catch (error) {
-      return { task, mode, status: 'invalid_model_response', error: error.message, iterations: iteration, turns, score: 0 };
-    }
-    lastReply = reply;
-    const files = readRequestedFiles(repository, reply.read);
-    const turn = { iteration, latencyMs: response.latencyMs, usage: response.usage, requestedFiles: Object.keys(files), summary: reply.summary };
-
-    if (reply.patch) {
-      const applied = await applyPatch(repository, reply.patch);
-      turn.patchApplied = applied.passed;
-      if (!applied.passed) {
-        feedback = `Patch was rejected:\n${truncate(applied.stderr || applied.stdout)}`;
-      } else {
-        lastTest = await runTaskCommand(repository, task.test);
-        turn.test = lastTest;
-        feedback = `Patch applied. Test result (exit ${lastTest.code}):\n${truncate(`${lastTest.stdout}\n${lastTest.stderr}`)}`;
-      }
-    } else if (Object.keys(files).length > 0) {
-      feedback = `Requested file contents:\n${JSON.stringify(files, null, 2)}`;
-    } else {
-      feedback = 'No patch or readable file request was supplied. Return a valid next action.';
-    }
-    turns.push(turn);
-
-    const diff = await gitOutput(repository, ['diff', '--no-ext-diff', '--unified=3']);
-    if (reply.ready && lastTest?.passed && diff.trim()) break;
-  }
-
-  const diff = await gitOutput(repository, ['diff', '--no-ext-diff', '--unified=3']);
-  const filesChanged = (await gitOutput(repository, ['diff', '--name-only'])).trim().split('\n').filter(Boolean);
-  const ready = Boolean(lastReply?.ready && lastTest?.passed && diff.trim());
-  const judge = await judgeChange(task, diff, lastTest, options);
-  const score = scoreRun({ ready, test: lastTest, filesChanged, judge, iterations: turns.length });
-  return { task, mode, status: ready ? 'ready' : 'not_ready', setup, test: lastTest, iterations: turns.length, turns, filesChanged, diff, judge, score };
-}
-
-async function judgeChange(task, diff, test, options) {
-  if (!diff.trim()) return { score: 0, reason: 'No source change was produced.' };
-  const prompt = `Evaluate a proposed fix for this GitHub Issue. Score only issue fidelity and code quality from 0 to 30. Do not reward tests merely passing; explain missing behavior. Return exactly JSON: {"score":number,"reason":"short"}.\n\nIssue: ${task.title}\n${task.body}\n\nDiff:\n${truncate(diff, 18_000)}\n\nTest output:\n${truncate(`${test?.stdout || ''}\n${test?.stderr || ''}`, 4_000)}`;
-  try {
-    const response = await callGemini(options.apiKey, options.model, prompt, 'You are an impartial senior code reviewer. Return JSON only.');
-    const result = parseJsonObject(response.text);
-    return { score: Math.max(0, Math.min(30, Number(result.score) || 0)), reason: String(result.reason || ''), latencyMs: response.latencyMs };
-  } catch (error) {
-    return { score: 0, reason: `Judge unavailable: ${error.message}` };
-  }
-}
-
-function scoreRun({ ready, test, filesChanged, judge, iterations }) {
-  const testScore = test?.passed ? 45 : 0;
-  const scopeScore = filesChanged.length > 0 && filesChanged.length <= 6 ? 15 : filesChanged.length <= 12 ? 8 : 0;
-  const safetyScore = filesChanged.every(file => !/(^|\/)(\.env|node_modules|\.github\/workflows)\b|(?:package-lock|yarn\.lock|pnpm-lock)/.test(file)) ? 10 : 0;
-  const iterationScore = ready ? Math.max(0, 10 - Math.max(0, iterations - 1) * 2) : 0;
-  return { total: testScore + scopeScore + safetyScore + iterationScore + (judge?.score || 0), testScore, scopeScore, safetyScore, iterationScore, judgeScore: judge?.score || 0 };
-}
-
-function average(values) {
-  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
-}
-
-function summarize(results) {
-  const byMode = {};
-  for (const mode of ['without_skills', 'with_skills']) {
-    const runs = results.filter(result => result.mode === mode);
-    byMode[mode] = {
-      runs: runs.length,
-      readyRate: runs.length ? runs.filter(run => run.status === 'ready').length / runs.length : 0,
-      testPassRate: runs.length ? runs.filter(run => run.test?.passed).length / runs.length : 0,
-      averageScore: average(runs.map(run => run.score?.total || 0)),
-      averageIterations: average(runs.map(run => run.iterations || 0)),
-    };
-  }
-  const paired = results.reduce((map, result) => {
-    const entry = map.get(result.task.id) || {};
-    entry[result.mode] = result;
-    map.set(result.task.id, entry);
-    return map;
-  }, new Map());
-  const deltas = [...paired.values()]
-    .filter(pair => pair.without_skills && pair.with_skills)
-    .map(pair => pair.with_skills.score.total - pair.without_skills.score.total);
-  return { byMode, pairedTasks: deltas.length, meanSkillScoreDelta: average(deltas) };
-}
-
-function markdownReport(report) {
-  const row = (name, values) => `| ${name} | ${values.runs} | ${(values.readyRate * 100).toFixed(1)}% | ${(values.testPassRate * 100).toFixed(1)}% | ${values.averageScore.toFixed(1)} | ${values.averageIterations.toFixed(2)} |`;
-  return `# ContextOS Skills Benchmark\n\nModel: \`${report.model}\`  \nTasks: ${report.tasks.length}  \nPaired tasks: ${report.summary.pairedTasks}\n\n| Mode | Runs | Ready | Tests pass | Mean quality (0–110) | Mean turns |\n| --- | ---: | ---: | ---: | ---: | ---: |\n${row('Without skills', report.summary.byMode.without_skills)}\n${row('With ContextOS skills', report.summary.byMode.with_skills)}\n\nMean paired score delta (with skills − without): **${report.summary.meanSkillScoreDelta.toFixed(1)}**\n\n## Per task\n\n| Issue | Mode | Status | Turns | Score |\n| --- | --- | --- | ---: | ---: |\n${report.results.map(run => `| [${run.task.id}](${run.task.issueUrl}) | ${run.mode} | ${run.status} | ${run.iterations} | ${run.score?.total || 0} |`).join('\n')}\n`;
-}
-
-async function main() {
-  const options = parseArgs(process.argv.slice(2));
-  if (options.help) return printHelp();
-  const token = process.env.GITHUB_TOKEN;
-  const tasks = options.tasks ? readTaskManifest(options.tasks) : await discoverTasks(options.count, options.query, token);
-  const manifestPath = options.tasks || path.join(options.output, `tasks-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
-  if (!options.tasks) writeJson(manifestPath, { generatedAt: new Date().toISOString(), query: options.query, tasks });
-  console.log(`Prepared ${tasks.length} reproducible GitHub Issue tasks: ${manifestPath}`);
-
-  if (options.dryRun || !options.allowCommands) {
-    console.log(options.dryRun ? 'Dry run complete; no model calls or repositories were executed.' : 'Pass --allow-commands to clone task repositories and run their declared setup/tests.');
-    return;
-  }
-  if (!process.env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY is required for a benchmark run');
-
-  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'contextos-benchmark-'));
-  try {
-    const results = [];
-    for (const task of tasks) {
-      for (const mode of ['without_skills', 'with_skills']) {
-        console.log(`\n[${mode}] ${task.id}`);
-        results.push(await runAgent(task, mode, { ...options, apiKey: process.env.GEMINI_API_KEY }, workspace));
-      }
-    }
-    const report = { generatedAt: new Date().toISOString(), model: options.model, tasks, results, summary: summarize(results) };
-    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
-    writeJson(path.join(options.output, `report-${stamp}.json`), report);
-    fs.writeFileSync(path.join(options.output, `report-${stamp}.md`), markdownReport(report));
-    console.log(`\nBenchmark complete. Reports written to ${options.output}`);
-  } finally {
-    fs.rmSync(workspace, { recursive: true, force: true });
-  }
-}
-
-module.exports = { buildAgentPrompt, parseAgentReply, parseArgs, scoreRun, summarize, safeRelativePath };
-
-if (require.main === module) {
-  main().catch(error => {
-    console.error(`[ERROR] ${error.message}`);
-    process.exit(1);
-  });
-}