contextos-agents 1.5.0

package/benchmarks/gemini-issues.js

@@ -0,0 +1,533 @@
+#!/usr/bin/env node
+
+/*
+ * A reproducible, paired benchmark for ContextOS skills.
+ *
+ * It discovers closed JavaScript GitHub Issues linked to merged pull requests,
+ * checks out the parent of each merge commit, and gives the same task to Gemini
+ * twice: once with a neutral instruction and once with ContextOS skills. The
+ * coding loop is intentionally controller-driven: the model may read files and
+ * return a unified diff, but it never receives shell access.
+ */
+
+'use strict';
+
+const crypto = require('crypto');
+const fs = require('fs');
+const https = require('https');
+const os = require('os');
+const path = require('path');
+const { spawn } = require('child_process');
+
+// The benchmark belongs to the package but operates on the project from which
+// it is invoked, including that project's installed .agents directory.
+const ROOT = process.cwd();
+const DEFAULT_COUNT = 20;
+const DEFAULT_MAX_ITERATIONS = 6;
+const DEFAULT_MODEL = 'gemini-2.5-flash';
+const DEFAULT_QUERY = 'is:issue is:closed language:JavaScript label:bug';
+const MAX_FILE_BYTES = 24 * 1024;
+const MAX_OUTPUT_BYTES = 16 * 1024;
+const DEFAULT_SKILL_FILES = [
+  '.agents/AGENTS.md',
+  '.agents/core/skills/engineering-workflow/SKILL.md',
+  '.agents/core/skills/system-design/SKILL.md',
+  '.agents/core/skills/node/SKILL.md',
+  '.agents/core/skills/security/SKILL.md',
+  '.agents/core/skills/typescript/SKILL.md',
+];
+
+function parseArgs(argv) {
+  const options = {
+    count: DEFAULT_COUNT,
+    maxIterations: DEFAULT_MAX_ITERATIONS,
+    model: DEFAULT_MODEL,
+    query: DEFAULT_QUERY,
+    output: path.join(ROOT, 'benchmarks', 'results'),
+    tasks: null,
+    allowCommands: false,
+    dryRun: false,
+  };
+
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+    const next = () => {
+      const value = argv[++index];
+      if (!value || value.startsWith('--')) throw new Error(`${arg} requires a value`);
+      return value;
+    };
+    if (arg === '--count') options.count = Number(next());
+    else if (arg === '--max-iterations') options.maxIterations = Number(next());
+    else if (arg === '--model') options.model = next();
+    else if (arg === '--query') options.query = next();
+    else if (arg === '--output') options.output = path.resolve(next());
+    else if (arg === '--tasks') options.tasks = path.resolve(next());
+    else if (arg === '--allow-commands') options.allowCommands = true;
+    else if (arg === '--dry-run') options.dryRun = true;
+    else if (arg === '--help' || arg === '-h') options.help = true;
+    else throw new Error(`Unknown option: ${arg}`);
+  }
+
+  if (!Number.isInteger(options.count) || options.count < 1 || options.count > 100) {
+    throw new Error('--count must be an integer from 1 to 100');
+  }
+  if (!Number.isInteger(options.maxIterations) || options.maxIterations < 1 || options.maxIterations > 12) {
+    throw new Error('--max-iterations must be an integer from 1 to 12');
+  }
+  return options;
+}
+
+function printHelp() {
+  console.log(`ContextOS Gemini Issue Benchmark
+
+Usage:
+  npm run benchmark -- [options]
+
+Options:
+  --count <n>             Issues to discover (default: ${DEFAULT_COUNT})
+  --tasks <file>          Re-run a saved, reproducible task manifest
+  --query <github-query>  GitHub issue search query
+  --model <name>          Gemini model (default: ${DEFAULT_MODEL})
+  --max-iterations <n>    Maximum model turns per run (default: ${DEFAULT_MAX_ITERATIONS})
+  --output <directory>    Report directory (default: benchmarks/results)
+  --allow-commands        Required before cloning repos or running setup/tests
+  --dry-run               Discover and validate tasks, but do not call Gemini or clone
+
+Environment:
+  GEMINI_API_KEY          Required for benchmark runs
+  GITHUB_TOKEN            Strongly recommended; discovery of 20 linked issues exceeds anonymous API limits
+
+The benchmark writes an immutable task manifest containing issue URLs and base
+commits before it executes. Re-run that manifest with --tasks for comparable results.`);
+}
+
+function truncate(value, limit = MAX_OUTPUT_BYTES) {
+  const text = String(value || '');
+  return text.length <= limit ? text : `${text.slice(0, limit)}\n…[truncated]`;
+}
+
+function safeRelativePath(repository, requestedPath) {
+  if (typeof requestedPath !== 'string' || requestedPath.length === 0 || requestedPath.length > 240) return null;
+  const resolved = path.resolve(repository, requestedPath);
+  const relative = path.relative(repository, resolved);
+  if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) return null;
+  return resolved;
+}
+
+function readRequestedFiles(repository, requestedPaths) {
+  const files = {};
+  for (const requestedPath of (requestedPaths || []).slice(0, 8)) {
+    const resolved = safeRelativePath(repository, requestedPath);
+    if (!resolved || !fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) continue;
+    if (fs.statSync(resolved).size > MAX_FILE_BYTES) continue;
+    files[requestedPath] = fs.readFileSync(resolved, 'utf8');
+  }
+  return files;
+}
+
+function parseJsonObject(text) {
+  const clean = String(text || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
+  const first = clean.indexOf('{');
+  const last = clean.lastIndexOf('}');
+  if (first < 0 || last <= first) throw new Error('Model response did not contain a JSON object');
+  return JSON.parse(clean.slice(first, last + 1));
+}
+
+function parseAgentReply(text) {
+  const reply = parseJsonObject(text);
+  if (!Array.isArray(reply.read)) reply.read = [];
+  if (typeof reply.patch !== 'string') reply.patch = '';
+  if (typeof reply.ready !== 'boolean') reply.ready = false;
+  if (typeof reply.summary !== 'string') reply.summary = '';
+  if (reply.patch && !reply.patch.startsWith('diff --git ')) {
+    throw new Error('Model patch must be a unified git diff starting with "diff --git"');
+  }
+  return reply;
+}
+
+function run(command, args, options = {}) {
+  const { cwd, timeoutMs = 120_000, allowFailure = false } = options;
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { cwd, shell: false, windowsHide: true });
+    let stdout = '';
+    let stderr = '';
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill('SIGTERM');
+    }, timeoutMs);
+    const append = (current, chunk) => truncate(current + chunk.toString(), MAX_OUTPUT_BYTES);
+    child.stdout.on('data', chunk => { stdout = append(stdout, chunk); });
+    child.stderr.on('data', chunk => { stderr = append(stderr, chunk); });
+    child.on('error', error => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.on('close', code => {
+      clearTimeout(timer);
+      const result = { code, stdout, stderr, timedOut, passed: code === 0 && !timedOut };
+      if (!result.passed && !allowFailure) {
+        const error = new Error(`${command} exited with code ${code}${timedOut ? ' (timed out)' : ''}`);
+        error.result = result;
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    });
+  });
+}
+
+function requestJson(url, { method = 'GET', headers = {}, body = null } = {}) {
+  return new Promise((resolve, reject) => {
+    const request = https.request(url, { method, headers }, response => {
+      let raw = '';
+      response.setEncoding('utf8');
+      response.on('data', chunk => { raw = truncate(raw + chunk, MAX_OUTPUT_BYTES); });
+      response.on('end', () => {
+        try {
+          resolve({ status: response.statusCode, body: JSON.parse(raw) });
+        } catch {
+          resolve({ status: response.statusCode, body: { message: raw } });
+        }
+      });
+    });
+    request.setTimeout(30_000, () => request.destroy(new Error('HTTP request timed out')));
+    request.on('error', reject);
+    if (body) request.write(body);
+    request.end();
+  });
+}
+
+async function githubJson(endpoint, token) {
+  const response = await requestJson(`https://api.github.com${endpoint}`, {
+    headers: {
+      Accept: 'application/vnd.github+json',
+      'X-GitHub-Api-Version': '2022-11-28',
+      'User-Agent': 'contextos-agents-benchmark',
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    },
+  });
+  if (response.status < 200 || response.status >= 300) {
+    throw new Error(`GitHub API ${response.status}: ${truncate(JSON.stringify(response.body), 500)}`);
+  }
+  return response.body;
+}
+
+function parseRepositoryFromApiUrl(repositoryUrl) {
+  const match = String(repositoryUrl).match(/\/repos\/([^/]+)\/([^/]+)$/);
+  if (!match) throw new Error(`Unexpected GitHub repository URL: ${repositoryUrl}`);
+  return { owner: match[1], repo: match[2] };
+}
+
+async function resolveIssueTask(issue, token) {
+  const { owner, repo } = parseRepositoryFromApiUrl(issue.repository_url);
+  const prefix = `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
+  const timeline = await githubJson(`${prefix}/issues/${issue.number}/timeline`, token);
+  const source = timeline.find(event => event.event === 'cross-referenced' && event.source?.issue?.pull_request)?.source?.issue;
+  if (!source?.number) return null;
+
+  const pull = await githubJson(`${prefix}/pulls/${source.number}`, token);
+  if (!pull.merged_at || !pull.merge_commit_sha) return null;
+  const mergeCommit = await githubJson(`${prefix}/commits/${pull.merge_commit_sha}`, token);
+  const baseSha = mergeCommit.parents?.[0]?.sha;
+  if (!baseSha) return null;
+
+  let packageJson;
+  try {
+    const content = await githubJson(`${prefix}/contents/package.json?ref=${encodeURIComponent(baseSha)}`, token);
+    packageJson = JSON.parse(Buffer.from(content.content, 'base64').toString('utf8'));
+  } catch {
+    return null;
+  }
+  if (!packageJson.scripts?.test) return null;
+
+  return {
+    id: `${owner}/${repo}#${issue.number}`,
+    repository: `${owner}/${repo}`,
+    issueNumber: issue.number,
+    issueUrl: issue.html_url,
+    title: issue.title,
+    body: issue.body || '',
+    baseSha,
+    linkedPullRequest: source.html_url,
+    setup: ['npm', 'ci', '--ignore-scripts'],
+    test: ['npm', 'test'],
+  };
+}
+
+async function discoverTasks(count, query, token) {
+  if (!token && count > 5) {
+    throw new Error('Set GITHUB_TOKEN to discover 20 tasks without GitHub API rate-limit failures.');
+  }
+  const tasks = [];
+  let page = 1;
+  while (tasks.length < count && page <= 10) {
+    const search = await githubJson(`/search/issues?q=${encodeURIComponent(query)}&per_page=100&page=${page}`, token);
+    const items = search.items || [];
+    if (items.length === 0) break;
+    for (const issue of items) {
+      if (tasks.length === count) break;
+      if (issue.pull_request) continue;
+      try {
+        const task = await resolveIssueTask(issue, token);
+        if (task) tasks.push(task);
+      } catch (error) {
+        console.warn(`[skip] ${issue.html_url}: ${error.message}`);
+      }
+    }
+    page++;
+  }
+  if (tasks.length < count) {
+    throw new Error(`Only resolved ${tasks.length}/${count} reproducible issues. Broaden --query and try again.`);
+  }
+  return tasks;
+}
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function readTaskManifest(file) {
+  const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+  const tasks = Array.isArray(parsed) ? parsed : parsed.tasks;
+  if (!Array.isArray(tasks) || tasks.length === 0) throw new Error('Task manifest must contain a non-empty tasks array');
+  for (const task of tasks) {
+    if (!/^[\w.-]+\/[\w.-]+$/.test(task.repository || '') || !/^[a-f0-9]{40}$/i.test(task.baseSha || '')) {
+      throw new Error(`Invalid repository or baseSha in task: ${task.id || task.issueUrl || 'unknown'}`);
+    }
+    if (!Array.isArray(task.setup) || !Array.isArray(task.test) || !task.issueUrl) {
+      throw new Error(`Task ${task.id || task.issueUrl} lacks setup, test, or issueUrl`);
+    }
+  }
+  return tasks;
+}
+
+function loadSkillContext() {
+  return DEFAULT_SKILL_FILES
+    .filter(file => fs.existsSync(path.join(ROOT, file)))
+    .map(file => `\n===== ${file} =====\n${fs.readFileSync(path.join(ROOT, file), 'utf8')}`)
+    .join('\n');
+}
+
+function buildAgentPrompt(task, mode, initialFiles, feedback = '') {
+  const skills = mode === 'with_skills'
+    ? `\nContextOS skills are authoritative guidance for this run:\n${loadSkillContext()}`
+    : '';
+  return `You are repairing a real JavaScript repository. You have no shell or network access.\n\nIssue: ${task.title}\n${task.body}\nIssue URL: ${task.issueUrl}\n\nReturn exactly one JSON object, with no markdown fence:\n{"read":["relative/file.js"],"patch":"diff --git ... or empty string","ready":false,"summary":"short explanation"}\n\nRules:\n- Request at most 8 repository files per turn before editing.\n- Only return a valid unified git diff in patch; do not invent command output.\n- Do not change lockfiles, CI, generated files, or dependencies unless the issue requires it.\n- Set ready to true only after you have supplied all required changes.\n- The controller will apply your patch and return test output.\n\nInitial files:\n${JSON.stringify(initialFiles, null, 2)}${feedback ? `\n\nController feedback:\n${feedback}` : ''}${skills}`;
+}
+
+function extractText(payload) {
+  if (typeof payload?.output_text === 'string') return payload.output_text;
+  const chunks = [];
+  const visit = value => {
+    if (Array.isArray(value)) return value.forEach(visit);
+    if (!value || typeof value !== 'object') return;
+    if (typeof value.text === 'string' && (!value.type || /text/.test(value.type))) {
+      chunks.push(value.text);
+    }
+    for (const key of ['output', 'outputs', 'content', 'parts', 'candidates', 'message', 'delta']) {
+      if (value[key]) visit(value[key]);
+    }
+  };
+  visit(payload.output || payload.outputs || payload.candidates);
+  return chunks.join('\n');
+}
+
+async function callGemini(apiKey, model, input, systemInstruction) {
+  const started = Date.now();
+  const response = await requestJson('https://generativelanguage.googleapis.com/v1beta/interactions', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },
+    body: JSON.stringify({
+      model,
+      input,
+      system_instruction: systemInstruction,
+      generation_config: { temperature: 0 },
+    }),
+  });
+  const payload = response.body;
+  if (response.status < 200 || response.status >= 300) throw new Error(`Gemini API ${response.status}: ${truncate(JSON.stringify(payload), 800)}`);
+  const text = extractText(payload);
+  if (!text) throw new Error('Gemini returned no text output');
+  return { text, latencyMs: Date.now() - started, usage: payload.usage_metadata || payload.usageMetadata || null };
+}
+
+async function gitOutput(repository, args) {
+  const result = await run('git', args, { cwd: repository, allowFailure: true });
+  return result.stdout;
+}
+
+async function cloneTask(task, workspace) {
+  const destination = path.join(workspace, task.id.replace(/[^a-zA-Z0-9._-]/g, '-'));
+  await run('git', ['clone', '--filter=blob:none', '--no-checkout', `https://github.com/${task.repository}.git`, destination], { cwd: workspace, timeoutMs: 300_000 });
+  await run('git', ['checkout', '--detach', task.baseSha], { cwd: destination, timeoutMs: 120_000 });
+  return destination;
+}
+
+async function applyPatch(repository, patch) {
+  const patchFile = path.join(os.tmpdir(), `contextos-benchmark-${crypto.randomUUID()}.patch`);
+  fs.writeFileSync(patchFile, patch);
+  try {
+    return await run('git', ['apply', '--whitespace=nowarn', patchFile], { cwd: repository, allowFailure: true });
+  } finally {
+    fs.rmSync(patchFile, { force: true });
+  }
+}
+
+async function runTaskCommand(repository, command, timeoutMs = 300_000) {
+  if (!Array.isArray(command) || command.length === 0) throw new Error('Task command must be a non-empty argument array');
+  return run(command[0], command.slice(1), { cwd: repository, timeoutMs, allowFailure: true });
+}
+
+async function runAgent(task, mode, options, workspace) {
+  const repository = await cloneTask(task, workspace);
+  const setup = await runTaskCommand(repository, task.setup);
+  if (!setup.passed) return { task, mode, status: 'setup_failed', setup, iterations: 0, score: 0 };
+
+  const initialFiles = readRequestedFiles(repository, ['package.json', 'README.md']);
+  let feedback = '';
+  let lastTest = null;
+  let lastReply = null;
+  const turns = [];
+  for (let iteration = 1; iteration <= options.maxIterations; iteration++) {
+    const response = await callGemini(
+      options.apiKey,
+      options.model,
+      buildAgentPrompt(task, mode, initialFiles, feedback),
+      'You are a careful software engineer. Follow the controller protocol exactly.'
+    );
+    let reply;
+    try {
+      reply = parseAgentReply(response.text);
+    } catch (error) {
+      return { task, mode, status: 'invalid_model_response', error: error.message, iterations: iteration, turns, score: 0 };
+    }
+    lastReply = reply;
+    const files = readRequestedFiles(repository, reply.read);
+    const turn = { iteration, latencyMs: response.latencyMs, usage: response.usage, requestedFiles: Object.keys(files), summary: reply.summary };
+
+    if (reply.patch) {
+      const applied = await applyPatch(repository, reply.patch);
+      turn.patchApplied = applied.passed;
+      if (!applied.passed) {
+        feedback = `Patch was rejected:\n${truncate(applied.stderr || applied.stdout)}`;
+      } else {
+        lastTest = await runTaskCommand(repository, task.test);
+        turn.test = lastTest;
+        feedback = `Patch applied. Test result (exit ${lastTest.code}):\n${truncate(`${lastTest.stdout}\n${lastTest.stderr}`)}`;
+      }
+    } else if (Object.keys(files).length > 0) {
+      feedback = `Requested file contents:\n${JSON.stringify(files, null, 2)}`;
+    } else {
+      feedback = 'No patch or readable file request was supplied. Return a valid next action.';
+    }
+    turns.push(turn);
+
+    const diff = await gitOutput(repository, ['diff', '--no-ext-diff', '--unified=3']);
+    if (reply.ready && lastTest?.passed && diff.trim()) break;
+  }
+
+  const diff = await gitOutput(repository, ['diff', '--no-ext-diff', '--unified=3']);
+  const filesChanged = (await gitOutput(repository, ['diff', '--name-only'])).trim().split('\n').filter(Boolean);
+  const ready = Boolean(lastReply?.ready && lastTest?.passed && diff.trim());
+  const judge = await judgeChange(task, diff, lastTest, options);
+  const score = scoreRun({ ready, test: lastTest, filesChanged, judge, iterations: turns.length });
+  return { task, mode, status: ready ? 'ready' : 'not_ready', setup, test: lastTest, iterations: turns.length, turns, filesChanged, diff, judge, score };
+}
+
+async function judgeChange(task, diff, test, options) {
+  if (!diff.trim()) return { score: 0, reason: 'No source change was produced.' };
+  const prompt = `Evaluate a proposed fix for this GitHub Issue. Score only issue fidelity and code quality from 0 to 30. Do not reward tests merely passing; explain missing behavior. Return exactly JSON: {"score":number,"reason":"short"}.\n\nIssue: ${task.title}\n${task.body}\n\nDiff:\n${truncate(diff, 18_000)}\n\nTest output:\n${truncate(`${test?.stdout || ''}\n${test?.stderr || ''}`, 4_000)}`;
+  try {
+    const response = await callGemini(options.apiKey, options.model, prompt, 'You are an impartial senior code reviewer. Return JSON only.');
+    const result = parseJsonObject(response.text);
+    return { score: Math.max(0, Math.min(30, Number(result.score) || 0)), reason: String(result.reason || ''), latencyMs: response.latencyMs };
+  } catch (error) {
+    return { score: 0, reason: `Judge unavailable: ${error.message}` };
+  }
+}
+
+function scoreRun({ ready, test, filesChanged, judge, iterations }) {
+  const testScore = test?.passed ? 45 : 0;
+  const scopeScore = filesChanged.length > 0 && filesChanged.length <= 6 ? 15 : filesChanged.length <= 12 ? 8 : 0;
+  const safetyScore = filesChanged.every(file => !/(^|\/)(\.env|node_modules|\.github\/workflows)\b|(?:package-lock|yarn\.lock|pnpm-lock)/.test(file)) ? 10 : 0;
+  const iterationScore = ready ? Math.max(0, 10 - Math.max(0, iterations - 1) * 2) : 0;
+  return { total: testScore + scopeScore + safetyScore + iterationScore + (judge?.score || 0), testScore, scopeScore, safetyScore, iterationScore, judgeScore: judge?.score || 0 };
+}
+
+function average(values) {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+
+function summarize(results) {
+  const byMode = {};
+  for (const mode of ['without_skills', 'with_skills']) {
+    const runs = results.filter(result => result.mode === mode);
+    byMode[mode] = {
+      runs: runs.length,
+      readyRate: runs.length ? runs.filter(run => run.status === 'ready').length / runs.length : 0,
+      testPassRate: runs.length ? runs.filter(run => run.test?.passed).length / runs.length : 0,
+      averageScore: average(runs.map(run => run.score?.total || 0)),
+      averageIterations: average(runs.map(run => run.iterations || 0)),
+    };
+  }
+  const paired = results.reduce((map, result) => {
+    const entry = map.get(result.task.id) || {};
+    entry[result.mode] = result;
+    map.set(result.task.id, entry);
+    return map;
+  }, new Map());
+  const deltas = [...paired.values()]
+    .filter(pair => pair.without_skills && pair.with_skills)
+    .map(pair => pair.with_skills.score.total - pair.without_skills.score.total);
+  return { byMode, pairedTasks: deltas.length, meanSkillScoreDelta: average(deltas) };
+}
+
+function markdownReport(report) {
+  const row = (name, values) => `| ${name} | ${values.runs} | ${(values.readyRate * 100).toFixed(1)}% | ${(values.testPassRate * 100).toFixed(1)}% | ${values.averageScore.toFixed(1)} | ${values.averageIterations.toFixed(2)} |`;
+  return `# ContextOS Skills Benchmark\n\nModel: \`${report.model}\`  \nTasks: ${report.tasks.length}  \nPaired tasks: ${report.summary.pairedTasks}\n\n| Mode | Runs | Ready | Tests pass | Mean quality (0–110) | Mean turns |\n| --- | ---: | ---: | ---: | ---: | ---: |\n${row('Without skills', report.summary.byMode.without_skills)}\n${row('With ContextOS skills', report.summary.byMode.with_skills)}\n\nMean paired score delta (with skills − without): **${report.summary.meanSkillScoreDelta.toFixed(1)}**\n\n## Per task\n\n| Issue | Mode | Status | Turns | Score |\n| --- | --- | --- | ---: | ---: |\n${report.results.map(run => `| [${run.task.id}](${run.task.issueUrl}) | ${run.mode} | ${run.status} | ${run.iterations} | ${run.score?.total || 0} |`).join('\n')}\n`;
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) return printHelp();
+  const token = process.env.GITHUB_TOKEN;
+  const tasks = options.tasks ? readTaskManifest(options.tasks) : await discoverTasks(options.count, options.query, token);
+  const manifestPath = options.tasks || path.join(options.output, `tasks-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
+  if (!options.tasks) writeJson(manifestPath, { generatedAt: new Date().toISOString(), query: options.query, tasks });
+  console.log(`Prepared ${tasks.length} reproducible GitHub Issue tasks: ${manifestPath}`);
+
+  if (options.dryRun || !options.allowCommands) {
+    console.log(options.dryRun ? 'Dry run complete; no model calls or repositories were executed.' : 'Pass --allow-commands to clone task repositories and run their declared setup/tests.');
+    return;
+  }
+  if (!process.env.GEMINI_API_KEY) throw new Error('GEMINI_API_KEY is required for a benchmark run');
+
+  const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'contextos-benchmark-'));
+  try {
+    const results = [];
+    for (const task of tasks) {
+      for (const mode of ['without_skills', 'with_skills']) {
+        console.log(`\n[${mode}] ${task.id}`);
+        results.push(await runAgent(task, mode, { ...options, apiKey: process.env.GEMINI_API_KEY }, workspace));
+      }
+    }
+    const report = { generatedAt: new Date().toISOString(), model: options.model, tasks, results, summary: summarize(results) };
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    writeJson(path.join(options.output, `report-${stamp}.json`), report);
+    fs.writeFileSync(path.join(options.output, `report-${stamp}.md`), markdownReport(report));
+    console.log(`\nBenchmark complete. Reports written to ${options.output}`);
+  } finally {
+    fs.rmSync(workspace, { recursive: true, force: true });
+  }
+}
+
+module.exports = { buildAgentPrompt, parseAgentReply, parseArgs, scoreRun, summarize, safeRelativePath };
+
+if (require.main === module) {
+  main().catch(error => {
+    console.error(`[ERROR] ${error.message}`);
+    process.exit(1);
+  });
+}