cgraphx 1.2.0 → 1.3.0

package/scripts/agent-eval/parse-multi-tool.mjs

@@ -1,242 +0,0 @@
-#!/usr/bin/env node
-// Parse multi-tool-eval transcripts (3 arms × 3 tasks × n reps).
-//
-// File-name contract: <repo>-<task>-<arm>-<rep>.jsonl
-//   task ∈ {flow, onboarding, impact}
-//   arm  ∈ {cgraphx-cli, cgraphx-mcp, gitnexus-mcp}
-//
-// Output:
-//   1. <out-dir>/results.jsonl — one JSON line per transcript
-//   2. stdout:
-//      - PER TASK × ARM × REP (detail)
-//      - AGGREGATE PER TASK × ARM (mean across reps)
-//      - AGGREGATE PER ARM (mean across task × reps — overall tool ranking)
-import { readFileSync, readdirSync, existsSync, writeFileSync } from 'fs';
-import { join } from 'path';
-
-const OUT = process.argv[2] || '/tmp/cg-multi-tool';
-const RUNS_DIR = join(OUT, 'runs');
-const RESULTS = join(OUT, 'results.jsonl');
-
-if (!existsSync(RUNS_DIR)) {
-  console.error(`no runs dir at ${RUNS_DIR}`);
-  process.exit(1);
-}
-
-function parse(file) {
-  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
-  const toolCounts = {};
-  const cliCommands = { cgraphx: [], gitnexus: [] };
-  const hookDenies = { cgraphx: 0, gitnexus: 0 };
-  let result = null;
-
-  for (const line of lines) {
-    let ev;
-    try { ev = JSON.parse(line); } catch { continue; }
-
-    if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type !== 'tool_use') continue;
-        toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
-        if (b.name === 'Bash' && typeof b.input?.command === 'string') {
-          const cmd = b.input.command;
-          if (/^cgraphx /.test(cmd) || / cgraphx /.test(cmd) || /codegraph\.js/.test(cmd)) {
-            cliCommands.cgraphx.push(cmd.slice(0, 80));
-          }
-          if (/gitnexus\/dist\/cli\/index\.js/.test(cmd) || /gitnexus\/run\.cjs/.test(cmd) || /^gitnexus /.test(cmd) || / gitnexus /.test(cmd)) {
-            cliCommands.gitnexus.push(cmd.slice(0, 80));
-          }
-        }
-      }
-    }
-
-    if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type !== 'tool_result') continue;
-        const text = typeof b.content === 'string'
-          ? b.content
-          : Array.isArray(b.content)
-            ? b.content.map(c => (typeof c === 'string' ? c : c?.text || '')).join('')
-            : '';
-        if (/禁用 cgraphx CLI/.test(text)) hookDenies.cgraphx++;
-        if (/禁用 gitnexus CLI/.test(text)) hookDenies.gitnexus++;
-      }
-    }
-
-    if (ev.type === 'result') result = ev;
-  }
-
-  let skillCgraphx = 0, skillGitnexus = 0;
-  for (const line of lines) {
-    let ev;
-    try { ev = JSON.parse(line); } catch { continue; }
-    if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type !== 'tool_use' || b.name !== 'Skill') continue;
-        const s = b.input?.skill || '';
-        if (s === 'cgraphx-search' || s === 'cgraphx') skillCgraphx++;
-        if (s.startsWith('gitnexus')) skillGitnexus++;
-      }
-    }
-  }
-
-  const u = result?.usage || {};
-  const tokGen = u.output_tokens || 0;
-  const tokFresh = (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
-  const tokCached = u.cache_read_input_tokens || 0;
-
-  const get = (n) => toolCounts[n] || 0;
-  const mcpCalls = Object.keys(toolCounts)
-    .filter(k => /^mcp__(cgraphx|gitnexus)__/.test(k))
-    .reduce((s, k) => s + toolCounts[k], 0);
-  // Per-tool MCP breakdown
-  const mcpByServer = {};
-  for (const k of Object.keys(toolCounts)) {
-    const m = k.match(/^mcp__(cgraphx|gitnexus)__/);
-    if (m) mcpByServer[m[1]] = (mcpByServer[m[1]] || 0) + toolCounts[k];
-  }
-
-  return {
-    read: get('Read'),
-    grep: get('Grep') + get('Glob'),
-    bash: get('Bash'),
-    cliCallsCgraphx: cliCommands.cgraphx.length,
-    cliCallsGitnexus: cliCommands.gitnexus.length,
-    mcpCalls,
-    mcpCgraphx: mcpByServer.cgraphx || 0,
-    mcpGitnexus: mcpByServer.gitnexus || 0,
-    skillCgraphx,
-    skillGitnexus,
-    hookDeniesCgraphx: hookDenies.cgraphx,
-    hookDeniesGitnexus: hookDenies.gitnexus,
-    durationSec: result ? Math.floor(result.duration_ms / 1000) : null,
-    numTurns: result?.num_turns ?? null,
-    costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
-    tokGen,
-    tokFresh,
-    tokCached,
-    tokBillable: tokGen + tokFresh,
-    ok: result?.subtype === 'success',
-  };
-}
-
-// task -> arm -> [runs]
-const data = {};
-const allMetrics = [];
-for (const f of readdirSync(RUNS_DIR)) {
-  const m = f.match(/^(.+)-(\w+)-(\w+-\w+)-(\d+)\.jsonl$/);
-  if (!m) continue;
-  const [, repo, task, arm, repStr] = m;
-  const parsed = parse(join(RUNS_DIR, f));
-  const rep = Number(repStr);
-  const record = { repo, task, arm, rep, file: f, ...parsed };
-  allMetrics.push(record);
-  (((data[task] ??= {})[arm]) ??= []).push(record);
-}
-
-writeFileSync(RESULTS, allMetrics
-  .sort((a, b) => a.task.localeCompare(b.task) || a.arm.localeCompare(b.arm) || a.rep - b.rep)
-  .map(r => JSON.stringify(r))
-  .join('\n') + '\n'
-);
-
-const pad = (s, n) => String(s).padEnd(n);
-const TASKS = ['flow', 'onboarding', 'impact'];
-const ARMS = ['cgraphx-cli', 'cgraphx-mcp', 'gitnexus-mcp'];
-const LABEL = {
-  'cgraphx-cli': 'A1 cgraphx-cli',
-  'cgraphx-mcp': 'A2 cgraphx-mcp',
-  'gitnexus-mcp': 'A3 gitnexus-mcp',
-};
-
-// ---- PER TASK × ARM × REP ----
-console.log('\n=== PER TASK × ARM × REP (detail) ===');
-console.log(
-  pad('task', 11), pad('arm', 18), pad('rep', 4),
-  pad('dur', 5), pad('read', 5), pad('grep', 5), pad('bash', 5),
-  pad('cgCli', 5), pad('gnCli', 5),
-  pad('cgMcp', 5), pad('gnMcp', 5),
-  pad('deny', 5),
-  pad('tokBill', 8), pad('cost', 8), 'ok'
-);
-for (const task of TASKS) {
-  for (const arm of ARMS) {
-    const runs = data[task]?.[arm] || [];
-    for (const r of runs) {
-      console.log(
-        pad(task, 11), pad(LABEL[arm], 18), pad(r.rep, 4),
-        pad(r.durationSec ?? '?', 5),
-        pad(r.read, 5), pad(r.grep, 5), pad(r.bash, 5),
-        pad(r.cliCallsCgraphx, 5), pad(r.cliCallsGitnexus, 5),
-        pad(r.mcpCgraphx, 5), pad(r.mcpGitnexus, 5),
-        pad(r.hookDeniesCgraphx + r.hookDeniesGitnexus, 5),
-        pad(r.tokBillable, 8),
-        pad('$' + (r.costUsdMain ?? 0).toFixed(4), 8),
-        r.ok ? '✓' : '✗'
-      );
-    }
-  }
-}
-
-const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
-
-// ---- AGGREGATE PER TASK × ARM ----
-console.log('\n=== AGGREGATE PER TASK × ARM (mean across reps) ===');
-console.log(
-  pad('task', 11), pad('arm', 18),
-  pad('read', 6), pad('grep', 6), pad('bash', 6),
-  pad('cgCli', 5), pad('gnCli', 5),
-  pad('cgMcp', 5), pad('gnMcp', 5),
-  pad('dur', 6), pad('tokBill', 8), pad('cost', 8), 'n'
-);
-for (const task of TASKS) {
-  for (const arm of ARMS) {
-    const runs = data[task]?.[arm] || [];
-    if (!runs.length) continue;
-    console.log(
-      pad(task, 11), pad(LABEL[arm], 18),
-      pad(avg(runs, r => r.read).toFixed(1), 6),
-      pad(avg(runs, r => r.grep).toFixed(1), 6),
-      pad(avg(runs, r => r.bash).toFixed(1), 6),
-      pad(avg(runs, r => r.cliCallsCgraphx).toFixed(1), 5),
-      pad(avg(runs, r => r.cliCallsGitnexus).toFixed(1), 5),
-      pad(avg(runs, r => r.mcpCgraphx).toFixed(1), 5),
-      pad(avg(runs, r => r.mcpGitnexus).toFixed(1), 5),
-      pad(avg(runs, r => r.durationSec).toFixed(0) + 's', 6),
-      pad(avg(runs, r => r.tokBillable).toFixed(0), 8),
-      pad('$' + avg(runs, r => r.costUsdMain).toFixed(4), 8),
-      runs.length
-    );
-  }
-}
-
-// ---- AGGREGATE PER ARM (across all tasks × reps) ----
-console.log('\n=== AGGREGATE PER ARM (mean across all tasks × reps) ===');
-console.log(
-  pad('arm', 18),
-  pad('read', 6), pad('grep', 6), pad('bash', 6),
-  pad('cgCli', 5), pad('gnCli', 5),
-  pad('cgMcp', 5), pad('gnMcp', 5),
-  pad('dur', 6), pad('tokBill', 8), pad('cost', 8), 'n'
-);
-for (const arm of ARMS) {
-  const all = [];
-  for (const task of TASKS) for (const r of (data[task]?.[arm] || [])) all.push(r);
-  if (!all.length) continue;
-  console.log(
-    pad(LABEL[arm], 18),
-    pad(avg(all, r => r.read).toFixed(1), 6),
-    pad(avg(all, r => r.grep).toFixed(1), 6),
-    pad(avg(all, r => r.bash).toFixed(1), 6),
-    pad(avg(all, r => r.cliCallsCgraphx).toFixed(1), 5),
-    pad(avg(all, r => r.cliCallsGitnexus).toFixed(1), 5),
-    pad(avg(all, r => r.mcpCgraphx).toFixed(1), 5),
-    pad(avg(all, r => r.mcpGitnexus).toFixed(1), 5),
-    pad(avg(all, r => r.durationSec).toFixed(0) + 's', 6),
-    pad(avg(all, r => r.tokBillable).toFixed(0), 8),
-    pad('$' + avg(all, r => r.costUsdMain).toFixed(4), 8),
-    all.length
-  );
-}
-
-console.log(`\nresults.jsonl: ${RESULTS}`);