aiforcecli-chat 0.1.0

package/tools/scorecard/fetch.mjs

@@ -0,0 +1,335 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+
+function parseArgs(argv) {
+  const args = {
+    config: path.join('tools', 'scorecard', 'config.json'),
+    sourceIds: undefined,
+    rawDir: path.join('generated', 'scorecard-fetch', 'raw'),
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--config') args.config = needValue(argv, ++i, a);
+    else if (a === '--source') {
+      const values = needValue(argv, ++i, a).split(',').map((s) => s.trim()).filter(Boolean);
+      args.sourceIds = [...(args.sourceIds ?? []), ...values];
+    } else if (a === '--raw-dir') args.rawDir = needValue(argv, ++i, a);
+    else if (a === '--help' || a === '-h') {
+      printHelp();
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${a}`);
+    }
+  }
+  return args;
+}
+
+function needValue(argv, i, flag) {
+  const v = argv[i];
+  if (!v || v.startsWith('--')) throw new Error(`${flag} requires a value`);
+  return v;
+}
+
+function printHelp() {
+  console.log(`Usage: node tools/scorecard/fetch.mjs [options]
+
+Options:
+  --config <path>       Generator config JSON
+  --source <id[,id]>    Fetch only selected source IDs
+  --raw-dir <dir>       Directory for raw fetched HTML snapshots
+`);
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+async function fetchText(url) {
+  const res = await fetch(url, {
+    headers: {
+      'user-agent': 'aiforcecli-scorecard-fetch/0.1 (+manual update command)',
+      accept: 'text/html,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.5',
+    },
+  });
+  if (!res.ok) throw new Error(`GET ${url} failed: ${res.status} ${res.statusText}`);
+  return await res.text();
+}
+
+function htmlToText(html) {
+  return html
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '\n')
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '\n')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/(p|div|li|tr|td|th|h\d|details|summary)>/gi, '\n')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&#x27;/g, "'")
+    .replace(/&quot;/g, '"')
+    .replace(/\r/g, '\n');
+}
+
+function textLines(html) {
+  return htmlToText(html)
+    .split('\n')
+    .map((line) => line.replace(/\s+/g, ' ').trim())
+    .filter(Boolean);
+}
+
+function compactModelName(name) {
+  return String(name ?? '')
+    .replace(/^▶\s*/, '')
+    .replace(/[()]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function stripTags(s) {
+  return htmlToText(s).replace(/\s+/g, ' ').trim();
+}
+
+function detailValue(html, label) {
+  const re = new RegExp(`${label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}[\\s\\S]*?<\\/strong>\\s*([^<\\n]+)`, 'i');
+  const m = html.match(re);
+  return m ? stripTags(m[1]) : undefined;
+}
+
+function parseAiderPolyglot(html, source) {
+  const htmlRows = parseAiderHtmlRows(html, source);
+  if (htmlRows.length > 0) return htmlRows;
+
+  const lines = textLines(html);
+  const rows = [];
+  let current;
+
+  function flush() {
+    if (!current || current.score == null) return;
+    rows.push({
+      source: source.source ?? source.id,
+      benchmark: source.benchmark ?? 'aider_polyglot',
+      url: source.url,
+      modelRaw: current.modelRaw,
+      metric: 'pass_rate_2',
+      score: current.score,
+      scoreScale: 'percent',
+      sampleSize: current.sampleSize,
+      date: current.date,
+      extra: current.extra,
+    });
+  }
+
+  for (const line of lines) {
+    if (line.startsWith('▶ ')) {
+      flush();
+      current = { modelRaw: compactModelName(line), extra: {} };
+      continue;
+    }
+    if (!current) continue;
+    let m = line.match(/^Pass rate 2\s*:\s*([\d.]+)/i);
+    if (m) {
+      current.score = Number(m[1]);
+      continue;
+    }
+    m = line.match(/^Test cases\s*:\s*(\d+)/i);
+    if (m) {
+      current.sampleSize = Number(m[1]);
+      continue;
+    }
+    m = line.match(/^Date\s*:\s*(\d{4}-\d{2}-\d{2})/i);
+    if (m) {
+      current.date = m[1];
+      continue;
+    }
+    m = line.match(/^Total cost\s*:\s*([\d.]+)/i);
+    if (m) current.extra.totalCostUsd = Number(m[1]);
+  }
+  flush();
+  return rows;
+}
+
+function parseAiderHtmlRows(html, source) {
+  const rows = [];
+  const rowRe = /<tr id="main-row-(\d+)"[\s\S]*?<\/tr>\s*<tr class="details-row" id="details-\1"[\s\S]*?<\/tr>/gi;
+  let match;
+  while ((match = rowRe.exec(html))) {
+    const block = match[0];
+    const modelMatch = block.match(/<td style="padding:\s*8px;"><span>([\s\S]*?)<\/span><\/td>/i);
+    const modelRaw = modelMatch ? compactModelName(stripTags(modelMatch[1])) : detailValue(block, 'Model');
+    const score = Number(detailValue(block, 'Pass rate 2'));
+    if (!modelRaw || !Number.isFinite(score)) continue;
+    rows.push({
+      source: source.source ?? source.id,
+      benchmark: source.benchmark ?? 'aider_polyglot',
+      url: source.url,
+      modelRaw,
+      metric: 'pass_rate_2',
+      score,
+      scoreScale: 'percent',
+      sampleSize: maybeNumber(detailValue(block, 'Test cases')),
+      date: detailValue(block, 'Date'),
+      extra: {
+        passRate1: maybeNumber(detailValue(block, 'Pass rate 1')),
+        passNum1: maybeNumber(detailValue(block, 'Pass num 1')),
+        passNum2: maybeNumber(detailValue(block, 'Pass num 2')),
+        totalCostUsd: maybeNumber(detailValue(block, 'Total cost')),
+        secondsPerCase: maybeNumber(detailValue(block, 'Seconds per case')),
+        editFormat: detailValue(block, 'Edit format'),
+      },
+    });
+  }
+  return rows;
+}
+
+function maybeNumber(v) {
+  if (v == null) return undefined;
+  const n = Number(String(v).replace(/[$,%]/g, '').trim());
+  return Number.isFinite(n) ? n : undefined;
+}
+
+function findJsonArrayAfter(text, key) {
+  const start = text.indexOf(key);
+  if (start < 0) return undefined;
+  const bracket = text.indexOf('[', start + key.length);
+  if (bracket < 0) return undefined;
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = bracket; i < text.length; i += 1) {
+    const ch = text[i];
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (ch === '\\') escaped = true;
+      else if (ch === '"') inString = false;
+      continue;
+    }
+    if (ch === '"') inString = true;
+    else if (ch === '[') depth += 1;
+    else if (ch === ']') {
+      depth -= 1;
+      if (depth === 0) return text.slice(bracket, i + 1);
+    }
+  }
+  return undefined;
+}
+
+function parseTerminalBench(html, source) {
+  const serialized = html.replace(/\\"/g, '"').replace(/\\u0026/g, '&');
+  const arr = findJsonArrayAfter(serialized, '"rows"');
+  if (arr) {
+    try {
+      const parsed = JSON.parse(arr);
+      if (Array.isArray(parsed)) {
+        return parsed
+          .filter((r) => typeof r.agent === 'string' && Array.isArray(r.model) && typeof r.accuracy === 'number')
+          .map((r) => ({
+            source: source.source ?? 'terminal_bench',
+            benchmark: source.benchmark ?? 'terminal_bench',
+            url: source.url,
+            modelRaw: `${r.agent} ${r.model.join(' + ')}`,
+            metric: 'accuracy',
+            score: r.accuracy,
+            scoreScale: '0-1',
+            date: r.date,
+            extra: {
+              agent: r.agent,
+              model: r.model,
+              stderr: r.stderr,
+              verified: r.verified,
+              agentName: r.agentName,
+              agentVersion: r.agentVersion,
+              modelNames: r.modelNames,
+              modelProviders: r.modelProviders,
+            },
+          }));
+      }
+    } catch {
+      // Fall through to text parser below.
+    }
+  }
+
+  const lines = textLines(html);
+  const rows = [];
+  const prefixes = source.agentPrefixes ?? ['Codex CLI', 'Claude Code', 'Gemini CLI', 'Terminus 2'];
+
+  for (let i = 1; i < lines.length; i += 1) {
+    const acc = lines[i].match(/^([\d.]+)%\s*±\s*([\d.]+)/);
+    if (!acc) continue;
+    const meta = lines[i - 1];
+    const dateMatch = meta.match(/\b\d{4}-\d{2}-\d{2}\b/);
+    if (!dateMatch || dateMatch.index == null) continue;
+    const beforeDate = meta.slice(0, dateMatch.index).trim();
+    const date = dateMatch[0];
+    const prefix = prefixes.find((p) => beforeDate === p || beforeDate.startsWith(`${p} `));
+    if (!prefix) continue;
+    const model = beforeDate.slice(prefix.length).trim();
+    if (!model) continue;
+    rows.push({
+      source: source.source ?? 'terminal_bench',
+      benchmark: source.benchmark ?? 'terminal_bench',
+      url: source.url,
+      modelRaw: `${prefix} ${model}`,
+      metric: 'accuracy',
+      score: Number(acc[1]),
+      scoreScale: 'percent',
+      date,
+      extra: {
+        agent: prefix,
+        model,
+        stderrPct: Number(acc[2]),
+      },
+    });
+  }
+  return rows;
+}
+
+function parseRows(html, source) {
+  if (source.parser === 'aiderPolyglot') return parseAiderPolyglot(html, source);
+  if (source.parser === 'terminalBench') return parseTerminalBench(html, source);
+  throw new Error(`Unsupported parser for ${source.id}: ${source.parser}`);
+}
+
+function outputSnapshot(source, rows, fetchedAt) {
+  return {
+    source: source.id,
+    fetchedAt,
+    url: source.url,
+    parser: source.parser,
+    rows,
+  };
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const config = readJson(args.config);
+  const selected = new Set(args.sourceIds ?? []);
+  const sources = (config.fetchSources ?? []).filter((s) => selected.size === 0 || selected.has(s.id));
+  if (sources.length === 0) throw new Error('No fetch sources selected.');
+
+  fs.mkdirSync(args.rawDir, { recursive: true });
+
+  let totalRows = 0;
+  for (const source of sources) {
+    console.log(`fetching ${source.id} from ${source.url}`);
+    const html = await fetchText(source.url);
+    const fetchedAt = new Date().toISOString();
+    const rawPath = path.join(args.rawDir, `${source.id}.html`);
+    fs.writeFileSync(rawPath, html);
+
+    const rows = parseRows(html, source);
+    totalRows += rows.length;
+    fs.mkdirSync(path.dirname(source.output), { recursive: true });
+    fs.writeFileSync(source.output, `${JSON.stringify(outputSnapshot(source, rows, fetchedAt), null, 2)}\n`);
+    console.log(`wrote ${source.output} (${rows.length} row(s))`);
+  }
+  console.log(`fetched ${sources.length} source(s), ${totalRows} normalized row(s)`);
+}
+
+try {
+  await main();
+} catch (err) {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(1);
+}

package/tools/scorecard/generate.mjs

@@ -0,0 +1,289 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+
+const TASK_TYPES = ['bugfix', 'feature', 'refactor', 'test', 'docs', 'security', 'perf', 'general'];
+
+function parseArgs(argv) {
+  const args = {
+    config: path.join('tools', 'scorecard', 'config.json'),
+    input: [path.join('tools', 'scorecard', 'snapshots')],
+    out: path.join('generated', 'scorecard'),
+    includeExamples: false,
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--config') args.config = needValue(argv, ++i, a);
+    else if (a === '--input') args.input.push(needValue(argv, ++i, a));
+    else if (a === '--out') args.out = needValue(argv, ++i, a);
+    else if (a === '--include-examples') args.includeExamples = true;
+    else if (a === '--help' || a === '-h') {
+      printHelp();
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${a}`);
+    }
+  }
+  return args;
+}
+
+function needValue(argv, i, flag) {
+  const v = argv[i];
+  if (!v || v.startsWith('--')) throw new Error(`${flag} requires a value`);
+  return v;
+}
+
+function printHelp() {
+  console.log(`Usage: node tools/scorecard/generate.mjs [options]
+
+Options:
+  --config <path>       Generator config JSON
+  --input <path>        Snapshot file or directory; can be repeated
+  --out <dir>           Output directory
+  --include-examples    Include *.example.json files
+`);
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function listJsonFiles(inputs, includeExamples) {
+  const files = [];
+  for (const input of inputs) {
+    if (!fs.existsSync(input)) continue;
+    const stat = fs.statSync(input);
+    if (stat.isDirectory()) {
+      for (const name of fs.readdirSync(input)) {
+        const full = path.join(input, name);
+        if (fs.statSync(full).isDirectory()) continue;
+        if (!name.endsWith('.json')) continue;
+        if (!includeExamples && name.endsWith('.example.json')) continue;
+        files.push(full);
+      }
+    } else if (input.endsWith('.json')) {
+      const name = path.basename(input);
+      if (includeExamples || !name.endsWith('.example.json')) files.push(input);
+    }
+  }
+  return [...new Set(files)].sort();
+}
+
+function extractRows(file) {
+  const data = readJson(file);
+  const rows = Array.isArray(data) ? data : data.rows;
+  if (!Array.isArray(rows)) throw new Error(`${file} must be an array or an object with rows[]`);
+  return rows.map((row, index) => ({ ...row, snapshotFile: file, snapshotIndex: index }));
+}
+
+function normalizeKey(s) {
+  return String(s ?? '')
+    .trim()
+    .toLowerCase()
+    .replace(/[_/]+/g, ' ')
+    .replace(/\s+/g, ' ');
+}
+
+function canonicalModel(row, aliases) {
+  if (row.modelCanonical) return row.modelCanonical;
+  const raw = normalizeKey(row.modelRaw);
+  return aliases[raw];
+}
+
+function clamp(n, lo, hi) {
+  return Math.max(lo, Math.min(hi, n));
+}
+
+function normalizeScore(row) {
+  const raw = Number(row.score);
+  if (!Number.isFinite(raw)) return undefined;
+  if (row.scoreScale === 'percent') return clamp(raw / 100, 0, 1);
+  if (row.scoreScale === '0-1') return clamp(raw, 0, 1);
+  return raw > 1 ? clamp(raw / 100, 0, 1) : clamp(raw, 0, 1);
+}
+
+function ageDays(dateLike, now) {
+  if (!dateLike) return undefined;
+  const t = new Date(dateLike).getTime();
+  if (!Number.isFinite(t)) return undefined;
+  return Math.max(0, (now.getTime() - t) / 86_400_000);
+}
+
+function freshnessWeight(row, config, now) {
+  const age = ageDays(row.date, now);
+  if (age == null) return 0.7;
+  const halfLife = Number(config.halfLifeDays ?? 180);
+  return clamp(Math.exp((-Math.log(2) * age) / halfLife), 0.05, 1);
+}
+
+function sampleConfidence(row, config) {
+  if (row.confidence != null) return clamp(Number(row.confidence), 0, 1);
+  const sampleSize = Number(row.sampleSize);
+  if (!Number.isFinite(sampleSize) || sampleSize <= 0) return Number(config.unknownSampleConfidence ?? 0.55);
+  const full = Number(config.sampleSizeFullConfidence ?? 500);
+  return clamp(Math.sqrt(sampleSize / full), 0.15, 1);
+}
+
+function rowWeight(row, task, config, now) {
+  const benchmark = row.benchmark ?? row.source;
+  const taskWeights = config.benchmarkTaskWeights?.[benchmark] ?? {};
+  const relevance = Number(taskWeights[task] ?? 0);
+  if (relevance <= 0) return 0;
+  const source = row.source ?? benchmark;
+  const reliability = Number(config.sourceReliability?.[source] ?? 0.5);
+  return relevance * reliability * freshnessWeight(row, config, now) * sampleConfidence(row, config);
+}
+
+function scoreRows(rows, config, now) {
+  const byArm = new Map();
+  const unmapped = [];
+  const invalid = [];
+
+  for (const row of rows) {
+    const model = canonicalModel(row, config.modelAliases ?? {});
+    const score = normalizeScore(row);
+    if (!model) {
+      unmapped.push(row);
+      continue;
+    }
+    if (score == null) {
+      invalid.push({ ...row, reason: 'score is missing or invalid' });
+      continue;
+    }
+    if (!byArm.has(model)) byArm.set(model, []);
+    byArm.get(model).push({ ...row, modelCanonical: model, normalizedScore: score });
+  }
+
+  const scores = {};
+  const compact = {};
+  for (const [model, modelRows] of [...byArm.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+    scores[model] = {};
+    compact[model] = {};
+    for (const task of TASK_TYPES) {
+      const evidence = [];
+      let weighted = 0;
+      let totalWeight = 0;
+      for (const row of modelRows) {
+        const weight = rowWeight(row, task, config, now);
+        if (weight <= 0) continue;
+        weighted += row.normalizedScore * weight;
+        totalWeight += weight;
+        evidence.push({
+          source: row.source,
+          benchmark: row.benchmark ?? row.source,
+          metric: row.metric,
+          score: round(row.normalizedScore, 4),
+          weight: round(weight, 4),
+          sampleSize: row.sampleSize,
+          date: row.date,
+          url: row.url,
+          modelRaw: row.modelRaw,
+        });
+      }
+      if (totalWeight <= 0) continue;
+      const score = weighted / totalWeight;
+      const confidence = totalWeight / (totalWeight + 2);
+      scores[model][task] = {
+        score: round(score, 4),
+        confidence: round(confidence, 4),
+        evidenceWeight: round(totalWeight, 4),
+        sources: evidence.sort((a, b) => b.weight - a.weight),
+      };
+      compact[model][task] = round(score, 4);
+    }
+    if (Object.keys(scores[model]).length === 0) delete scores[model];
+    if (Object.keys(compact[model]).length === 0) delete compact[model];
+  }
+
+  return { scores, compact, unmapped, invalid };
+}
+
+function round(n, digits) {
+  const p = 10 ** digits;
+  return Math.round(n * p) / p;
+}
+
+function makeReport(result, files, now) {
+  const lines = [];
+  lines.push('# Generated Scorecard Report');
+  lines.push('');
+  lines.push(`Generated: ${now.toISOString()}`);
+  lines.push(`Snapshots: ${files.length}`);
+  lines.push(`Mapped models: ${Object.keys(result.scores).length}`);
+  lines.push(`Unmapped rows: ${result.unmapped.length}`);
+  lines.push(`Invalid rows: ${result.invalid.length}`);
+  lines.push('');
+  lines.push('## Snapshot Files');
+  lines.push('');
+  for (const file of files) lines.push(`- ${file}`);
+  lines.push('');
+  lines.push('## Scores');
+  lines.push('');
+  for (const [model, tasks] of Object.entries(result.scores)) {
+    lines.push(`### ${model}`);
+    lines.push('');
+    lines.push('| Task | Score | Confidence | Evidence Weight | Top Evidence |');
+    lines.push('| --- | ---: | ---: | ---: | --- |');
+    for (const task of TASK_TYPES) {
+      const value = tasks[task];
+      if (!value) continue;
+      const top = value.sources[0];
+      const label = top ? `${top.benchmark}/${top.metric} (${top.modelRaw})` : '';
+      lines.push(`| ${task} | ${pct(value.score)} | ${pct(value.confidence)} | ${value.evidenceWeight} | ${label} |`);
+    }
+    lines.push('');
+  }
+  if (result.unmapped.length > 0) {
+    lines.push('## Unmapped Models');
+    lines.push('');
+    for (const row of result.unmapped.slice(0, 50)) {
+      lines.push(`- ${row.modelRaw ?? '(missing modelRaw)'} from ${row.source ?? row.benchmark ?? '(unknown source)'} in ${row.snapshotFile}`);
+    }
+    if (result.unmapped.length > 50) lines.push(`- ...${result.unmapped.length - 50} more`);
+    lines.push('');
+  }
+  return `${lines.join('\n')}\n`;
+}
+
+function pct(n) {
+  return `${Math.round(n * 1000) / 10}%`;
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const now = new Date();
+  const config = readJson(args.config);
+  const files = listJsonFiles(args.input, args.includeExamples);
+  const rows = files.flatMap(extractRows);
+  const result = scoreRows(rows, config, now);
+  const versionDate = now.toISOString().slice(0, 10).replaceAll('-', '.');
+  const output = {
+    version: `${config.versionPrefix ?? 'manual'}.${versionDate}`,
+    generatedAt: now.toISOString(),
+    taskTypes: TASK_TYPES,
+    notes: [
+      'Generated scorecard artifact. It is not used by the application unless explicitly wired in later.',
+      'Scores are normalized public benchmark priors, not private repo outcomes.',
+    ],
+    scores: result.scores,
+  };
+
+  fs.mkdirSync(args.out, { recursive: true });
+  fs.writeFileSync(path.join(args.out, 'scorecard.json'), `${JSON.stringify(output, null, 2)}\n`);
+  fs.writeFileSync(path.join(args.out, 'scorecard.compact.json'), `${JSON.stringify(result.compact, null, 2)}\n`);
+  fs.writeFileSync(path.join(args.out, 'unmapped-models.json'), `${JSON.stringify(result.unmapped, null, 2)}\n`);
+  fs.writeFileSync(path.join(args.out, 'invalid-rows.json'), `${JSON.stringify(result.invalid, null, 2)}\n`);
+  fs.writeFileSync(path.join(args.out, 'scorecard-report.md'), makeReport(result, files, now));
+
+  console.log(`scorecard generator read ${rows.length} row(s) from ${files.length} file(s)`);
+  console.log(`wrote ${args.out}`);
+  if (result.unmapped.length > 0) console.log(`unmapped rows: ${result.unmapped.length}`);
+  if (result.invalid.length > 0) console.log(`invalid rows: ${result.invalid.length}`);
+}
+
+try {
+  main();
+} catch (err) {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(1);
+}

package/tools/scorecard/generated/example/invalid-rows.json

@@ -0,0 +1 @@
+[]