kushi-agents 5.0.2 → 5.0.3

package/src/eval-aggregator.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// kushi v5.0.3 — eval-aggregator.mjs
+//
+// Reads one (or many) per-run JSON files produced by plugin/skills/eval/run-evals.ps1,
+// computes per-skill pass_rate / mean_duration_ms / mean_tokens_total + stddev, writes
+// Evidence/_evals/benchmark.json, optionally compares against evals/baseline.json and
+// flags regressions:
+//   - pass_rate drop  >= 10 percentage points
+//   - mean_duration   >= 50 % slower
+//   - mean_tokens     >= 50 % more
+//
+// With --update-baseline, overwrites evals/baseline.json with the current run's metrics.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const REGRESSION = {
+  passRatePpDrop: 10,
+  durationFactor: 1.5,
+  tokensFactor: 1.5,
+};
+
+function parseArgs(argv) {
+  const out = { runs: [], bench: null, baseline: null, updateBaseline: false };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--run') out.runs.push(argv[++i]);
+    else if (a === '--runs-dir') {
+      const dir = argv[++i];
+      for (const f of fs.readdirSync(dir)) {
+        if (f.endsWith('.json') && f !== 'benchmark.json') out.runs.push(path.join(dir, f));
+      }
+    }
+    else if (a === '--bench') out.bench = argv[++i];
+    else if (a === '--baseline') out.baseline = argv[++i];
+    else if (a === '--update-baseline') out.updateBaseline = true;
+  }
+  return out;
+}
+
+export function loadRun(filePath) {
+  const text = fs.readFileSync(filePath, 'utf-8');
+  const obj = JSON.parse(text);
+  if (!obj || !Array.isArray(obj.skills)) {
+    throw new Error(`${filePath}: not a kushi.evals.run/v1 JSON (missing skills[])`);
+  }
+  return obj;
+}
+
+function mean(xs) {
+  if (xs.length === 0) return 0;
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+
+function stddev(xs) {
+  if (xs.length < 2) return 0;
+  const m = mean(xs);
+  const v = xs.reduce((a, b) => a + (b - m) * (b - m), 0) / (xs.length - 1);
+  return Math.sqrt(v);
+}
+
+export function aggregate(runs) {
+  // skill -> { cases: [{pass, duration_ms, tokens}] }
+  const bySkill = new Map();
+  for (const run of runs) {
+    for (const s of run.skills) {
+      if (!bySkill.has(s.skill)) bySkill.set(s.skill, []);
+      for (const c of s.cases) {
+        if (c.error && String(c.error).startsWith('skipped')) continue;
+        bySkill.get(s.skill).push({
+          pass: !!c.pass,
+          duration_ms: c.duration_ms || 0,
+          tokens: (c.tokens_in || 0) + (c.tokens_out || 0),
+        });
+      }
+    }
+  }
+  const skills = [];
+  for (const [skill, cases] of bySkill.entries()) {
+    const n = cases.length;
+    const passes = cases.filter((c) => c.pass).length;
+    const durations = cases.map((c) => c.duration_ms);
+    const tokens = cases.map((c) => c.tokens).filter((t) => t > 0);
+    skills.push({
+      skill,
+      n,
+      pass_rate: n === 0 ? 0 : passes / n,
+      mean_duration_ms: Math.round(mean(durations)),
+      stddev_duration_ms: Math.round(stddev(durations)),
+      mean_tokens_total: Math.round(mean(tokens)),
+      stddev_tokens_total: Math.round(stddev(tokens)),
+    });
+  }
+  skills.sort((a, b) => a.skill.localeCompare(b.skill));
+  return { schema: 'kushi.evals.benchmark/v1', generated_at: new Date().toISOString(), skills };
+}
+
+export function detectRegressions(bench, baseline) {
+  if (!baseline || !Array.isArray(baseline.skills)) return [];
+  const byName = new Map(baseline.skills.map((s) => [s.skill, s]));
+  const regs = [];
+  for (const cur of bench.skills) {
+    const base = byName.get(cur.skill);
+    if (!base) continue;
+    const ppDrop = (base.pass_rate - cur.pass_rate) * 100;
+    if (ppDrop >= REGRESSION.passRatePpDrop) {
+      regs.push({
+        skill: cur.skill,
+        metric: 'pass_rate',
+        baseline: base.pass_rate,
+        current: cur.pass_rate,
+        delta_pp: Number(ppDrop.toFixed(1)),
+        threshold_pp: REGRESSION.passRatePpDrop,
+      });
+    }
+    if (base.mean_duration_ms > 0) {
+      const factor = cur.mean_duration_ms / base.mean_duration_ms;
+      if (factor >= REGRESSION.durationFactor) {
+        regs.push({
+          skill: cur.skill,
+          metric: 'mean_duration_ms',
+          baseline: base.mean_duration_ms,
+          current: cur.mean_duration_ms,
+          factor: Number(factor.toFixed(2)),
+          threshold_factor: REGRESSION.durationFactor,
+        });
+      }
+    }
+    if (base.mean_tokens_total > 0) {
+      const factor = cur.mean_tokens_total / base.mean_tokens_total;
+      if (factor >= REGRESSION.tokensFactor) {
+        regs.push({
+          skill: cur.skill,
+          metric: 'mean_tokens_total',
+          baseline: base.mean_tokens_total,
+          current: cur.mean_tokens_total,
+          factor: Number(factor.toFixed(2)),
+          threshold_factor: REGRESSION.tokensFactor,
+        });
+      }
+    }
+  }
+  return regs;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.runs.length === 0) {
+    console.error('eval-aggregator: at least one --run <file> required');
+    process.exit(2);
+  }
+
+  const runs = args.runs.map(loadRun);
+  const bench = aggregate(runs);
+
+  let baseline = null;
+  if (args.baseline && fs.existsSync(args.baseline)) {
+    baseline = JSON.parse(fs.readFileSync(args.baseline, 'utf-8'));
+  }
+  const regressions = detectRegressions(bench, baseline);
+  bench.summary = {
+    n_skills: bench.skills.length,
+    mean_pass_rate: Number(mean(bench.skills.map((s) => s.pass_rate)).toFixed(3)),
+    regressions,
+  };
+
+  if (args.bench) {
+    fs.mkdirSync(path.dirname(args.bench), { recursive: true });
+    fs.writeFileSync(args.bench, JSON.stringify(bench, null, 2));
+    console.log(`eval-aggregator: wrote ${args.bench}`);
+  } else {
+    process.stdout.write(JSON.stringify(bench, null, 2));
+  }
+
+  if (args.updateBaseline && args.baseline) {
+    fs.mkdirSync(path.dirname(args.baseline), { recursive: true });
+    fs.writeFileSync(
+      args.baseline,
+      JSON.stringify(
+        {
+          schema: 'kushi.evals.baseline/v1',
+          generated_at: bench.generated_at,
+          skills: bench.skills,
+        },
+        null,
+        2,
+      ),
+    );
+    console.log(`eval-aggregator: baseline updated -> ${args.baseline}`);
+  }
+
+  if (regressions.length > 0) {
+    console.error(`eval-aggregator: ${regressions.length} regression(s) flagged:`);
+    for (const r of regressions) console.error('  -', JSON.stringify(r));
+    process.exit(1);
+  }
+}
+
+// Only run when invoked directly, not when imported by tests.
+const isDirect = (() => {
+  try {
+    const argv1 = path.resolve(process.argv[1] || '');
+    const here = path.resolve(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1'));
+    return argv1 === here;
+  } catch { return false; }
+})();
+if (isDirect) {
+  main().catch((e) => { console.error(e); process.exit(2); });
+}

package/src/eval-aggregator.test.mjs

@@ -0,0 +1,64 @@
+// kushi v5.0.3 — eval-aggregator unit tests.
+//
+// Sanity-checks aggregation + regression detection logic. Pure functions, no IO
+// against the real repo.
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { aggregate, detectRegressions } from './eval-aggregator.mjs';
+
+function makeRun(skill, cases) {
+  return {
+    schema: 'kushi.evals.run/v1',
+    mode: 'test',
+    skills: [{ skill, cases }],
+  };
+}
+
+test('aggregate: computes pass_rate per skill', () => {
+  const run = makeRun('foo', [
+    { pass: true, duration_ms: 100, tokens_in: 10, tokens_out: 5 },
+    { pass: false, duration_ms: 200, tokens_in: 10, tokens_out: 5 },
+    { pass: true, duration_ms: 150, tokens_in: 10, tokens_out: 5 },
+  ]);
+  const b = aggregate([run]);
+  assert.equal(b.skills.length, 1);
+  assert.equal(b.skills[0].skill, 'foo');
+  assert.equal(b.skills[0].n, 3);
+  assert.equal(b.skills[0].pass_rate.toFixed(2), '0.67');
+  assert.equal(b.skills[0].mean_duration_ms, 150);
+});
+
+test('aggregate: ignores skipped cases', () => {
+  const run = makeRun('foo', [
+    { pass: true, duration_ms: 100, tokens_in: 1, tokens_out: 1 },
+    { pass: false, duration_ms: 0, tokens_in: 0, tokens_out: 0, error: 'skipped: llm needs -Live' },
+  ]);
+  const b = aggregate([run]);
+  assert.equal(b.skills[0].n, 1);
+  assert.equal(b.skills[0].pass_rate, 1);
+});
+
+test('detectRegressions: pass-rate drop >= 10pp flags', () => {
+  const baseline = { skills: [{ skill: 'foo', pass_rate: 1.0, mean_duration_ms: 100, mean_tokens_total: 50 }] };
+  const bench = { skills: [{ skill: 'foo', pass_rate: 0.85, mean_duration_ms: 100, mean_tokens_total: 50 }] };
+  const regs = detectRegressions(bench, baseline);
+  assert.equal(regs.length, 1);
+  assert.equal(regs[0].metric, 'pass_rate');
+  assert.ok(regs[0].delta_pp >= 10);
+});
+
+test('detectRegressions: latency factor >=1.5x flags; <1.5x does not', () => {
+  const baseline = { skills: [{ skill: 'foo', pass_rate: 1.0, mean_duration_ms: 100, mean_tokens_total: 50 }] };
+  const fast = { skills: [{ skill: 'foo', pass_rate: 1.0, mean_duration_ms: 140, mean_tokens_total: 50 }] };
+  const slow = { skills: [{ skill: 'foo', pass_rate: 1.0, mean_duration_ms: 160, mean_tokens_total: 50 }] };
+  assert.equal(detectRegressions(fast, baseline).length, 0);
+  assert.equal(detectRegressions(slow, baseline).length, 1);
+  assert.equal(detectRegressions(slow, baseline)[0].metric, 'mean_duration_ms');
+});
+
+test('detectRegressions: no-op when baseline missing the skill', () => {
+  const baseline = { skills: [{ skill: 'other', pass_rate: 1.0, mean_duration_ms: 100, mean_tokens_total: 50 }] };
+  const bench = { skills: [{ skill: 'foo', pass_rate: 0.0, mean_duration_ms: 1000, mean_tokens_total: 99999 }] };
+  assert.deepEqual(detectRegressions(bench, baseline), []);
+});

package/src/eval-runner.test.mjs

@@ -0,0 +1,69 @@
+// kushi v5.0.3 — eval-runner smoke tests.
+//
+// Asserts the runner script + schema + at least the canary evals files are
+// present and parseable. Does NOT exec pwsh (keeps the test suite OS-agnostic
+// and fast — the pwsh integration is covered by self-check D33 + the
+// `npm run eval:canary` smoke).
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+test('eval-runner: run-evals.ps1 ships in plugin/skills/eval/', () => {
+  const p = path.join(repoRoot, 'plugin/skills/eval/run-evals.ps1');
+  assert.ok(fs.existsSync(p), `missing: ${p}`);
+  const txt = fs.readFileSync(p, 'utf-8');
+  assert.match(txt, /param\(/, 'run-evals.ps1 has no [CmdletBinding] param block');
+  assert.match(txt, /\$Canary/, 'run-evals.ps1 missing -Canary flag');
+  assert.match(txt, /\$All/, 'run-evals.ps1 missing -All flag');
+  assert.match(txt, /\$UpdateBaseline/, 'run-evals.ps1 missing -UpdateBaseline flag');
+});
+
+test('eval-runner: evals.schema.json validates structurally', () => {
+  const p = path.join(repoRoot, 'plugin/skills/eval/evals.schema.json');
+  assert.ok(fs.existsSync(p), `missing: ${p}`);
+  const s = JSON.parse(fs.readFileSync(p, 'utf-8'));
+  assert.equal(s.title, 'Kushi per-skill evals file');
+  assert.ok(s.definitions && s.definitions.case);
+  assert.ok(s.definitions.case.required.includes('expected_assertions'));
+});
+
+test('eval-runner: every plugin/skills/<name>/ (except eval/) has evals/evals.json that parses', () => {
+  const skillsDir = path.join(repoRoot, 'plugin/skills');
+  const skills = fs.readdirSync(skillsDir, { withFileTypes: true })
+    .filter((d) => d.isDirectory() && d.name !== 'eval')
+    .map((d) => d.name);
+  const missing = [];
+  for (const skill of skills) {
+    const f = path.join(skillsDir, skill, 'evals/evals.json');
+    if (!fs.existsSync(f)) { missing.push(skill); continue; }
+    const obj = JSON.parse(fs.readFileSync(f, 'utf-8'));
+    assert.equal(obj.skill, skill, `${skill}: evals.json declares skill '${obj.skill}'`);
+    assert.ok(Array.isArray(obj.cases) && obj.cases.length >= 2, `${skill}: needs >=2 cases`);
+    for (const c of obj.cases) {
+      assert.ok(c.id && c.name && c.input !== undefined, `${skill}/${c.id}: missing core fields`);
+      assert.ok(Array.isArray(c.expected_assertions) && c.expected_assertions.length >= 1,
+        `${skill}/${c.id}: needs >=1 assertion`);
+      assert.ok(['script', 'llm'].includes(c.grader_type), `${skill}/${c.id}: bad grader_type`);
+    }
+  }
+  assert.deepEqual(missing, [], `skills missing evals/evals.json: ${missing.join(', ')}`);
+});
+
+test('eval-runner: canary subset includes the documented six skills', () => {
+  const expected = new Set([
+    'ask-project', 'bootstrap-project', 'refresh-project',
+    'link-entities', 'build-state', 'self-check',
+  ]);
+  const found = new Set();
+  for (const skill of expected) {
+    const f = path.join(repoRoot, 'plugin/skills', skill, 'evals/evals.json');
+    const obj = JSON.parse(fs.readFileSync(f, 'utf-8'));
+    if (obj.cases.some((c) => c.canary === true)) found.add(skill);
+  }
+  assert.deepEqual([...found].sort(), [...expected].sort());
+});