kiku-agent 1.3.1 → 1.3.2

package/server/agent/eval/modules/core/cli.js

@@ -0,0 +1,453 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { cases, securityCases, allCases, validateCases } from '../evaluation/cases.js';
+import { DEFAULTS, REPO_ROOT, normalizeBaseUrl, normalizeTimeoutMs } from './config.js';
+import { loadReport, verifyCorpusFreshness, acquireRunLock } from './store.js';
+import { runSuite } from '../evaluation/runner.js';
+import { runWorkerMain } from '../evaluation/worker.js';
+import { judgeReport } from '../analysis/judge.js';
+import { applyGates } from '../analysis/gates.js';
+import { createSaver } from './store.js';
+import { diffReports } from '../analysis/compare.js';
+import { writeDashboardReport } from '../reporting/dashboard.js';
+import { writeBenchmarkPng } from '../reporting/benchmarkPng.js';
+import { runWebSmoke, runResearchSmoke, runTokenCalibration } from '../utils/smokes.js';
+import { spawnSync } from 'node:child_process';
+
+const SUBCOMMANDS = ['validate', 'run', 'judge', 'compare', 'dashboard', 'dashboard-png', 'dashboard-enterprise', 'smoke', 'calibrate', 'gate', 'analyze', 'ui', '__worker'];
+
+const HELP = `Kiku eval harness — single entry point (dev-only, never published to npm)
+
+Usage:
+  node server/agent/eval/modules/cli.js <command> [options]
+  npm run eval -- <command> [options]
+
+Commands:
+  validate                       Validate eval corpus (no LM Studio needed)
+  run [options]                  Run catalog/security suite with real agent + LM Studio
+  judge --out <f> --judge-model <id>   Grade an existing report with an LLM judge
+  compare --baseline <f> --candidate <f> [--json] [--fail-on-regression]
+  dashboard [--in <f>] [--out <f>] [--compare <f>]
+  dashboard-png [--in <f>] [--out <f>] [--width <n>]
+  dashboard-enterprise           Launch interactive Kiku Eval Studio web UI (primary)
+  smoke web|research             External-tool / research-orchestrator availability checks
+  calibrate [--model <id>]       Compare context estimator vs real prompt tokens
+  gate --in <f> [--gates-file <f>]     Apply gates.json thresholds to a report
+  analyze [--mode all|codebase|report] [--report <f>] [--out <f>] [--markdown-out <f>] [--skip-corpus]
+  ui [--port <n>] [--open]       Launch interactive Kiku Eval Studio web UI
+
+run options:
+  --suite catalog|security|all (default: catalog)  --ids p01,w01  --limit 3
+  --model <id>  --base-url <url>  --timeout-ms <n>  --out <file>  --resume
+  --judge-model <id>  --gate  --dashboard  --save-as <snapshot-name>
+
+Examples:
+  npm run eval:validate
+  npm run eval:quick
+  npm run eval:run -- --suite all --limit 5
+  npm run eval:judge -- --out artifacts/eval/live.json --judge-model google/gemma-4-e4b
+  npm run eval:compare -- --baseline a.json --candidate b.json
+  npm run eval:dashboard
+`;
+
+function opt(spec) {
+  return parseArgs({ args: spec, options: {
+    model: { type: 'string' },
+    'base-url': { type: 'string' },
+    'timeout-ms': { type: 'string' },
+    out: { type: 'string' },
+    in: { type: 'string' },
+    suite: { type: 'string' },
+    ids: { type: 'string' },
+    limit: { type: 'string' },
+    'judge-model': { type: 'string' },
+    resume: { type: 'boolean' },
+    'retry-failed': { type: 'boolean' },
+    'retry-errors': { type: 'boolean' },
+    from: { type: 'string' },
+    gate: { type: 'boolean' },
+    'gates-file': { type: 'string' },
+    dashboard: { type: 'boolean' },
+    'dashboard-out': { type: 'string' },
+    'save-as': { type: 'string' },
+    baseline: { type: 'string' },
+    candidate: { type: 'string' },
+    compare: { type: 'string' },
+    json: { type: 'boolean' },
+    'fail-on-regression': { type: 'boolean' },
+    id: { type: 'string' },
+    mode: { type: 'string' },
+    width: { type: 'string' },
+    report: { type: 'string' },
+    gates: { type: 'string' },
+    'markdown-out': { type: 'string' },
+    'skip-corpus': { type: 'boolean' },
+    root: { type: 'string' },
+    port: { type: 'string' },
+    open: { type: 'boolean' },
+    help: { type: 'boolean', short: 'h' },
+  }, allowPositionals: true });
+}
+
+async function cmdValidate() {
+  validateCases();
+  console.log(JSON.stringify({
+    valid: true,
+    count: allCases.length,
+    catalogCount: cases.length,
+    securityCount: securityCases.length,
+    categories: [...new Set(allCases.map(c => c.category))]
+  }));
+}
+
+async function cmdRun(raw) {
+  const { values } = opt(raw);
+  await runSuite({
+    suite: values.suite ?? DEFAULTS.suite,
+    ids: values.ids,
+    limit: values.limit,
+    model: values.model ?? DEFAULTS.model,
+    baseUrl: normalizeBaseUrl(values['base-url'] ?? DEFAULTS.baseUrl),
+    timeoutMs: normalizeTimeoutMs(values['timeout-ms'] ?? DEFAULTS.timeoutMs),
+    out: values.out ?? DEFAULTS.out,
+    from: values.from,
+    resume: values.resume ?? false,
+    retryFailed: values['retry-failed'] ?? false,
+    retryErrors: values['retry-errors'] ?? false,
+    judgeModel: values['judge-model'],
+    gate: values.gate ?? false,
+    gatesFile: values['gates-file'] ?? DEFAULTS.gatesFile,
+    saveAs: values['save-as'],
+    dashboard: values.dashboard ?? false,
+    dashboardOut: values['dashboard-out'] ?? DEFAULTS.dashboardOut,
+  });
+}
+
+async function cmdJudge(raw) {
+  const { values } = opt(raw);
+  const outPath = path.resolve(REPO_ROOT, values.out ?? DEFAULTS.out);
+  if (!values['judge-model']) throw new Error('judge requires --judge-model <id>');
+  const report = await loadReport(outPath);
+  await verifyCorpusFreshness(report);
+  const save = createSaver(outPath, report);
+  const lock = await acquireRunLock(outPath);
+  try {
+    await judgeReport(report,
+      { judgeModel: values['judge-model'], baseUrl: normalizeBaseUrl(values['base-url'] ?? DEFAULTS.baseUrl), timeoutMs: normalizeTimeoutMs(values['timeout-ms'] ?? DEFAULTS.timeoutMs) },
+      save);
+    await save();
+  } finally {
+    await lock.release();
+  }
+  console.log(JSON.stringify(report.summary, null, 2));
+}
+
+async function cmdCompare(raw) {
+  const { values } = opt(raw);
+  if (!values.baseline || !values.candidate) throw new Error('compare requires --baseline <file> --candidate <file>');
+  const base = JSON.parse(await fs.readFile(path.resolve(REPO_ROOT, values.baseline), 'utf8'));
+  const cand = JSON.parse(await fs.readFile(path.resolve(REPO_ROOT, values.candidate), 'utf8'));
+  const diff = diffReports(base, cand);
+  if (values.json) {
+    console.log(JSON.stringify(diff, null, 2));
+  } else {
+    console.log('=== Kiku Evaluation Report Comparison ===');
+    console.log('Baseline:', values.baseline);
+    console.log('Candidate:', values.candidate);
+    console.log('\n--- Metric Deltas ---');
+    for (const [m, data] of Object.entries(diff.metricDeltas)) {
+      const deltaSign = data.delta !== null && data.delta > 0 ? `+${data.delta}` : `${data.delta ?? '-'}`;
+      console.log(`${m.padEnd(25)}: ${String(data.baseline ?? '-').padEnd(10)} -> ${String(data.candidate ?? '-').padEnd(10)} (delta: ${deltaSign})`);
+    }
+    console.log('\n--- Case Diff Summary ---');
+    console.log(`Regressions : ${diff.caseDiffs.summary.regressionsCount}`);
+    console.log(`Improvements: ${diff.caseDiffs.summary.improvementsCount}`);
+    console.log(`Unchanged   : ${diff.caseDiffs.summary.unchangedCount}`);
+    if (diff.caseDiffs.regressions.length > 0) {
+      console.log('\n[REGRESSION] Regressions detected:');
+      for (const r of diff.caseDiffs.regressions) {
+        console.log(`  - [${r.id}] (${r.category}): base=${r.baseline.status}/matched=${r.baseline.allMatched}/tool=${r.baseline.toolCorrect} -> cand=${r.candidate.status}/matched=${r.candidate.allMatched}/tool=${r.candidate.toolCorrect}`);
+      }
+    }
+  }
+  if (values['fail-on-regression'] && diff.caseDiffs.regressions.length > 0) process.exit(2);
+}
+
+async function cmdDashboard(raw) {
+  const { values } = opt(raw);
+  const inPath = path.resolve(REPO_ROOT, values.in ?? DEFAULTS.out);
+  const outPath = path.resolve(REPO_ROOT, values.out ?? DEFAULTS.dashboardOut);
+  const report = JSON.parse(await fs.readFile(inPath, 'utf8'));
+  let compareReport = null;
+  if (values.compare) compareReport = JSON.parse(await fs.readFile(path.resolve(REPO_ROOT, values.compare), 'utf8'));
+  await writeDashboardReport(report, outPath, compareReport);
+
+  let lastMtime = 0;
+  try {
+    const st = await fs.stat(inPath);
+    lastMtime = st.mtimeMs;
+  } catch {}
+
+  const watcher = setInterval(async () => {
+    try {
+      const st = await fs.stat(inPath);
+      if (st.mtimeMs !== lastMtime) {
+        lastMtime = st.mtimeMs;
+        const freshRaw = await fs.readFile(inPath, 'utf8');
+        const freshReport = JSON.parse(freshRaw);
+        await writeDashboardReport(freshReport, outPath, compareReport);
+      }
+    } catch {}
+  }, 1000);
+  if (typeof watcher.unref === 'function') {
+    watcher.unref();
+  }
+
+  const shouldOpen = values.open !== false;
+  if (shouldOpen) {
+    let targetUrl = `file:///${outPath.replace(/\\/g, '/')}`;
+    try {
+      const ping3001 = await fetch('http://localhost:3001/api/eval/status', { signal: AbortSignal.timeout(600) });
+      if (ping3001.ok) {
+        targetUrl = 'http://localhost:3001/eval/dashboard';
+      } else {
+        const ping3000 = await fetch('http://localhost:3000/api/eval/status', { signal: AbortSignal.timeout(600) });
+        if (ping3000.ok) targetUrl = 'http://localhost:3000/eval/dashboard';
+      }
+    } catch {}
+    const safeUrl = targetUrl.replace(/["`$&|;<>^%]/g, '');
+    const openCmd = process.platform === 'win32'
+      ? `start "" "${safeUrl}"`
+      : (process.platform === 'darwin' ? `open "${safeUrl}"` : `xdg-open "${safeUrl}"`);
+    const cp = await import('node:child_process');
+    cp.exec(openCmd, () => {});
+  }
+}
+
+async function cmdDashboardPng(raw) {
+  const { values } = opt(raw);
+  const inPath = path.resolve(REPO_ROOT, values.in ?? DEFAULTS.out);
+  const outPath = path.resolve(REPO_ROOT, values.out ?? DEFAULTS.dashboardPngOut);
+  const report = JSON.parse(await fs.readFile(inPath, 'utf8'));
+  const width = Number(values.width ?? 1440) || 1440;
+  const mode = values.mode ?? 'full';
+  await writeBenchmarkPng(report, outPath, { width, mode });
+}
+
+async function cmdDashboardEnterprise(raw) {
+  return cmdUi(raw);
+}
+
+async function cmdSmoke(raw) {
+  const [target, ...rest] = raw;
+  const { values } = opt(rest);
+  if (target === 'web') {
+    await runWebSmoke({ out: values.out ?? DEFAULTS.webSmokeOut });
+  } else if (target === 'research') {
+    await runResearchSmoke({
+      out: values.out ?? DEFAULTS.researchSmokeOut,
+      model: values.model ?? DEFAULTS.model,
+      baseUrl: normalizeBaseUrl(values['base-url'] ?? DEFAULTS.baseUrl),
+      timeoutMs: normalizeTimeoutMs(values['timeout-ms'] ?? DEFAULTS.timeoutMs),
+    });
+  } else {
+    throw new Error('smoke requires a target: web | research');
+  }
+}
+
+async function cmdCalibrate(raw) {
+  const { values } = opt(raw);
+  await runTokenCalibration({
+    out: values.out ?? DEFAULTS.tokenCalibrationOut,
+    model: values.model ?? DEFAULTS.model,
+    baseUrl: normalizeBaseUrl(values['base-url'] ?? DEFAULTS.baseUrl),
+  });
+}
+
+async function cmdGate(raw) {
+  const { values } = opt(raw);
+  const inPath = path.resolve(REPO_ROOT, values.in ?? DEFAULTS.out);
+  const report = JSON.parse(await fs.readFile(inPath, 'utf8'));
+  const save = createSaver(inPath, report);
+  const lock = await acquireRunLock(inPath);
+  try {
+    const result = await applyGates(report, values['gates-file'] ?? DEFAULTS.gatesFile, save);
+    console.log(JSON.stringify(result, null, 2));
+    if (!result.passed) process.exitCode = 2;
+  } finally {
+    await lock.release();
+  }
+}
+
+async function cmdAnalyze(raw) {
+  const { values } = opt(raw);
+  const script = new URL('../../analyze.py', import.meta.url);
+  const scriptPath = fileURLToPath(script);
+  const args = [scriptPath, '--mode', values.mode ?? 'all'];
+  if (values.report) args.push('--report', values.report);
+  if (values.out) args.push('--out', values.out);
+  if (values['markdown-out']) args.push('--markdown-out', values['markdown-out']);
+  if (values.gates) args.push('--gates', values.gates);
+  if (values.root) args.push('--root', values.root);
+  if (values['skip-corpus']) args.push('--skip-corpus');
+  const candidates = process.platform === 'win32'
+    ? [['python', []], ['py', ['-3']], ['python3', []]]
+    : [['python3', []], ['python', []]];
+  let lastError = null;
+  for (const [bin, prefix] of candidates) {
+    const result = spawnSync(bin, [...prefix, ...args], { cwd: REPO_ROOT, stdio: 'inherit' });
+    if (result.error?.code === 'ENOENT') { lastError = result.error; continue; }
+    if (typeof result.status === 'number' && result.status !== 0) process.exit(result.status);
+    return;
+  }
+  throw new Error(`Python interpreter not found for analyze (${lastError?.message ?? 'no candidates'})`);
+}
+
+async function cmdUi(raw) {
+  const { values } = opt(raw);
+  const port = Number(values.port || 3001);
+  const shouldOpen = values.open ?? false;
+  const { createServer } = await import('node:http');
+  const { handleEvalApiRequest } = await import('../server/evalApi.js');
+
+  const server = createServer(async (req, res) => {
+    try {
+      const handled = await handleEvalApiRequest(req, res);
+      if (!handled) {
+        res.writeHead(302, { Location: '/eval' });
+        res.end();
+      }
+    } catch (err) {
+      res.writeHead(500, { 'Content-Type': 'text/plain' });
+      res.end('Server error: ' + err.message);
+    }
+  });
+
+  server.on('error', (err) => {
+    if (err.code === 'EADDRINUSE') {
+      if (shouldOpen) {
+        const url = `http://localhost:${port}/eval`;
+        const openCmd = process.platform === 'win32' ? `start "" "${url}"` : (process.platform === 'darwin' ? `open "${url}"` : `xdg-open "${url}"`);
+        import('node:child_process').then(cp => {
+          cp.exec(openCmd, () => {});
+        });
+      }
+      return;
+    }
+    throw err;
+  });
+
+  server.listen(port, () => {
+    const url = `http://localhost:${port}/eval`;
+    console.log(`[Eval Studio] Running at ${url}`);
+    if (shouldOpen) {
+      const openCmd = process.platform === 'win32' ? `start "" "${url}"` : (process.platform === 'darwin' ? `open "${url}"` : `xdg-open "${url}"`);
+      import('node:child_process').then(cp => {
+        cp.exec(openCmd, () => {});
+      });
+    }
+  });
+}
+
+async function cmdLegacy(argv) {
+  const { values } = parseArgs({ args: argv, options: {
+    worker: { type: 'string' },
+    out: { type: 'string', default: DEFAULTS.out },
+    suite: { type: 'string', default: DEFAULTS.suite },
+    dashboard: { type: 'boolean', default: false },
+    'dashboard-out': { type: 'string', default: DEFAULTS.dashboardOut },
+    ids: { type: 'string' },
+    limit: { type: 'string' },
+    model: { type: 'string', default: DEFAULTS.model },
+    'base-url': { type: 'string', default: DEFAULTS.baseUrl },
+    'timeout-ms': { type: 'string', default: String(DEFAULTS.timeoutMs) },
+    'judge-model': { type: 'string' },
+    'judge-only': { type: 'boolean', default: false },
+    resume: { type: 'boolean', default: false },
+    validate: { type: 'boolean', default: false },
+    gate: { type: 'boolean', default: false },
+    'gates-file': { type: 'string', default: DEFAULTS.gatesFile },
+    'save-as': { type: 'string' },
+  }, allowPositionals: true });
+
+  validateCases();
+  const timeoutMs = normalizeTimeoutMs(values['timeout-ms']);
+  const baseUrl = normalizeBaseUrl(values['base-url']);
+
+  if (values.validate) {
+    await cmdValidate();
+    return;
+  }
+  if (values.worker) {
+    await runWorkerMain({ caseId: values.worker, model: values.model, baseUrl, timeoutMs });
+    return;
+  }
+  await runSuite({
+    suite: values.suite,
+    ids: values.ids,
+    limit: values.limit,
+    model: values.model,
+    baseUrl,
+    timeoutMs,
+    out: values.out,
+    resume: values.resume,
+    judgeModel: values['judge-model'],
+    judgeOnly: values['judge-only'],
+    gate: values.gate,
+    gatesFile: values['gates-file'],
+    saveAs: values['save-as'],
+    dashboard: values.dashboard,
+    dashboardOut: values['dashboard-out'],
+  });
+}
+
+export async function runCli(argv = process.argv.slice(2)) {
+  const [first, ...rest] = argv;
+  if (!first || first === '--help' || first === '-h') {
+    console.log(HELP);
+    return;
+  }
+  if (rest.includes('--help') || rest.includes('-h')) {
+    console.log(HELP);
+    return;
+  }
+  if (!SUBCOMMANDS.includes(first)) {
+    await cmdLegacy(argv);
+    return;
+  }
+  if (first === '__worker') {
+    const { values } = opt(rest);
+    if (!values.id) throw new Error('__worker requires --id <caseId>');
+    await runWorkerMain({
+      caseId: values.id,
+      model: values.model ?? DEFAULTS.model,
+      baseUrl: normalizeBaseUrl(values['base-url'] ?? DEFAULTS.baseUrl),
+      timeoutMs: normalizeTimeoutMs(values['timeout-ms'] ?? DEFAULTS.timeoutMs),
+    });
+    return;
+  }
+  switch (first) {
+    case 'validate': await cmdValidate(); break;
+    case 'run': await cmdRun(rest); break;
+    case 'judge': await cmdJudge(rest); break;
+    case 'compare': await cmdCompare(rest); break;
+    case 'dashboard': await cmdDashboard(rest); break;
+    case 'dashboard-png': await cmdDashboardPng(rest); break;
+    case 'dashboard-enterprise': await cmdDashboardEnterprise(rest); break;
+    case 'smoke': await cmdSmoke(rest); break;
+    case 'calibrate': await cmdCalibrate(rest); break;
+    case 'gate': await cmdGate(rest); break;
+    case 'analyze': await cmdAnalyze(rest); break;
+    case 'ui': await cmdUi(rest); break;
+    default: throw new Error(`Unknown command: ${first}`);
+  }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  runCli(process.argv.slice(2)).catch(err => {
+    console.error(err);
+    process.exit(1);
+  });
+}

package/server/agent/eval/modules/core/config.js

@@ -0,0 +1,60 @@
+import { fileURLToPath } from 'node:url';
+
+export const REPO_ROOT = fileURLToPath(new URL('../../../../../', import.meta.url));
+
+export const DEFAULTS = {
+  model: 'google/gemma-4-e4b',
+  baseUrl: 'http://127.0.0.1:1234/v1',
+  timeoutMs: 240000,
+  workerGraceMs: 15000,
+  out: 'artifacts/eval/live.json',
+  suite: 'catalog',
+  dashboardOut: 'artifacts/eval/dashboard.html',
+  dashboardPngOut: 'artifacts/eval/benchmark.png',
+  gatesFile: 'server/agent/eval/gates.json',
+  snapshotsDir: 'artifacts/eval/snapshots',
+  webSmokeOut: 'artifacts/eval/web-smoke.json',
+  researchSmokeOut: 'artifacts/eval/research-smoke.json',
+  tokenCalibrationOut: 'artifacts/eval/token-calibration.json',
+};
+
+export const ALLOWED_TOOLS = [
+  'get_profile_and_fandom',
+  'get_quotes',
+  'search_works',
+  'get_work_detail',
+  'get_character_roles',
+  'search_discography',
+  'get_music_videos',
+  'get_media_links',
+  'get_brand_endorsements',
+  'get_career_timeline',
+  'get_awards',
+  'get_gallery_photos',
+  'global_search',
+];
+
+export const TRACKED_AGENT_FILES = [
+  'server/agent/agentCore.js',
+  'server/agent/llmClient.js',
+  'server/agent/config.js',
+  'server/agent/tools.js',
+  'server/agent/textToolProtocol.js',
+  'server/agent/contextCompactor.js',
+];
+
+export function normalizeBaseUrl(raw = DEFAULTS.baseUrl) {
+  const base = new URL(raw);
+  if (!['127.0.0.1', 'localhost', '[::1]'].includes(base.hostname) || base.username || base.password || base.search) {
+    throw new Error('This harness only supports local, credential-free LM Studio URLs');
+  }
+  return raw;
+}
+
+export function normalizeTimeoutMs(raw = DEFAULTS.timeoutMs) {
+  const timeoutMs = Number(raw);
+  if (!Number.isInteger(timeoutMs) || timeoutMs < 1000) {
+    throw new Error('timeout-ms must be an integer >= 1000');
+  }
+  return timeoutMs;
+}