kiku-agent 1.3.0 → 1.3.2

package/server/agent/eval/modules/evaluation/worker.js

@@ -0,0 +1,122 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { ALLOWED_TOOLS } from '../core/config.js';
+import { allCases, expandedCases } from './cases.js';
+import {
+  scoreCoverage,
+  evaluateToolCorrectness,
+  detectTrajectoryLoops,
+  computeLatencyBreakdown,
+  checkGroundedness,
+} from './scoring.js';
+
+const isConnectionError = err => {
+  const msg = (err?.message || String(err)).toLowerCase();
+  return msg.includes('fetch failed') ||
+    msg.includes('econnrefused') ||
+    msg.includes('etimedout') ||
+    msg.includes('socket hang up') ||
+    msg.includes('no models loaded') ||
+    msg.includes('network') ||
+    msg.includes('connection');
+};
+
+export async function runSingleCase(caseId, { model, baseUrl, timeoutMs, customQuestion }) {
+  const c = allCases.find(item => item.id === caseId) || (expandedCases && expandedCases.find(item => item.id === caseId));
+  if (!c) throw new Error('Unknown case');
+
+  process.env.KIKU_DATA_DIR = await fs.mkdtemp(path.join(os.tmpdir(), 'kiku-eval-'));
+  process.env.KIKU_AUTO_RESEARCH = 'off';
+  const [{ KikuAgent }, { AgentMemory }, { McpClientManager }, { ClarificationConfig, ClarificationAnalytics }] = await Promise.all([
+    import('../../../agentCore.js'), import('../../../memory.js'), import('../../../mcpClient.js'), import('../../../clarificationAnalytics.js'),
+  ]);
+  const dir = process.env.KIKU_DATA_DIR;
+  const clarifyConfig = new ClarificationConfig({ storagePath: path.join(dir, 'clarify.json') });
+  clarifyConfig.config.enabled = false;
+  const agent = new KikuAgent({
+    provider: 'lmstudio', baseUrl, model, apiKey: 'lm-studio',
+    memory: new AgentMemory({ persistence: false }), sessionPersistence: false,
+    mcpManager: new McpClientManager({ autoLoad: false, configPaths: [] }),
+    clarifyConfig, clarifyAnalytics: new ClarificationAnalytics({ storagePath: path.join(dir, 'analytics.json') }),
+    allowedToolNames: ALLOWED_TOOLS,
+  });
+  const trace = [];
+  let steps = 0;
+  const start = performance.now();
+  let result;
+  try {
+    await agent.ready();
+    const query = customQuestion || c.question;
+    let answer;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        trace.length = 0;
+        steps = 0;
+        answer = await agent.run(query, {
+          signal: AbortSignal.timeout(timeoutMs),
+          onStepStart: step => {
+            steps = step;
+            console.log('EVAL_STEP=' + JSON.stringify({ event: 'step_start', step }));
+          },
+          onToolStart: (name, parameters) => {
+            trace.push({ event: 'start', name, parameters });
+            console.log('EVAL_STEP=' + JSON.stringify({ event: 'tool_start', name, parameters }));
+          },
+          onToolResult: (name, value, durationMs) => {
+            trace.push({ event: 'result', name, value, durationMs });
+            console.log('EVAL_STEP=' + JSON.stringify({ event: 'tool_result', name, durationMs }));
+          },
+          onNotice: message => trace.push({ event: 'notice', message }),
+          onError: error => trace.push({ event: 'error', message: error.message }),
+        });
+        break;
+      } catch (err) {
+        if (attempt < 3 && isConnectionError(err)) {
+          console.log('EVAL_STEP=' + JSON.stringify({ event: 'retry', attempt, error: err.message }));
+          try {
+            const { wakeUpLmStudio } = await import('../../../lmStudioWakeup.js');
+            await wakeUpLmStudio({ baseUrl, model });
+          } catch {}
+          await new Promise(r => setTimeout(r, 2000 * attempt));
+          continue;
+        }
+        throw err;
+      }
+    }
+    result = {
+      id: c.id,
+      category: c.category,
+      language: c.language,
+      question: query,
+      status: 'completed',
+      answer,
+      coverage: scoreCoverage(c, answer),
+      toolCorrectness: evaluateToolCorrectness(c, trace),
+      trajectoryHealth: detectTrajectoryLoops(trace),
+      groundedness: checkGroundedness(c, answer, trace),
+      metrics: agent.lastQueryMetrics,
+      telemetry: agent.lastTelemetry
+    };
+  } catch (error) {
+    result = { id: c.id, category: c.category, language: c.language, question: c.question, status: 'error', error: error.message };
+  } finally {
+    await agent.mcpManager.disconnectAll();
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+  result.durationMs = Math.round(performance.now() - start);
+  result.dimension = c.dimension || null;
+  result.difficulty = c.difficulty || null;
+  result.judgeRequired = Boolean(c.judgeRequired);
+  result.steps = steps;
+  result.trace = trace;
+  result.latencyBreakdown = computeLatencyBreakdown(result);
+  return result;
+}
+
+export async function runWorkerMain({ caseId, model, baseUrl, timeoutMs }) {
+  const result = await runSingleCase(caseId, { model, baseUrl, timeoutMs });
+  console.log('EVAL_RESULT=' + JSON.stringify(result));
+  process.exitCode = result.status === 'completed' ? 0 : 1;
+  return result;
+}

package/server/agent/eval/modules/reporting/benchmark.js

@@ -0,0 +1,265 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const BENCHMARK_TARGETS = {
+  passAt1: { min: 0.75, label: '>= 75.0%' },
+  trajectoryOptimality: { min: 0.8, label: '>= 0.80' },
+  toolF1: { min: 0.9, label: '>= 0.90' },
+  hallucinationRate: { max: 0.05, label: '<= 5.0%' },
+  costPerTask: { max: 0.04, label: '<= $0.04 / task' }
+};
+
+export const BENCHMARK_ROWS = [
+  { dimension: 'Execution Capability', indicator: 'Task Success Rate (Pass@1)', benchmark: 'GAIA / SWE-bench', target: '>= 75.0%', key: 'passAt1', format: 'percent', iso: 'ISO/IEC 42001 9.2', nist: 'NIST AI RMF Measure 2.1' },
+  { dimension: 'Trajectory Efficiency', indicator: 'Step Optimality Ratio', benchmark: 'AgentBench', target: '>= 0.80', key: 'trajectoryOptimality', format: 'ratio', iso: 'ISO/IEC 42001 9.3', nist: 'NIST AI RMF Measure 2.4' },
+  { dimension: 'Tool Calling Precision', indicator: 'Tool Execution F1-Score', benchmark: 'ToolBench / ToolQA', target: '>= 0.90', key: 'toolF1', format: 'ratio', iso: 'ISO/IEC 42001 8.3', nist: 'NIST AI RMF Measure 2.2' },
+  { dimension: 'Grounded Faithfulness', indicator: 'Hallucination Index (H_rate)', benchmark: 'TruthfulQA / RAG Triad', target: '<= 5.0%', key: 'hallucinationRate', format: 'percent', invert: true, iso: 'ISO/IEC 42001 9.4', nist: 'NIST AI RMF Govern 4.2' },
+  { dimension: 'Operational Efficiency', indicator: 'Cost-to-Resolution Index (CRI)', benchmark: 'Custom Token/Sec SLA', target: '<= $0.04 / task', key: 'costPerTask', format: 'cost', invert: true, iso: 'ISO/IEC 42001 10.2', nist: 'NIST AI RMF Manage 1.3' }
+];
+
+export const FAILURE_MODES = [
+  { key: 'toolSchema', label: 'Tool Schema Error' },
+  { key: 'contextOverflow', label: 'Context Overflow' },
+  { key: 'replanning', label: 'Infinite Re-planning' },
+  { key: 'permissionTimeout', label: 'Permission / Timeout' }
+];
+
+export const ROADMAP = [
+  { phase: 'Phase 1', weeks: 'Weeks 1-3', title: 'Dynamic Trajectory Pruning & Error Recovery', actions: ['Cut re-planning loops when trajectory length exceeds 1.5 x L* with loop detection', 'Retry failed tool calls through a structured parser before resending the request'] },
+  { phase: 'Phase 2', weeks: 'Weeks 4-6', title: 'Deterministic State Machine Integration', actions: ['Split core logic into a Plan to Act to Verify to Output finite state machine', 'Verify output against ground truth rules before responding to push H_rate below 2.0%'] },
+  { phase: 'Phase 3', weeks: 'Weeks 7-8', title: 'Continuous Evals & Synthetic Regression Testing', actions: ['Run GAIA and SWE-bench aligned suites inside the CI pipeline', 'Generate synthetic edge cases covering safety and memory drift on every prompt update'] }
+];
+
+export const PARETO_REFERENCES = [
+  { label: 'Agent Model v2', successRate: 82.4, cost: 0.027, kind: 'candidate' },
+  { label: 'Baseline v1', successRate: 68.0, cost: 0.045, kind: 'baseline' },
+  { label: 'Legacy Script', successRate: 52.1, cost: 0.012, kind: 'legacy' }
+];
+
+export const COST_MODEL = {
+  perOutputToken: 0.00001,
+  perToolCall: 0.002,
+  perSecond: 0.001
+};
+
+export function binomial(n, k) {
+  const nn = Math.floor(Number(n));
+  const kk = Math.floor(Number(k));
+  if (!Number.isFinite(nn) || !Number.isFinite(kk) || nn < 0 || kk < 0 || kk > nn) return 0;
+  const kEff = Math.min(kk, nn - kk);
+  let result = 1;
+  for (let i = 1; i <= kEff; i++) {
+    result = result * (nn - kEff + i) / i;
+  }
+  return Math.round(result);
+}
+
+export function passAtK(n, c, k = 1) {
+  const nn = Math.floor(Number(n));
+  const cc = Math.floor(Number(c));
+  const kk = Math.floor(Number(k));
+  if (!Number.isFinite(nn) || !Number.isFinite(cc) || !Number.isFinite(kk) || nn <= 0 || kk <= 0) return null;
+  if (cc <= 0) return 0;
+  if (cc >= nn) return 1;
+  if (kk > nn) return Number((cc / nn).toFixed(4));
+  const total = binomial(nn, kk);
+  const fail = binomial(nn - cc, kk);
+  if (!total) return null;
+  return Number((1 - fail / total).toFixed(4));
+}
+
+export function trajectoryOptimality(lStar, l) {
+  const a = Number(lStar);
+  const b = Number(l);
+  if (!Number.isFinite(a) || !Number.isFinite(b) || a <= 0 || b <= 0) return null;
+  return Number((a / Math.max(b, a)).toFixed(4));
+}
+
+export function faithfulnessScore(supported, total) {
+  const s = Number(supported);
+  const m = Number(total);
+  if (!Number.isFinite(s) || !Number.isFinite(m) || m <= 0 || s < 0) return null;
+  return Number((Math.min(s, m) / m).toFixed(4));
+}
+
+export function toolF1(precision, recall) {
+  const p = Number(precision);
+  const r = Number(recall);
+  if (!Number.isFinite(p) || !Number.isFinite(r) || p < 0 || r < 0) return null;
+  if (p + r <= 0) return 0;
+  return Number(((2 * p * r) / (p + r)).toFixed(4));
+}
+
+export function rowOptimalPath(row) {
+  const expected = Array.isArray(row?.toolCorrectness?.expected) ? row.toolCorrectness.expected.length : 0;
+  const groups = Number(row?.coverage?.total ?? 0);
+  const fallback = Number.isFinite(groups) && groups > 0 ? groups : 1;
+  const lStar = expected > 0 ? expected : fallback;
+  return Math.max(1, lStar);
+}
+
+export function rowActualSteps(row) {
+  const trace = Array.isArray(row?.trace) ? row.trace : [];
+  const starts = trace.filter(t => t && t.event === 'start').length;
+  if (starts > 0) return starts;
+  if (row && row.status === 'completed') return 1;
+  return 0;
+}
+
+export function estimateRowCost(row, model = COST_MODEL) {
+  const trace = Array.isArray(row?.trace) ? row.trace : [];
+  const toolCalls = trace.filter(t => t && t.event === 'start').length;
+  const seconds = Number.isFinite(row?.durationMs) ? Math.max(0, row.durationMs) / 1000 : 0;
+  let outputTokens = row?.latencyBreakdown?.outputTokens;
+  if (!Number.isFinite(outputTokens)) {
+    const answerLen = typeof row?.answer === 'string' ? row.answer.length : 0;
+    outputTokens = Math.round(answerLen / 4);
+  }
+  const cost = outputTokens * model.perOutputToken + toolCalls * model.perToolCall + seconds * model.perSecond;
+  return Number(cost.toFixed(5));
+}
+
+export function failureModeOfRow(row) {
+  if (!row || typeof row !== 'object') return null;
+  const healthy = row.status === 'completed' && row.coverage?.allMatched === true && row.trajectoryHealth?.hasLoop !== true;
+  if (healthy) return null;
+  const trace = Array.isArray(row.trace) ? row.trace : [];
+  const blob = JSON.stringify({ error: row.error ?? null, trace: trace.slice(0, 40) }).toLowerCase();
+  const errText = String(row.error ?? '').toLowerCase();
+  const timeoutHit = errText.includes('timeout') || errText.includes('permission') || errText.includes('denied') || errText.includes('unauthorized') || errText.includes('abort') || blob.includes('timeout') && row.status === 'error';
+  if (timeoutHit) return 'permissionTimeout';
+  const lStar = rowOptimalPath(row);
+  const steps = rowActualSteps(row);
+  const loopCount = Number(row.trajectoryHealth?.loopCount ?? 0);
+  const thrashing = row.trajectoryHealth?.isThrashing === true;
+  if (thrashing || loopCount >= 2 || (steps > 0 && steps > 1.5 * lStar && steps >= 3)) return 'replanning';
+  const schemaHit = blob.includes('schema') || blob.includes('invalid') || blob.includes('validation') || blob.includes('parse') || trace.some(t => t && t.event === 'error');
+  const toolMiss = row.toolCorrectness && row.toolCorrectness.notApplicable !== true && (Number(row.toolCorrectness.precision ?? 1) < 0.5 || (row.toolCorrectness.missing ?? []).length > 0);
+  if (schemaHit || toolMiss) return 'toolSchema';
+  const answerLen = typeof row.answer === 'string' ? row.answer.length : 0;
+  const tooLong = steps > 6 || answerLen > 4000 || (Number(row.durationMs) > 60000);
+  if (tooLong) return 'contextOverflow';
+  if (row.status === 'error') return 'permissionTimeout';
+  if (row.coverage?.allMatched === false) return 'toolSchema';
+  return 'contextOverflow';
+}
+
+export function meanFinite(values) {
+  const list = (Array.isArray(values) ? values : []).filter(v => Number.isFinite(v));
+  if (!list.length) return null;
+  return Number((list.reduce((a, b) => a + b, 0) / list.length).toFixed(4));
+}
+
+export function round4(v) {
+  if (v === null || v === undefined) return null;
+  const n = Number(v);
+  if (!Number.isFinite(n)) return null;
+  return Number(n.toFixed(4));
+}
+
+export function checkTarget(key, value) {
+  const rule = BENCHMARK_TARGETS[key];
+  if (!rule || value === null || value === undefined || !Number.isFinite(Number(value))) return null;
+  const v = Number(value);
+  if (rule.min !== undefined) return v >= rule.min;
+  if (rule.max !== undefined) return v <= rule.max;
+  return null;
+}
+
+export function computeBenchmarkSummary(rows = [], summary = {}) {
+  const list = Array.isArray(rows) ? rows : [];
+  const completed = list.filter(r => r && r.status === 'completed');
+  const coverageRows = completed.filter(r => r.coverage && Number(r.coverage.total) > 0);
+  const passCount = coverageRows.filter(r => {
+    if (r.judgeResult && !r.judgeResult.judgeError) {
+      return r.judgeResult.score === 1.0 || r.judgeResult.verdict === 'PASS';
+    }
+    return r.coverage.allMatched === true;
+  }).length;
+  const passAt1 = coverageRows.length ? round4(passCount / coverageRows.length) : null;
+  const etas = completed.map(r => {
+    const steps = rowActualSteps(r);
+    if (!steps) return null;
+    return trajectoryOptimality(rowOptimalPath(r), steps);
+  }).filter(v => v !== null);
+  const trajectory = meanFinite(etas);
+  const f1s = completed.map(r => (r.toolCorrectness && r.toolCorrectness.notApplicable !== true && Number.isFinite(r.toolCorrectness.f1) ? r.toolCorrectness.f1 : null)).filter(v => v !== null);
+  const toolScore = meanFinite(f1s);
+  const supports = completed.map(r => {
+    if (!r.groundedness || r.groundedness.notApplicable === true) return null;
+    if (Number.isFinite(r.groundedness.supportRatio)) return r.groundedness.supportRatio;
+    return r.groundedness.isGrounded === true ? 1 : 0;
+  }).filter(v => v !== null);
+  const faith = meanFinite(supports);
+  const hallucinationRate = faith === null ? null : round4(1 - faith);
+  const costs = completed.map(r => estimateRowCost(r));
+  const costPerTask = costs.length ? round4(costs.reduce((a, b) => a + b, 0) / costs.length) : null;
+  const counts = { toolSchema: 0, contextOverflow: 0, replanning: 0, permissionTimeout: 0 };
+  let failures = 0;
+  for (const r of list) {
+    const mode = failureModeOfRow(r);
+    if (mode && counts[mode] !== undefined) {
+      counts[mode] += 1;
+      failures += 1;
+    }
+  }
+  const distribution = {};
+  for (const m of FAILURE_MODES) {
+    distribution[m.key] = {
+      label: m.label,
+      count: counts[m.key] ?? 0,
+      rate: failures ? round4((counts[m.key] ?? 0) / failures) : 0
+    };
+  }
+  const livePoint = passAt1 !== null && costPerTask !== null ? { label: 'Live run', successRate: round4(passAt1 * 100), cost: costPerTask, kind: 'live' } : null;
+  const values = { passAt1, trajectoryOptimality: trajectory, toolF1: toolScore, hallucinationRate, costPerTask };
+  const status = {};
+  for (const k of Object.keys(values)) status[k] = checkTarget(k, values[k]);
+  return {
+    values,
+    status,
+    targets: BENCHMARK_TARGETS,
+    failureDistribution: distribution,
+    failureTotal: failures,
+    pareto: livePoint ? [...PARETO_REFERENCES, livePoint] : [...PARETO_REFERENCES],
+    livePoint,
+    denominators: {
+      completed: completed.length,
+      coverageEligible: coverageRows.length,
+      toolEligible: f1s.length,
+      groundedEligible: supports.length,
+      costSamples: costs.length
+    }
+  };
+}
+
+export function benchmarkGatesReport(rows = [], summary = {}) {
+  const bench = computeBenchmarkSummary(rows, summary);
+  const results = BENCHMARK_ROWS.map(r => {
+    const actual = bench.values[r.key] ?? null;
+    const rule = BENCHMARK_TARGETS[r.key] ?? {};
+    const pass = checkTarget(r.key, actual);
+    return { metric: r.key, dimension: r.dimension, threshold: rule, actual, pass };
+  });
+  return { passed: results.every(r => r.pass === true), results, benchmark: bench };
+}
+
+export async function writeBenchmarkSnapshot(rows, summary, outputPath) {
+  const bench = computeBenchmarkSummary(rows, summary);
+  const { default: fs } = await import('node:fs/promises');
+  const dir = path.dirname(outputPath);
+  await fs.mkdir(dir, { recursive: true });
+  await fs.writeFile(outputPath, JSON.stringify({ generatedAt: new Date().toISOString(), ...bench }, null, 2), 'utf8');
+  return outputPath;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  const mod = await import('../core/store.js');
+  const root = fileURLToPath(new URL('../../../../', import.meta.url));
+  const inPath = path.resolve(root, process.argv[2] || 'artifacts/eval/live.json');
+  const { default: fs } = await import('node:fs/promises');
+  const raw = await fs.readFile(inPath, 'utf8');
+  const report = JSON.parse(raw);
+  const bench = computeBenchmarkSummary(report.rows || [], report.summary || {});
+  console.log(JSON.stringify(bench, null, 2));
+}