kiku-agent 1.3.1 → 1.3.2

package/server/agent/eval/modules/reporting/benchmarkPng.js

@@ -0,0 +1,487 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { deflateSync } from 'node:zlib';
+import { BENCHMARK_ROWS, ROADMAP, computeBenchmarkSummary } from './benchmark.js';
+
+const REPO_ROOT = fileURLToPath(new URL('../../../../', import.meta.url));
+
+const FONT = {
+  32: [0, 0, 0, 0, 0, 0, 0],
+  33: [4, 4, 4, 4, 4, 0, 4],
+  34: [10, 10, 10, 0, 0, 0, 0],
+  35: [10, 10, 31, 10, 31, 10, 10],
+  36: [4, 15, 20, 14, 5, 30, 4],
+  37: [24, 25, 2, 4, 8, 13, 11],
+  38: [12, 18, 20, 8, 21, 18, 13],
+  39: [4, 4, 8, 0, 0, 0, 0],
+  40: [2, 4, 8, 8, 8, 4, 2],
+  41: [8, 4, 2, 2, 2, 4, 8],
+  42: [0, 4, 21, 14, 21, 4, 0],
+  43: [0, 4, 4, 31, 4, 4, 0],
+  44: [0, 0, 0, 0, 12, 4, 8],
+  45: [0, 0, 0, 31, 0, 0, 0],
+  46: [0, 0, 0, 0, 12, 12, 0],
+  47: [1, 1, 2, 4, 8, 16, 16],
+  48: [14, 17, 19, 21, 25, 17, 14],
+  49: [4, 12, 4, 4, 4, 4, 14],
+  50: [14, 17, 1, 2, 4, 8, 31],
+  51: [31, 2, 4, 2, 1, 17, 14],
+  52: [2, 6, 10, 18, 31, 2, 2],
+  53: [31, 16, 30, 1, 1, 17, 14],
+  54: [6, 8, 16, 30, 17, 17, 14],
+  55: [31, 1, 2, 4, 8, 8, 8],
+  56: [14, 17, 17, 14, 17, 17, 14],
+  57: [14, 17, 17, 15, 1, 2, 12],
+  58: [0, 12, 12, 0, 12, 12, 0],
+  59: [0, 12, 12, 0, 12, 4, 8],
+  60: [2, 4, 8, 16, 8, 4, 2],
+  61: [0, 0, 31, 0, 31, 0, 0],
+  62: [8, 4, 2, 1, 2, 4, 8],
+  63: [14, 17, 1, 2, 4, 0, 4],
+  64: [14, 17, 27, 21, 23, 16, 14],
+  65: [14, 17, 17, 31, 17, 17, 17],
+  66: [30, 17, 17, 30, 17, 17, 30],
+  67: [14, 17, 16, 16, 16, 17, 14],
+  68: [28, 18, 17, 17, 17, 18, 28],
+  69: [31, 16, 16, 30, 16, 16, 31],
+  70: [31, 16, 16, 30, 16, 16, 16],
+  71: [14, 17, 16, 23, 17, 17, 14],
+  72: [17, 17, 17, 31, 17, 17, 17],
+  73: [14, 4, 4, 4, 4, 4, 14],
+  74: [7, 2, 2, 2, 2, 18, 12],
+  75: [17, 18, 20, 24, 20, 18, 17],
+  76: [16, 16, 16, 16, 16, 16, 31],
+  77: [17, 27, 21, 21, 17, 17, 17],
+  78: [17, 25, 25, 21, 19, 19, 17],
+  79: [14, 17, 17, 17, 17, 17, 14],
+  80: [30, 17, 17, 30, 16, 16, 16],
+  81: [14, 17, 17, 17, 21, 18, 13],
+  82: [30, 17, 17, 30, 20, 18, 17],
+  83: [14, 17, 16, 14, 1, 17, 14],
+  84: [31, 4, 4, 4, 4, 4, 4],
+  85: [17, 17, 17, 17, 17, 17, 14],
+  86: [17, 17, 17, 17, 17, 10, 4],
+  87: [17, 17, 17, 21, 21, 27, 17],
+  88: [17, 17, 10, 4, 10, 17, 17],
+  89: [17, 17, 10, 4, 4, 4, 4],
+  90: [31, 1, 2, 4, 8, 16, 31],
+  91: [14, 8, 8, 8, 8, 8, 14],
+  92: [16, 16, 8, 4, 2, 1, 1],
+  93: [14, 2, 2, 2, 2, 2, 14],
+  94: [4, 10, 17, 0, 0, 0, 0],
+  95: [0, 0, 0, 0, 0, 0, 31],
+  96: [8, 4, 2, 0, 0, 0, 0],
+  97: [0, 0, 14, 1, 15, 17, 15],
+  98: [16, 16, 30, 17, 17, 17, 30],
+  99: [0, 0, 14, 16, 16, 17, 14],
+  100: [1, 1, 15, 17, 17, 17, 15],
+  101: [0, 0, 14, 17, 30, 16, 14],
+  102: [6, 8, 8, 30, 8, 8, 8],
+  103: [0, 14, 17, 17, 15, 16, 14],
+  104: [16, 16, 30, 17, 17, 17, 17],
+  105: [4, 0, 12, 4, 4, 4, 14],
+  106: [2, 0, 6, 2, 2, 18, 12],
+  107: [16, 16, 18, 20, 24, 20, 18],
+  108: [12, 4, 4, 4, 4, 4, 14],
+  109: [0, 0, 26, 21, 21, 21, 21],
+  110: [0, 0, 30, 17, 17, 17, 17],
+  111: [0, 0, 14, 17, 17, 17, 14],
+  112: [0, 0, 30, 17, 17, 30, 16],
+  113: [0, 0, 15, 17, 17, 15, 1],
+  114: [0, 0, 13, 18, 16, 16, 16],
+  115: [0, 0, 15, 16, 14, 1, 30],
+  116: [8, 8, 30, 8, 8, 9, 6],
+  117: [0, 0, 17, 17, 17, 17, 15],
+  118: [0, 0, 17, 17, 17, 10, 4],
+  119: [0, 0, 17, 17, 21, 21, 10],
+  120: [0, 0, 17, 10, 4, 10, 17],
+  121: [0, 0, 17, 17, 15, 1, 14],
+  122: [0, 0, 31, 2, 4, 8, 31],
+  123: [2, 4, 4, 8, 4, 4, 2],
+  124: [4, 4, 4, 4, 4, 4, 4],
+  125: [8, 4, 4, 2, 4, 4, 8],
+  126: [0, 0, 0, 9, 21, 0, 0]
+};
+
+const C_BG = [11, 15, 25];
+const C_CARD = [22, 27, 46];
+const C_LINE = [45, 55, 85];
+const C_TEXT = [241, 245, 249];
+const C_MUTED = [148, 163, 184];
+const C_ROSE = [244, 63, 94];
+const C_GREEN = [52, 211, 153];
+const C_AMBER = [251, 191, 36];
+const C_RED = [248, 113, 113];
+const C_PURPLE = [165, 180, 252];
+const C_GRAY = [148, 163, 184];
+
+const PARETO_COLORS = { candidate: C_GREEN, baseline: C_AMBER, legacy: C_GRAY, live: C_ROSE };
+
+function asciiOnly(s) {
+  return String(s ?? '').split('').map(ch => {
+    const code = ch.charCodeAt(0);
+    if (code >= 32 && code <= 126) return ch;
+    if (code === 8594) return '>';
+    return ' ';
+  }).join('');
+}
+
+function charStep(scale) {
+  return 6 * scale;
+}
+
+function textWidth(s, scale) {
+  return asciiOnly(s).length * charStep(scale);
+}
+
+function wrapText(s, maxChars) {
+  const words = asciiOnly(s).split(' ').filter(w => w.length > 0);
+  const lines = [];
+  let cur = '';
+  for (const w of words) {
+    if ((cur + (cur ? ' ' : '') + w).length <= maxChars) {
+      cur = cur ? cur + ' ' + w : w;
+    } else {
+      if (cur) lines.push(cur);
+      if (w.length > maxChars) {
+        for (let i = 0; i < w.length; i += maxChars) lines.push(w.slice(i, i + maxChars));
+        cur = '';
+      } else {
+        cur = w;
+      }
+    }
+  }
+  if (cur) lines.push(cur);
+  if (!lines.length) lines.push('');
+  return lines;
+}
+
+function fitText(s, maxChars) {
+  const t = asciiOnly(s);
+  if (t.length <= maxChars) return t;
+  return t.slice(0, Math.max(0, maxChars - 1)) + '>';
+}
+
+function makeCanvas(w, h) {
+  return { w, h, buf: Buffer.alloc(w * h * 3, 0) };
+}
+
+function fillRect(cv, x, y, w, h, c) {
+  const x0 = Math.max(0, Math.round(x));
+  const y0 = Math.max(0, Math.round(y));
+  const x1 = Math.min(cv.w, Math.round(x + w));
+  const y1 = Math.min(cv.h, Math.round(y + h));
+  for (let yy = y0; yy < y1; yy++) {
+    for (let xx = x0; xx < x1; xx++) {
+      const o = (yy * cv.w + xx) * 3;
+      cv.buf[o] = c[0];
+      cv.buf[o + 1] = c[1];
+      cv.buf[o + 2] = c[2];
+    }
+  }
+}
+
+function drawChar(cv, ch, x, y, scale, c) {
+  const code = asciiOnly(ch).charCodeAt(0);
+  const glyph = FONT[code] || FONT[32];
+  for (let r = 0; r < 7; r++) {
+    const bits = glyph[r];
+    for (let col = 0; col < 5; col++) {
+      if (bits & (1 << (4 - col))) {
+        fillRect(cv, x + col * scale, y + r * scale, scale, scale, c);
+      }
+    }
+  }
+}
+
+function drawText(cv, s, x, y, scale, c) {
+  const t = asciiOnly(s);
+  for (let i = 0; i < t.length; i++) {
+    drawChar(cv, t[i], x + i * charStep(scale), y, scale, c);
+  }
+}
+
+function drawCircle(cv, cx, cy, r, c) {
+  const rr = Math.round(r);
+  for (let yy = -rr; yy <= rr; yy++) {
+    for (let xx = -rr; xx <= rr; xx++) {
+      if (xx * xx + yy * yy <= rr * rr) {
+        const px = Math.round(cx) + xx;
+        const py = Math.round(cy) + yy;
+        if (px >= 0 && py >= 0 && px < cv.w && py < cv.h) {
+          const o = (py * cv.w + px) * 3;
+          cv.buf[o] = c[0];
+          cv.buf[o + 1] = c[1];
+          cv.buf[o + 2] = c[2];
+        }
+      }
+    }
+  }
+}
+
+const CRC_TABLE = (() => {
+  const t = new Int32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    t[n] = c;
+  }
+  return t;
+})();
+
+function crc32(buf) {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 255] ^ (c >>> 8);
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+function chunk(type, data) {
+  const len = Buffer.alloc(4);
+  len.writeUInt32BE(data.length, 0);
+  const td = Buffer.from(type, 'ascii');
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(Buffer.concat([td, data])), 0);
+  return Buffer.concat([len, td, data, crc]);
+}
+
+function encodePng(cv) {
+  const stride = cv.w * 3 + 1;
+  const raw = Buffer.alloc(stride * cv.h, 0);
+  for (let y = 0; y < cv.h; y++) {
+    raw[y * stride] = 0;
+    cv.buf.copy(raw, y * stride + 1, y * cv.w * 3, (y + 1) * cv.w * 3);
+  }
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(cv.w, 0);
+  ihdr.writeUInt32BE(cv.h, 4);
+  ihdr[8] = 8;
+  ihdr[9] = 2;
+  ihdr[10] = 0;
+  ihdr[11] = 0;
+  ihdr[12] = 0;
+  const sig = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
+  return Buffer.concat([sig, chunk('IHDR', ihdr), chunk('IDAT', deflateSync(raw)), chunk('IEND', Buffer.alloc(0))]);
+}
+
+function formatObserved(row, values) {
+  const v = values[row.key];
+  if (v === null || v === undefined) return 'N-A';
+  if (row.format === 'percent') return (v * 100).toFixed(1) + '%';
+  if (row.format === 'cost') return '$' + Number(v).toFixed(3);
+  return Number(v).toFixed(2);
+}
+
+function statusOf(statusMap, key) {
+  const st = statusMap[key];
+  if (st === true) return { label: 'PASS', color: C_GREEN };
+  if (st === false) return { label: 'FAIL', color: C_RED };
+  return { label: 'N-A', color: C_MUTED };
+}
+
+export function renderBenchmarkPng(rows = [], summary = {}, opts = {}) {
+  const width = Math.max(900, Math.min(2400, Number(opts.width) || 1440));
+  const S = 2;
+  const ST = 3;
+  const margin = 48;
+  const W = width - margin * 2;
+  const lineH = 7 * S + 8;
+  const mathOnly = (opts.mode || 'full') === 'math';
+  const bench = summary && summary.benchmark && summary.benchmark.values ? summary.benchmark : computeBenchmarkSummary(rows, summary);
+  const values = bench.values;
+  const model = asciiOnly((summary && summary.model) || (rows && rows.model) || 'agent-model');
+  const stamp = new Date().toISOString().slice(0, 19).replace('T', ' ');
+
+  const colDim = Math.round(W * 0.24);
+  const colInd = Math.round(W * 0.35);
+  const colTgt = Math.round(W * 0.15);
+  const colObs = Math.round(W * 0.13);
+  const colSta = W - colDim - colInd - colTgt - colObs;
+  const maxDim = Math.floor(colDim / charStep(S)) - 1;
+  const maxInd = Math.floor(colInd / charStep(S)) - 1;
+  const maxTgt = Math.floor(colTgt / charStep(S)) - 1;
+  const maxObs = Math.floor(colObs / charStep(S)) - 1;
+
+  const tableRows = BENCHMARK_ROWS.map(r => {
+    const dim = wrapText(r.dimension, maxDim);
+    const ind = wrapText(r.indicator + '  [' + r.benchmark + ']', maxInd);
+    const lines = Math.max(dim.length, ind.length, 1);
+    return { def: r, observed: formatObserved(r, values), state: statusOf(bench.status, r.key), dim, ind, lines };
+  });
+
+  const failKeys = Object.keys(bench.failureDistribution || {});
+  const formulas = [
+    'Pass@k = E[ 1 - C(n-c,k) / C(n,k) ]   (n runs, c passes)',
+    'eta_traj = L* / max(L, L*)   (L* shortest path, L actual)',
+    'S_faith = sum I(Claim in K) / m   (K tool output, m claims)'
+  ];
+  const road = ROADMAP.map(p => ({
+    head: p.phase + ' ' + p.weeks + ': ' + p.title,
+    lines: p.actions.reduce((acc, a) => acc.concat(wrapText('- ' + a, Math.floor(W / charStep(S)) - 4)), [])
+  }));
+
+  const rowH = tableRows.map(r => r.lines * lineH + 18);
+  const tableH = 52 + rowH.reduce((a, b) => a + b, 0);
+  const paretoH = 430;
+  const failH = 60 + failKeys.length * 52;
+  const formulaH = 60 + formulas.length * 34;
+  const roadH = road.reduce((a, p) => a + 44 + p.lines.length * 30, 56);
+  const totalH = mathOnly
+    ? 330 + tableH + formulaH
+    : 390 + tableH + 40 + paretoH + 40 + failH + 40 + formulaH + 40 + roadH + 60;
+
+  const cv = makeCanvas(width, totalH);
+  fillRect(cv, 0, 0, width, totalH, C_BG);
+  let y = 0;
+  fillRect(cv, 0, y, width, 150, C_CARD);
+  fillRect(cv, 0, 150, width, 4, C_ROSE);
+  drawText(cv, 'Kiku Agent Benchmark', margin, y + 30, ST, C_TEXT);
+  drawText(cv, 'ISO-IEC 42001 + NIST AI RMF  |  model: ' + model, margin, y + 92, S, C_MUTED);
+  drawText(cv, stamp + ' UTC', width - margin - textWidth(stamp + ' UTC', S), y + 92, S, C_MUTED);
+  y = 190;
+
+  const drawSection = (title) => {
+    drawText(cv, title, margin, y, S, C_PURPLE);
+    y += 40;
+    return y;
+  };
+
+  drawSection('Benchmark Dashboard');
+  const tx = margin;
+  fillRect(cv, tx, y, W, tableH, C_CARD);
+  drawText(cv, 'Metric Dimension', tx + 12, y + 14, S, C_MUTED);
+  drawText(cv, 'Indicator [Benchmark]', tx + colDim + 12, y + 14, S, C_MUTED);
+  drawText(cv, 'Target', tx + colDim + colInd + 12, y + 14, S, C_MUTED);
+  drawText(cv, 'Observed', tx + colDim + colInd + colTgt + 12, y + 14, S, C_MUTED);
+  drawText(cv, 'Status', tx + colDim + colInd + colTgt + colObs + 12, y + 14, S, C_MUTED);
+  let ry = y + 52;
+  tableRows.forEach((r, idx) => {
+    if (idx > 0) fillRect(cv, tx, ry - 9, W, 2, C_LINE);
+    r.dim.forEach((ln, i) => drawText(cv, ln, tx + 12, ry + i * lineH, S, C_TEXT));
+    r.ind.forEach((ln, i) => drawText(cv, ln, tx + colDim + 12, ry + i * lineH, S, i === 0 ? C_TEXT : C_MUTED));
+    drawText(cv, fitText(r.def.target, maxTgt), tx + colDim + colInd + 12, ry, S, C_MUTED);
+    drawText(cv, fitText(r.observed, maxObs), tx + colDim + colInd + colTgt + 12, ry, S, C_TEXT);
+    const bx = tx + colDim + colInd + colTgt + colObs + 12;
+    fillRect(cv, bx, ry - 2, 78, 26, r.state.color);
+    drawText(cv, r.state.label, bx + 8, ry + 2, S, C_BG);
+    ry += rowH[idx];
+  });
+  y += tableH + 40;
+
+  if (!mathOnly) {
+  drawSection('Trajectory vs Cost Pareto Frontier');
+  const px = margin;
+  const py = y;
+  const pw = W;
+  const ph = paretoH - 60;
+  fillRect(cv, px, py, pw, ph + 40, C_CARD);
+  const ox = px + 90;
+  const oy = py + ph - 30;
+  const axW = pw - 150;
+  const axH = ph - 80;
+  fillRect(cv, ox, oy, axW, 2, C_LINE);
+  fillRect(cv, ox, oy - axH, 2, axH, C_LINE);
+  for (let g = 0; g <= 4; g++) {
+    const gx = ox + Math.round(axW * g / 4);
+    fillRect(cv, gx, oy - axH, 2, axH, [28, 34, 56]);
+    drawText(cv, '$' + (0.08 * g / 4).toFixed(2), gx - 24, oy + 10, S, C_MUTED);
+  }
+  for (let g = 0; g <= 3; g++) {
+    const gy = oy - Math.round(axH * g / 3);
+    fillRect(cv, ox, gy, axW, 2, [28, 34, 56]);
+    drawText(cv, String(40 + Math.round(60 * g / 3)), ox - 48, gy - 6, S, C_MUTED);
+  }
+  const capX = ox + Math.round(axW / 2) - Math.round(textWidth('Cost per task (USD) 0.00 - 0.08', S) / 2);
+  drawText(cv, 'Cost per task (USD) 0.00 - 0.08', capX, oy + 32, S, C_MUTED);
+  drawText(cv, 'Success %', ox - 48, py + 8, S, C_MUTED);
+  const dotX = c => ox + Math.round(Math.min(Math.max(Number(c) || 0, 0), 0.08) * (axW / 0.08));
+  const dotY = s => oy - Math.round(((Math.min(Math.max(Number(s) || 0, 40), 100) - 40) / 60) * axH);
+  (bench.pareto || []).forEach((p, i) => {
+    const cx = dotX(p.cost);
+    const cy = dotY(p.successRate);
+    drawCircle(cv, cx, cy, p.kind === 'live' ? 9 : 7, PARETO_COLORS[p.kind] || C_TEXT);
+    const label = p.label + ' (' + p.successRate + '%, $' + Number(p.cost).toFixed(3) + ')';
+    const lx = Math.min(cx + 14, px + pw - textWidth(label, S) - 8);
+    const ly = Math.max(py + 8, cy - 24 - (i % 2) * 20);
+    drawText(cv, label, lx, ly, S, PARETO_COLORS[p.kind] || C_TEXT);
+  });
+  y += paretoH + 40;
+  }
+
+  if (!mathOnly) {
+  drawSection('Failure Mode Distribution');
+  fillRect(cv, margin, y, W, failH - 20, C_CARD);
+  let fy = y + 16;
+  for (const key of failKeys) {
+    const f = bench.failureDistribution[key];
+    const pct = Math.round((f.rate || 0) * 1000) / 10;
+    drawText(cv, fitText(f.label, 30), margin + 12, fy, S, C_TEXT);
+    const barX = margin + 380;
+    const barW = W - 380 - 200;
+    fillRect(cv, barX, fy, barW, 20, C_LINE);
+    fillRect(cv, barX, fy, Math.round(barW * (f.rate || 0)), 20, C_ROSE);
+    drawText(cv, pct.toFixed(1) + '% (' + f.count + ')', barX + barW + 16, fy, S, C_MUTED);
+    fy += 52;
+  }
+  y += failH + 40;
+  }
+
+  drawSection('Formulas');
+  fillRect(cv, margin, y, W, formulaH - 20, C_CARD);
+  let my = y + 16;
+  for (const f of formulas) {
+    drawText(cv, fitText(f, Math.floor(W / charStep(S)) - 4), margin + 12, my, S, C_TEXT);
+    my += 34;
+  }
+  y += formulaH + 40;
+
+  if (!mathOnly) {
+  drawSection('Enhancement Roadmap');
+  fillRect(cv, margin, y, W, roadH - 16, C_CARD);
+  let dy = y + 16;
+  for (const p of road) {
+    drawText(cv, fitText(p.head, Math.floor(W / charStep(S)) - 4), margin + 12, dy, S, C_PURPLE);
+    dy += 34;
+    for (const ln of p.lines) {
+      drawText(cv, ln, margin + 28, dy, S, C_TEXT);
+      dy += 30;
+    }
+    dy += 10;
+  }
+  y += roadH + 40;
+  }
+
+  drawText(cv, 'kiku-agent eval | offline snapshot', margin, totalH - 36, S, C_MUTED);
+  return encodePng(cv);
+}
+
+export async function writeBenchmarkPng(report, outputPath, opts = {}) {
+  const meta = report.metadata || {};
+  const base = report.summary || {};
+  const summary = Object.assign({}, base, { model: base.model || meta.model });
+  const png = renderBenchmarkPng(report.rows || [], summary, opts);
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, png);
+  return outputPath;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  const { values } = parseArgs({
+    options: {
+      in: { type: 'string', default: 'artifacts/eval/live.json' },
+      out: { type: 'string', default: 'artifacts/eval/benchmark.png' },
+      width: { type: 'string', default: '1440' },
+      mode: { type: 'string', default: 'full' }
+    }
+  });
+  const inPath = path.resolve(REPO_ROOT, values.in);
+  const outPath = path.resolve(REPO_ROOT, values.out);
+  try {
+    const report = JSON.parse(await fs.readFile(inPath, 'utf8'));
+    await writeBenchmarkPng(report, outPath, { width: Number(values.width), mode: values.mode });
+  } catch (err) {
+    console.error(String((err && err.message) || err));
+    process.exit(1);
+  }
+}