kiku-agent 1.3.1 → 1.3.3

package/server/agent/eval/modules/core/store.js

@@ -0,0 +1,236 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { allCases } from '../evaluation/cases.js';
+import { summarize } from '../evaluation/scoring.js';
+import { REPO_ROOT, TRACKED_AGENT_FILES } from './config.js';
+
+const activeRunLocks = new Set();
+const STALE_LOCK_MS = 60 * 60 * 1000;
+
+function lockConflictError(message) {
+  const error = new Error(message);
+  error.status = 409;
+  return error;
+}
+
+export function corpusHash(cases = allCases) {
+  return createHash('sha256').update(JSON.stringify(cases)).digest('hex');
+}
+
+export async function hashFile(relPath) {
+  const data = await fs.readFile(path.join(REPO_ROOT, relPath));
+  return createHash('sha256').update(data).digest('hex');
+}
+
+export async function computeSourceHashes() {
+  const files = [...TRACKED_AGENT_FILES, ...new Set(allCases.flatMap(c => c.sources))];
+  const entries = await Promise.all(files.map(async f => [f, await hashFile(f)]));
+  return Object.fromEntries(entries);
+}
+
+export function gitRevision(cwd = REPO_ROOT) {
+  try {
+    return execFileSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf8' }).trim();
+  } catch {
+    return null;
+  }
+}
+
+export function buildMetadata({ model, baseUrl, timeoutMs, caseIds, extra = {} }) {
+  return {
+    startedAt: new Date().toISOString(),
+    node: process.version,
+    platform: process.platform,
+    model,
+    baseUrl,
+    revision: gitRevision(),
+    timeoutMs,
+    caseIds,
+    scope: 'Frozen repository catalog QA; no external fact verification, web, research, persistent memory, or interactive clarification. Fresh real KikuAgent and real LLMClient per question. Production temperature/output limits/iteration settings.',
+    scoring: 'Lexical fact coverage is a diagnostic only. Optional LLM judge is fallible and not human-validated. No measured population hallucination rate.',
+    ...extra,
+  };
+}
+
+export async function finalizeMetadata(metadata) {
+  metadata.corpusHash = corpusHash();
+  metadata.sourceHashes = await computeSourceHashes();
+  return metadata;
+}
+
+export async function verifyCorpusFreshness(report) {
+  const reportedIds = new Set(report.rows?.map(r => r.id) || report.metadata?.caseIds || []);
+  const relevantCases = allCases.filter(c => reportedIds.has(c.id));
+  const relevantSources = new Set((relevantCases.length ? relevantCases : allCases).flatMap(c => c.sources || []));
+  for (const source of relevantSources) {
+    const current = await hashFile(source);
+    if (report.metadata?.sourceHashes?.[source] && current !== report.metadata.sourceHashes[source]) {
+      throw new Error(`Reference source changed since inference: ${source}`);
+    }
+  }
+}
+
+export function assertResumeCompatible(prior, next) {
+  for (const field of ['model', 'baseUrl']) {
+    if (JSON.stringify(prior.metadata[field]) !== JSON.stringify(next.metadata[field])) {
+      throw new Error(`Resume mismatch: ${field}`);
+    }
+  }
+  if (prior.metadata.timeoutMs && next.metadata.timeoutMs && prior.metadata.timeoutMs !== next.metadata.timeoutMs) {
+    next.metadata.timeoutMs = prior.metadata.timeoutMs;
+  }
+  if (JSON.stringify(prior.metadata.corpusHash) !== JSON.stringify(next.metadata.corpusHash)) {
+    const priorIds = prior.metadata?.caseIds || prior.rows?.map(r => r.id) || [];
+    const nextIds = new Set(next.metadata?.caseIds || []);
+    const missing = priorIds.filter(id => !nextIds.has(id));
+    if (missing.length > 0) {
+      throw new Error(`Resume mismatch: corpusHash (missing cases: ${missing.slice(0, 5).join(', ')})`);
+    }
+  }
+}
+
+export async function loadReport(outPath) {
+  return JSON.parse(await fs.readFile(outPath, 'utf8'));
+}
+
+export async function loadReportWithBackups(outPath) {
+  for (const candidate of [outPath, `${outPath}.bak.1`, `${outPath}.bak`]) {
+    try {
+      const report = await loadReport(candidate);
+      if (Array.isArray(report?.rows) && report.rows.length > 0) return report;
+    } catch {}
+  }
+  return null;
+}
+
+export async function acquireRunLock(outPath, { staleMs = STALE_LOCK_MS } = {}) {
+  const key = `${path.resolve(outPath).toLowerCase()}`;
+  if (activeRunLocks.has(key)) {
+    throw lockConflictError(`Another evaluation is already writing ${outPath} in this process`);
+  }
+  const lockPath = `${path.resolve(outPath)}.lock`;
+  for (;;) {
+    try {
+      await fs.writeFile(lockPath, String(process.pid), { flag: 'wx' });
+      break;
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      let stale = true;
+      try {
+        const stat = await fs.stat(lockPath);
+        stale = (Date.now() - stat.mtimeMs) > staleMs;
+      } catch {}
+      if (!stale) {
+        throw lockConflictError(`Another evaluation run holds the lock for ${outPath}. If no run is active, delete ${lockPath}.`);
+      }
+      await fs.rm(lockPath, { force: true }).catch(() => {});
+    }
+  }
+  activeRunLocks.add(key);
+  let released = false;
+  return {
+    async touch() {
+      if (!released) await fs.utimes(lockPath, new Date(), new Date()).catch(() => {});
+    },
+    async release() {
+      if (released) return;
+      released = true;
+      activeRunLocks.delete(key);
+      await fs.rm(lockPath, { force: true }).catch(() => {});
+    },
+  };
+}
+
+export async function rotateBackups(filePath, maxKeep = 5) {
+  try {
+    const stat = await fs.stat(filePath).catch(() => null);
+    if (!stat || stat.size === 0) return;
+    for (let i = maxKeep - 1; i >= 1; i--) {
+      const src = `${filePath}.bak.${i}`;
+      const dst = `${filePath}.bak.${i + 1}`;
+      await fs.rename(src, dst).catch(() => {});
+    }
+    await fs.copyFile(filePath, `${filePath}.bak.1`).catch(() => {});
+  } catch {}
+}
+
+export async function archiveSessionReport(report, outPath) {
+  if (!report?.rows?.length) return null;
+  if (report.metadata?.archivedAt) return null;
+  try {
+    const historyDir = path.join(path.dirname(outPath), 'history');
+    await fs.mkdir(historyDir, { recursive: true });
+    const now = new Date();
+    const pad = n => String(n).padStart(2, '0');
+    const ms = String(now.getMilliseconds()).padStart(3, '0');
+    const timestamp = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}_${pad(now.getHours())}-${pad(now.getMinutes())}-${pad(now.getSeconds())}-${ms}`;
+    const planned = report.metadata?.caseIds?.length || report.rows.length;
+    const archive = {
+      ...report,
+      metadata: {
+        ...report.metadata,
+        archivedAt: now.toISOString(),
+        archivedPartial: report.rows.length < planned,
+      },
+    };
+    const archivePath = path.join(historyDir, `run_${timestamp}.json`);
+    await fs.writeFile(archivePath, JSON.stringify(archive, null, 2), 'utf8');
+    report.metadata.archivedAt = now.toISOString();
+
+    const files = await fs.readdir(historyDir);
+    const runFiles = files.filter(f => f.startsWith('run_') && f.endsWith('.json')).sort();
+    if (runFiles.length > 20) {
+      const toDelete = runFiles.slice(0, runFiles.length - 20);
+      for (const f of toDelete) {
+        await fs.unlink(path.join(historyDir, f)).catch(() => {});
+      }
+    }
+    return archivePath;
+  } catch {
+    return null;
+  }
+}
+
+export function createSaver(outPath, report) {
+  let unflushedResults = 0;
+  let lastFlushTime = Date.now();
+  let hasArchived = false;
+
+  return async function save(latestRow = null, options = {}) {
+    if (latestRow) unflushedResults++;
+    const shouldFlush = !latestRow || options.force || unflushedResults >= 5 || (Date.now() - lastFlushTime) >= 30000;
+    if (shouldFlush) {
+      report.summary = summarize(report.rows, report.metadata?.caseIds?.length || report.rows.length);
+      await fs.mkdir(path.dirname(outPath), { recursive: true });
+      await fs.writeFile(`${outPath}.tmp`, JSON.stringify(report, null, 2));
+      await fs.rename(`${outPath}.tmp`, outPath);
+      unflushedResults = 0;
+      lastFlushTime = Date.now();
+
+      if (report.metadata?.finishedAt && !hasArchived && report.rows?.length > 0) {
+        hasArchived = true;
+        await archiveSessionReport(report, outPath).catch(() => {});
+      }
+    }
+  };
+}
+
+export function blockedReportPathFor(outPath) {
+  const resolved = path.resolve(outPath);
+  return resolved.endsWith('.json') ? `${resolved.slice(0, -5)}.blocked.json` : `${resolved}.blocked.json`;
+}
+
+export async function writeBlockedReport(outPath, { reason, planned }) {
+  const blockedPath = blockedReportPathFor(outPath);
+  await fs.mkdir(path.dirname(blockedPath), { recursive: true });
+  await fs.writeFile(blockedPath, JSON.stringify({
+    status: 'blocked',
+    reason,
+    planned,
+    completed: 0,
+    writtenAt: new Date().toISOString(),
+  }, null, 2));
+  return blockedPath;
+}