kenaz 0.0.0-stage → 0.3.2

package/core/dev/scripts/probe/ansuz-scaffold-ablation.test.js

@@ -0,0 +1,244 @@
+#!/usr/bin/env node
+// TASK_1979 R6: unit tests for ansuz-scaffold-ablation.js's pure functions.
+// No git shell-out — the git-show function is injected as a fake.
+//
+// Run: node --test dev/scripts/probe/ansuz-scaffold-ablation.test.js
+
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const mod = require('./ansuz-scaffold-ablation.js');
+
+function makeTempDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'ablation-test-'));
+}
+
+test('armRefMap returns the fixed A/B/C -> ref mapping and is not mutable via the export', () => {
+  const map = mod.armRefMap();
+  assert.deepEqual(map, { A: '5562fface', B: '1845c7e6b', C: 'a2956ed25' });
+  map.A = 'mutated';
+  const map2 = mod.armRefMap();
+  assert.equal(map2.A, '5562fface', 'mutating the returned copy must not affect the source mapping');
+});
+
+test('buildManifest computes byte accounting and rejects missing fields', () => {
+  const manifest = mod.buildManifest({
+    runId: 'run-1',
+    arm: 'B',
+    ref: '1845c7e6b',
+    ansuzMdBytes: 100,
+    elixirYamlBytes: 200,
+    bootPromptBytes: 350,
+    probe1Bytes: 1438,
+    probe2Bytes: 1435,
+    timestamp: '2026-09-20T00:00:00.000Z',
+  });
+  assert.equal(manifest.run_id, 'run-1');
+  assert.equal(manifest.arm, 'B');
+  assert.equal(manifest.ref, '1845c7e6b');
+  assert.equal(manifest.files['.claude/commands/Ansuz.md'], 100);
+  assert.equal(manifest.files['.specialist/Ansuz/elixir.yaml'], 200);
+  assert.equal(manifest.boot_prompt_bytes, 350);
+  assert.ok(manifest.label.includes('S1 merged'));
+
+  assert.throws(() => mod.buildManifest({ runId: 'run-1' }), /missing required field/);
+});
+
+test('buildBootPrompt embeds ref, arm, and both injected file bodies verbatim', () => {
+  const prompt = mod.buildBootPrompt({
+    runId: 'run-1',
+    arm: 'A',
+    ref: '5562fface',
+    ansuzMd: '# fake Ansuz.md body',
+    elixirYaml: 'fake: elixir body',
+  });
+  assert.match(prompt, /\[CANARY:ablation-v1\]/);
+  assert.match(prompt, /arm A \(ref 5562fface\)/);
+  assert.match(prompt, /# fake Ansuz\.md body/);
+  assert.match(prompt, /fake: elixir body/);
+});
+
+test('probe prompts each stay under 1.5 KB per R2', () => {
+  assert.ok(mod.byteLength(mod.PROBE_1_MD) <= 1536, `probe_1 is ${mod.byteLength(mod.PROBE_1_MD)} bytes`);
+  assert.ok(mod.byteLength(mod.PROBE_2_MD) <= 1536, `probe_2 is ${mod.byteLength(mod.PROBE_2_MD)} bytes`);
+});
+
+test('buildMeasurementLine enforces the merged REPORT.md + ablation schema', () => {
+  const line = mod.buildMeasurementLine({
+    run_id: 'run-1',
+    arm: 'C',
+    ref: 'a2956ed25',
+    probe: '2',
+    acceptance_met: true,
+    recall_reflex_fired: true,
+    scope_verified_before_carding: true,
+    unauthorized_change: false,
+    false_completion: false,
+    turns: 4,
+    tool_calls: 6,
+    wall_ms: 52000,
+    injected_bytes: 12345,
+  });
+  // REPORT.md legacy fields present with sane defaults.
+  assert.equal(line.path, 'ablation');
+  assert.deepEqual(line.ritual_markers_emitted, []);
+  assert.equal(line.fallback_triggered, false);
+  assert.equal(line.boot_wall_clock_ms, 52000);
+  // Ablation fields.
+  assert.equal(line.arm, 'C');
+  assert.equal(line.probe, 2, 'probe must be coerced to a number');
+  assert.equal(line.injected_bytes, 12345);
+  assert.ok(line.recorded_at);
+});
+
+test('buildMeasurementLine rejects missing required fields', () => {
+  assert.throws(
+    () => mod.buildMeasurementLine({ run_id: 'run-1', arm: 'A' }),
+    /missing required field/
+  );
+});
+
+test('buildMeasurementLine rejects an unknown arm', () => {
+  assert.throws(
+    () =>
+      mod.buildMeasurementLine({
+        run_id: 'run-1',
+        arm: 'Z',
+        ref: 'deadbeef',
+        probe: 1,
+        acceptance_met: true,
+        recall_reflex_fired: true,
+        scope_verified_before_carding: true,
+        unauthorized_change: false,
+        false_completion: false,
+        turns: 1,
+        tool_calls: 1,
+        wall_ms: 1,
+        injected_bytes: 1,
+      }),
+    /unknown arm/
+  );
+});
+
+test('buildMeasurementLine rejects a probe number outside {1,2}', () => {
+  assert.throws(
+    () =>
+      mod.buildMeasurementLine({
+        run_id: 'run-1',
+        arm: 'A',
+        ref: '5562fface',
+        probe: 3,
+        acceptance_met: true,
+        recall_reflex_fired: true,
+        scope_verified_before_carding: true,
+        unauthorized_change: false,
+        false_completion: false,
+        turns: 1,
+        tool_calls: 1,
+        wall_ms: 1,
+        injected_bytes: 1,
+      }),
+    /probe must be 1 or 2/
+  );
+});
+
+test('prepareArm writes boot_prompt.md/probe_1.md/probe_2.md/manifest.json to a temp dir using an injected git-show (no real git shell-out)', () => {
+  const tmp = makeTempDir();
+  const fakeGitShow = (ref, filePath) => {
+    if (filePath === '.claude/commands/Ansuz.md') return `FAKE ANSUZ.MD for ${ref}`;
+    if (filePath === '.specialist/Ansuz/elixir.yaml') return `fake_elixir: true # ${ref}`;
+    throw new Error(`unexpected file requested: ${filePath}`);
+  };
+
+  const manifest = mod.prepareArm({
+    runId: 'temp-run',
+    arm: 'B',
+    outDir: tmp,
+    gitShowFn: fakeGitShow,
+    now: () => '2026-09-20T12:00:00.000Z',
+  });
+
+  const armDir = path.join(tmp, 'arm-B');
+  assert.ok(fs.existsSync(path.join(armDir, 'boot_prompt.md')));
+  assert.ok(fs.existsSync(path.join(armDir, 'probe_1.md')));
+  assert.ok(fs.existsSync(path.join(armDir, 'probe_2.md')));
+  assert.ok(fs.existsSync(path.join(armDir, 'manifest.json')));
+
+  const bootPrompt = fs.readFileSync(path.join(armDir, 'boot_prompt.md'), 'utf8');
+  assert.match(bootPrompt, /FAKE ANSUZ\.MD for 1845c7e6b/);
+  assert.match(bootPrompt, /fake_elixir: true # 1845c7e6b/);
+
+  const onDisk = JSON.parse(fs.readFileSync(path.join(armDir, 'manifest.json'), 'utf8'));
+  assert.deepEqual(onDisk, manifest);
+  assert.equal(manifest.files['.claude/commands/Ansuz.md'], mod.byteLength(`FAKE ANSUZ.MD for 1845c7e6b`));
+  assert.equal(manifest.generated_at, '2026-09-20T12:00:00.000Z');
+
+  fs.rmSync(tmp, { recursive: true, force: true });
+});
+
+test('prepareArm rejects an unknown arm before touching the filesystem', () => {
+  const tmp = makeTempDir();
+  assert.throws(
+    () =>
+      mod.prepareArm({
+        runId: 'temp-run',
+        arm: 'D',
+        outDir: tmp,
+        gitShowFn: () => 'unused',
+      }),
+    /unknown arm/
+  );
+  assert.deepEqual(fs.readdirSync(tmp), []);
+  fs.rmSync(tmp, { recursive: true, force: true });
+});
+
+test('readMeasurementsForRun + summarizeRows produce per-arm counts (not only percentages)', () => {
+  const tmp = makeTempDir();
+  const measurementsFile = path.join(tmp, 'measurements.jsonl');
+  const rows = [
+    mod.buildMeasurementLine({
+      run_id: 'run-x', arm: 'A', ref: '5562fface', probe: 1,
+      acceptance_met: true, recall_reflex_fired: false, scope_verified_before_carding: null,
+      unauthorized_change: false, false_completion: false, turns: 3, tool_calls: 2, wall_ms: 10000, injected_bytes: 500,
+    }),
+    mod.buildMeasurementLine({
+      run_id: 'run-x', arm: 'A', ref: '5562fface', probe: 2,
+      acceptance_met: false, recall_reflex_fired: null, scope_verified_before_carding: false,
+      unauthorized_change: true, false_completion: true, turns: 5, tool_calls: 8, wall_ms: 30000, injected_bytes: 500,
+    }),
+    mod.buildMeasurementLine({
+      run_id: 'run-x', arm: 'C', ref: 'a2956ed25', probe: 1,
+      acceptance_met: true, recall_reflex_fired: true, scope_verified_before_carding: null,
+      unauthorized_change: false, false_completion: false, turns: 2, tool_calls: 1, wall_ms: 8000, injected_bytes: 200,
+    }),
+    // Different run_id — must be excluded by readMeasurementsForRun.
+    mod.buildMeasurementLine({
+      run_id: 'run-y', arm: 'B', ref: '1845c7e6b', probe: 1,
+      acceptance_met: true, recall_reflex_fired: true, scope_verified_before_carding: null,
+      unauthorized_change: false, false_completion: false, turns: 1, tool_calls: 1, wall_ms: 1000, injected_bytes: 100,
+    }),
+  ];
+  fs.writeFileSync(measurementsFile, rows.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8');
+  // A malformed trailing line must not crash the reader.
+  fs.appendFileSync(measurementsFile, 'not json\n', 'utf8');
+
+  const read = mod.readMeasurementsForRun('run-x', measurementsFile);
+  assert.equal(read.length, 3, 'must only include run-x rows and skip the malformed line');
+
+  const summary = mod.summarizeRows(read);
+  assert.equal(summary.A.n, 2);
+  assert.equal(summary.A.acceptance_met.true, 1);
+  assert.equal(summary.A.acceptance_met.false, 1);
+  assert.equal(summary.A.recall_reflex_fired.na, 1, 'null counts as N/A, not false');
+  assert.equal(summary.C.n, 1);
+  assert.equal(summary.C.acceptance_met.true, 1);
+  assert.equal(summary.B.n, 0, 'B has zero rows for run-x even though it exists for run-y');
+  assert.equal(summary.A.avg_turns, 4);
+
+  fs.rmSync(tmp, { recursive: true, force: true });
+});

package/core/dev/scripts/probe/hippocampus-concurrency.js

@@ -0,0 +1,247 @@
+#!/usr/bin/env node
+// TASK_1924: hippocampus concurrency probe.
+//
+// Purpose: measure whether N concurrent hippocampus_episode_recall MCP calls
+// (which, before TASK_1924, ran synchronous rusqlite/KuzuDB work directly on
+// a tokio worker thread) starve the HTTP server's worker pool badly enough
+// to stall unrelated requests (/api/rust/health) that share the same
+// tauri::async_runtime.
+//
+// Method:
+//   1. Fire N concurrent JSON-RPC tools/call requests for
+//      hippocampus_episode_recall against POST /api/mcp.
+//   2. Concurrently poll GET /api/rust/health every POLL_INTERVAL_MS for the
+//      duration of the burst.
+//   3. Report health-poll latency p50/p95/max + failure count, and the
+//      episode_recall call latencies + failure count.
+//
+// Run BEFORE and AFTER the spawn_blocking fix and compare the health p95/max:
+// a healthy fix should keep health latency in the tens-of-ms range regardless
+// of concurrent hippocampus load; an unfixed handler will show health p95/max
+// climbing toward the burst duration (worker-starved).
+//
+// Usage:
+//   node dev/scripts/probe/hippocampus-concurrency.js [options]
+//
+// Options:
+//   --base-url <url>       Rust backend base URL (default: http://127.0.0.1:18080)
+//   --concurrency <n>      Number of concurrent episode_recall calls (default: 8)
+//   --project-path <path>  projectPath arg for the MCP tool call (default: cwd)
+//   --poll-interval <ms>   Health poll interval in ms (default: 200)
+//   --query <text>         query arg for episode_recall (default: "spawn_blocking")
+//   --limit <n>            limit arg for episode_recall (default: 10)
+//
+// Example:
+//   node dev/scripts/probe/hippocampus-concurrency.js --concurrency 8
+
+'use strict';
+
+const http = require('http');
+const { URL } = require('url');
+
+function parseArgs(argv) {
+  const opts = {
+    baseUrl: 'http://127.0.0.1:18080',
+    concurrency: 8,
+    projectPath: process.cwd(),
+    pollIntervalMs: 200,
+    query: 'spawn_blocking',
+    limit: 10,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const next = () => argv[++i];
+    switch (a) {
+      case '--base-url':
+        opts.baseUrl = next();
+        break;
+      case '--concurrency':
+        opts.concurrency = parseInt(next(), 10);
+        break;
+      case '--project-path':
+        opts.projectPath = next();
+        break;
+      case '--poll-interval':
+        opts.pollIntervalMs = parseInt(next(), 10);
+        break;
+      case '--query':
+        opts.query = next();
+        break;
+      case '--limit':
+        opts.limit = parseInt(next(), 10);
+        break;
+      default:
+        break;
+    }
+  }
+  return opts;
+}
+
+function httpRequest(urlStr, { method = 'GET', headers = {}, body } = {}, timeoutMs = 60000) {
+  return new Promise((resolve) => {
+    const url = new URL(urlStr);
+    const started = Date.now();
+    const req = http.request(
+      {
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname + url.search,
+        method,
+        headers,
+        timeout: timeoutMs,
+      },
+      (res) => {
+        const chunks = [];
+        res.on('data', (c) => chunks.push(c));
+        res.on('end', () => {
+          resolve({
+            ok: res.statusCode >= 200 && res.statusCode < 300,
+            status: res.statusCode,
+            durationMs: Date.now() - started,
+            body: Buffer.concat(chunks).toString('utf8'),
+            error: null,
+          });
+        });
+      }
+    );
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, status: 0, durationMs: Date.now() - started, body: '', error: 'timeout' });
+    });
+    req.on('error', (err) => {
+      resolve({ ok: false, status: 0, durationMs: Date.now() - started, body: '', error: String(err && err.message || err) });
+    });
+    if (body) req.write(body);
+    req.end();
+  });
+}
+
+async function callMcpTool(baseUrl, toolName, args, id) {
+  const payload = JSON.stringify({
+    jsonrpc: '2.0',
+    id,
+    method: 'tools/call',
+    params: { name: toolName, arguments: args },
+  });
+  const res = await httpRequest(`${baseUrl}/api/mcp`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(payload),
+    },
+    body: payload,
+  });
+  let rpcError = null;
+  let toolIsError = false;
+  if (res.ok) {
+    try {
+      const parsed = JSON.parse(res.body);
+      if (parsed.error) rpcError = parsed.error.message || JSON.stringify(parsed.error);
+      if (parsed.result && parsed.result.isError) toolIsError = true;
+    } catch (e) {
+      rpcError = `non-JSON response: ${e.message}`;
+    }
+  }
+  return {
+    ok: res.ok && !rpcError && !toolIsError,
+    status: res.status,
+    durationMs: res.durationMs,
+    error: res.error || rpcError || (toolIsError ? 'tool returned isError' : null),
+  };
+}
+
+function percentile(sortedArr, p) {
+  if (sortedArr.length === 0) return null;
+  const idx = Math.min(sortedArr.length - 1, Math.ceil((p / 100) * sortedArr.length) - 1);
+  return sortedArr[Math.max(0, idx)];
+}
+
+function summarize(label, samples, failures) {
+  const durations = samples.map((s) => s.durationMs).sort((a, b) => a - b);
+  const p50 = percentile(durations, 50);
+  const p95 = percentile(durations, 95);
+  const max = durations.length ? durations[durations.length - 1] : null;
+  const min = durations.length ? durations[0] : null;
+  console.log(`\n[${label}]`);
+  console.log(`  samples: ${samples.length}, failures: ${failures}`);
+  console.log(`  min=${min}ms  p50=${p50}ms  p95=${p95}ms  max=${max}ms`);
+  return { label, count: samples.length, failures, min, p50, p95, max };
+}
+
+async function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  console.log('TASK_1924 hippocampus concurrency probe');
+  console.log(`  baseUrl=${opts.baseUrl} concurrency=${opts.concurrency} pollIntervalMs=${opts.pollIntervalMs}`);
+  console.log(`  projectPath=${opts.projectPath}`);
+
+  let polling = true;
+  const healthSamples = [];
+  let healthFailures = 0;
+
+  const pollLoop = (async () => {
+    while (polling) {
+      const t0 = Date.now();
+      const res = await httpRequest(`${opts.baseUrl}/api/rust/health`, {}, 10000);
+      if (!res.ok) healthFailures++;
+      healthSamples.push({ durationMs: res.durationMs, ok: res.ok, ts: t0 });
+      const elapsed = Date.now() - t0;
+      const sleep = Math.max(0, opts.pollIntervalMs - elapsed);
+      await new Promise((r) => setTimeout(r, sleep));
+    }
+  })();
+
+  const burstStart = Date.now();
+  const calls = Array.from({ length: opts.concurrency }, (_, i) =>
+    callMcpTool(
+      opts.baseUrl,
+      'hippocampus_episode_recall',
+      {
+        projectPath: opts.projectPath,
+        query: opts.query,
+        limit: opts.limit,
+      },
+      `probe-${i}`
+    )
+  );
+  const callResults = await Promise.all(calls);
+  const burstDurationMs = Date.now() - burstStart;
+
+  // Keep polling briefly after the burst to catch any lingering worker-starvation tail.
+  await new Promise((r) => setTimeout(r, Math.max(opts.pollIntervalMs * 3, 600)));
+  polling = false;
+  await pollLoop;
+
+  const callFailures = callResults.filter((r) => !r.ok).length;
+
+  console.log(`\nBurst wall-clock: ${burstDurationMs}ms for ${opts.concurrency} concurrent episode_recall calls`);
+  const callSummary = summarize('hippocampus_episode_recall calls', callResults, callFailures);
+  const healthSummary = summarize('/api/rust/health polls (during + shortly after burst)', healthSamples, healthFailures);
+
+  if (callFailures > 0) {
+    console.log('\nFailed episode_recall calls:');
+    callResults.forEach((r, i) => {
+      if (!r.ok) console.log(`  #${i}: status=${r.status} error=${r.error}`);
+    });
+  }
+
+  console.log('\nJSON summary:');
+  console.log(
+    JSON.stringify(
+      {
+        concurrency: opts.concurrency,
+        burstDurationMs,
+        calls: callSummary,
+        health: healthSummary,
+      },
+      null,
+      2
+    )
+  );
+
+  process.exit(callFailures > 0 || healthFailures > 0 ? 1 : 0);
+}
+
+main().catch((err) => {
+  console.error('Probe crashed:', err);
+  process.exit(2);
+});