mixdog 0.7.1

package/scripts/io-mini-bench.mjs

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+import { performance } from 'node:perf_hooks';
+import { existsSync, readFileSync } from 'node:fs';
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, isAbsolute, join, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+process.env.CLAUDE_PLUGIN_ROOT ||= ROOT;
+// SAFETY: under Claude Code, CLAUDE_PLUGIN_DATA points at the LIVE plugin data
+// dir; this bench rm's its data dir on exit, so it MUST own an isolated temp dir.
+// Unconditional assignment — never `||=`, which would adopt and later delete real data.
+const BENCH_PLUGIN_DATA = join(tmpdir(), `mixdog-io-mini-bench-data-${process.pid}`);
+process.env.CLAUDE_PLUGIN_DATA = BENCH_PLUGIN_DATA;
+
+const { executeBuiltinTool } = await import(pathToFileURL(join(ROOT, 'src/agent/orchestrator/tools/builtin.mjs')).href);
+const { executePatchTool, closeNativePatchServerForTests } = await import(pathToFileURL(join(ROOT, 'src/agent/orchestrator/tools/patch.mjs')).href);
+
+const ITERS = Math.max(1, Number(process.env.MIXDOG_IO_MINI_BENCH_ITERS || 5));
+const OUTPUT_JSON = process.argv.includes('--json') || /^(1|true|yes)$/i.test(String(process.env.MIXDOG_IO_MINI_BENCH_JSON || ''));
+const CHECK_THRESHOLDS = process.argv.includes('--check-thresholds') || /^(1|true|yes)$/i.test(String(process.env.MIXDOG_IO_MINI_BENCH_CHECK || ''));
+const WORKDIR = await mkdtemp(join(tmpdir(), 'mixdog-io-mini-bench-'));
+const DEFAULT_BASELINE_PATH = join(ROOT, 'scripts', 'io-mini-bench-baseline.json');
+
+function argValue(name) {
+  const prefix = `${name}=`;
+  const found = process.argv.find((arg) => arg.startsWith(prefix));
+  return found ? found.slice(prefix.length) : '';
+}
+
+function resolveBaselinePath(raw) {
+  const value = raw || process.env.MIXDOG_IO_MINI_BENCH_BASELINE || DEFAULT_BASELINE_PATH;
+  return isAbsolute(value) ? value : resolve(ROOT, value);
+}
+
+const DEFAULT_THRESHOLDS_MS = {
+  'small-edit': 250,
+  'multi-edit': 300,
+  'large-dry-run-patch': 1500,
+  'range-read-cache-hit': 80,
+  'grep-cache-hit': 80,
+};
+
+function loadThresholds() {
+  const raw = process.env.MIXDOG_IO_MINI_BENCH_THRESHOLDS_JSON;
+  if (!raw) return DEFAULT_THRESHOLDS_MS;
+  try {
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== 'object') return DEFAULT_THRESHOLDS_MS;
+    return { ...DEFAULT_THRESHOLDS_MS, ...parsed };
+  } catch (err) {
+    throw new Error(`invalid MIXDOG_IO_MINI_BENCH_THRESHOLDS_JSON: ${err?.message || String(err)}`);
+  }
+}
+
+function thresholdsFromBaseline(path) {
+  if (!path || !existsSync(path)) return null;
+  const parsed = JSON.parse(readFileSync(path, 'utf8'));
+  const explicit = parsed?.thresholds_ms || parsed?.thresholds;
+  if (explicit && typeof explicit === 'object') return explicit;
+  const multiplier = Number(parsed?.threshold_multiplier ?? parsed?.thresholdMultiplier ?? 2);
+  const rows = Array.isArray(parsed?.rows) ? parsed.rows : [];
+  const out = {};
+  for (const row of rows) {
+    if (!row?.case) continue;
+    const median = Number(row.median_ms);
+    if (Number.isFinite(median) && median > 0) out[row.case] = Number((median * (Number.isFinite(multiplier) ? multiplier : 2)).toFixed(2));
+  }
+  return Object.keys(out).length > 0 ? out : null;
+}
+
+function assertOk(label, out) {
+  const text = String(out ?? '');
+  if (/^Error(\s|\[|:)/.test(text)) throw new Error(`${label} failed: ${text.split('\n')[0]}`);
+  return text;
+}
+
+function patchText(lines) {
+  return `${lines.join('\n')}\n`;
+}
+
+function median(values) {
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.floor(sorted.length / 2)];
+}
+
+async function timeCase(name, setup, run) {
+  const samples = [];
+  for (let i = 0; i < ITERS; i++) {
+    const dir = await mkdtemp(join(WORKDIR, `${name}-${i}-`));
+    try {
+      await setup(dir, i);
+      const started = performance.now();
+      await run(dir, i);
+      samples.push(performance.now() - started);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  }
+  const sum = samples.reduce((acc, value) => acc + value, 0);
+  return {
+    case: name,
+    iterations: ITERS,
+    min_ms: Number(Math.min(...samples).toFixed(2)),
+    median_ms: Number(median(samples).toFixed(2)),
+    avg_ms: Number((sum / samples.length).toFixed(2)),
+    max_ms: Number(Math.max(...samples).toFixed(2)),
+  };
+}
+
+async function main() {
+  await mkdir(process.env.CLAUDE_PLUGIN_DATA, { recursive: true });
+  const rows = [];
+
+  rows.push(await timeCase('small-edit', async (dir, i) => {
+    await writeFile(join(dir, 'small.txt'), `alpha\nTARGET-${i}\nomega\n`, 'utf8');
+    assertOk('small-edit warm read', await executeBuiltinTool('read', { path: 'small.txt' }, dir, { readStateScope: `mini-small-${i}` }));
+  }, async (dir, i) => {
+    assertOk('small-edit edit', await executeBuiltinTool('edit', {
+      path: 'small.txt',
+      old_string: `TARGET-${i}`,
+      new_string: `DONE-${i}`,
+    }, dir, { readStateScope: `mini-small-${i}` }));
+  }));
+
+  rows.push(await timeCase('multi-edit', async (dir, i) => {
+    await writeFile(join(dir, 'multi.txt'), `LEFT-${i}\ncenter\nRIGHT-${i}\n`, 'utf8');
+    assertOk('multi-edit warm read', await executeBuiltinTool('read', { path: 'multi.txt' }, dir, { readStateScope: `mini-multi-${i}` }));
+  }, async (dir, i) => {
+    assertOk('multi-edit edit', await executeBuiltinTool('edit', {
+      path: 'multi.txt',
+      edits: [
+        { old_string: `LEFT-${i}`, new_string: `LEFT-DONE-${i}` },
+        { old_string: `RIGHT-${i}`, new_string: `RIGHT-DONE-${i}` },
+      ],
+    }, dir, { readStateScope: `mini-multi-${i}` }));
+  }));
+
+  rows.push(await timeCase('large-dry-run-patch', async (dir, i) => {
+    const lines = Array.from({ length: 16000 }, (_, n) => n === 9000 ? `PATCH-ME-${i}` : `line-${n + 1}`);
+    await writeFile(join(dir, 'large.txt'), `${lines.join('\n')}\n`, 'utf8');
+  }, async (dir, i) => {
+    assertOk('large-dry-run-patch apply_patch', await executePatchTool('apply_patch', {
+      patch: patchText([
+        '--- a/large.txt',
+        '+++ b/large.txt',
+        '@@ -9001,1 +9001,1 @@',
+        `-PATCH-ME-${i}`,
+        `+PATCHED-${i}`,
+      ]),
+      base_path: dir,
+      dry_run: true,
+    }, dir, { readStateScope: `mini-patch-${i}` }));
+  }));
+
+  rows.push(await timeCase('range-read-cache-hit', async (dir, i) => {
+    const lines = Array.from({ length: 12000 }, (_, n) => `range-${i}-${n + 1}`);
+    await writeFile(join(dir, 'range.txt'), `${lines.join('\n')}\n`, 'utf8');
+    assertOk('range-read warm', await executeBuiltinTool('read', { path: 'range.txt', offset: 9000, limit: 3 }, dir, { readStateScope: `mini-read-${i}` }));
+  }, async (dir, i) => {
+    assertOk('range-read cached', await executeBuiltinTool('read', { path: 'range.txt', offset: 9000, limit: 3 }, dir, { readStateScope: `mini-read-${i}` }));
+  }));
+
+  rows.push(await timeCase('grep-cache-hit', async (dir, i) => {
+    const lines = Array.from({ length: 2000 }, (_, n) => n % 200 === 0 ? `needle-${i}-${n}` : `hay-${n}`);
+    await writeFile(join(dir, 'grep.txt'), `${lines.join('\n')}\n`, 'utf8');
+    assertOk('grep warm', await executeBuiltinTool('grep', {
+      pattern: `needle-${i}`,
+      path: 'grep.txt',
+      output_mode: 'content',
+      head_limit: 3,
+    }, dir, { readStateScope: `mini-grep-${i}` }));
+  }, async (dir, i) => {
+    assertOk('grep cached', await executeBuiltinTool('grep', {
+      pattern: `needle-${i}`,
+      path: 'grep.txt',
+      output_mode: 'content',
+      head_limit: 3,
+    }, dir, { readStateScope: `mini-grep-${i}` }));
+  }));
+
+  const baselinePath = resolveBaselinePath(argValue('--baseline'));
+  const baselineThresholds = CHECK_THRESHOLDS ? thresholdsFromBaseline(baselinePath) : null;
+  const thresholds = CHECK_THRESHOLDS ? { ...loadThresholds(), ...(baselineThresholds || {}) } : null;
+  const failures = thresholds
+    ? rows
+      .filter((row) => Number.isFinite(Number(thresholds[row.case])) && row.median_ms > Number(thresholds[row.case]))
+      .map((row) => ({ case: row.case, median_ms: row.median_ms, threshold_ms: Number(thresholds[row.case]) }))
+    : [];
+
+  if (OUTPUT_JSON) {
+    console.log(JSON.stringify({ iterations: ITERS, rows, ...(thresholds ? { baselinePath, thresholds, failures } : {}) }, null, 2));
+  } else {
+    console.table(rows);
+    if (thresholds) {
+      if (failures.length === 0) {
+        console.log('thresholds: pass');
+      } else {
+        console.log(`thresholds: fail ${failures.map((f) => `${f.case} ${f.median_ms}ms>${f.threshold_ms}ms`).join(', ')}`);
+      }
+    }
+    console.log('hint: pass --json or MIXDOG_IO_MINI_BENCH_JSON=1 for machine-readable output');
+  }
+  if (failures.length > 0) {
+    throw new Error(`io-mini-bench threshold failure: ${failures.map((f) => `${f.case} median ${f.median_ms}ms > ${f.threshold_ms}ms`).join('; ')}`);
+  }
+}
+
+try {
+  await main();
+} finally {
+  await closeNativePatchServerForTests().catch(() => {});
+  await rm(WORKDIR, { recursive: true, force: true });
+  await rm(BENCH_PLUGIN_DATA, { recursive: true, force: true });
+}