depgraph-core 1.9.0 → 1.9.2

package/scripts/benchmark.js

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+// Measures real response sizes from depgraph-mcp.js for each tool variant.
+// Usage:
+//   node scripts/benchmark.js
+//   BENCH_DIR=/other/project node scripts/benchmark.js
+'use strict';
+const { execSync } = require('child_process');
+const path = require('path');
+
+const SERVER = path.resolve(__dirname, '..', 'depgraph-mcp.js');
+const DIR    = process.env.BENCH_DIR ?? path.resolve(__dirname, '..');
+
+const os   = require('os');
+const fs   = require('fs');
+const TMPF = path.join(os.tmpdir(), `depgraph-bench-${process.pid}.txt`);
+
+function call(toolName, args) {
+  const messages = [
+    {
+      jsonrpc: '2.0', id: 1, method: 'initialize',
+      params: { protocolVersion: '2024-11-05', capabilities: {},
+                clientInfo: { name: 'bench', version: '1.0' } },
+    },
+    {
+      jsonrpc: '2.0', id: 2, method: 'tools/call',
+      params: { name: toolName, arguments: args },
+    },
+  ].map(m => JSON.stringify(m)).join('\n') + '\n';
+
+  fs.writeFileSync(TMPF, messages, 'utf-8');
+
+  const out = execSync(
+    `cat ${TMPF} | node ${SERVER}`,
+    { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
+  );
+
+  const lines = out.trim().split('\n').filter(Boolean);
+  const result = JSON.parse(lines[lines.length - 1]);
+  return result?.result?.content?.[0]?.text ?? '';
+}
+
+// ~3.5 chars/token for dense JSON; ~4.0 for prose (matches Claude/GPT-4 closely enough)
+const tokEst = (s, prose = false) => Math.round(s.length / (prose ? 4.0 : 3.5));
+
+const CASES = [
+  {
+    label: 'analyze_project — full (BASELINE)',
+    fn: () => call('analyze_project', { projectDir: DIR }),
+  },
+  {
+    label: 'analyze_project — verbosity:"overview"',
+    fn: () => call('analyze_project', { projectDir: DIR, verbosity: 'overview' }),
+  },
+  {
+    label: 'analyze_project — verbosity:"sketch"',
+    fn: () => call('analyze_project', { projectDir: DIR, verbosity: 'sketch' }),
+  },
+  {
+    label: 'analyze_project — focus + depth:2 (overview)',
+    fn: () => call('analyze_project',
+      { projectDir: DIR, focus: 'buildGraph', depth: 2, verbosity: 'overview' }),
+  },
+  {
+    label: 'analyze_project — focus + depth:1 (overview)',
+    fn: () => call('analyze_project',
+      { projectDir: DIR, focus: 'buildGraph', depth: 1, verbosity: 'overview' }),
+  },
+  {
+    label: 'get_graph_summary — overview',
+    fn: () => call('get_graph_summary', { projectDir: DIR }),
+  },
+  {
+    label: 'get_graph_summary — verbosity:"sketch"',
+    fn: () => call('get_graph_summary', { projectDir: DIR, verbosity: 'sketch' }),
+  },
+  {
+    label: 'get_graph_summary — format:"prose"',
+    fn: () => call('get_graph_summary', { projectDir: DIR, format: 'prose' }),
+    prose: true,
+  },
+  {
+    label: 'describe_node("buildGraph")',
+    fn: () => call('describe_node', { projectDir: DIR, nodeName: 'buildGraph' }),
+  },
+  {
+    label: 'describe_node("simulateImpact")',
+    fn: () => call('describe_node', { projectDir: DIR, nodeName: 'simulateImpact' }),
+  },
+  {
+    label: 'simulate_impact("buildGraph", ...)',
+    fn: () => call('simulate_impact', {
+      projectDir: DIR,
+      targetNode: 'buildGraph',
+      changeDescription: 'removing parsed arg',
+    }),
+  },
+];
+
+console.log(`\nBenchmarking against: ${DIR}`);
+console.log(`MCP server:           ${SERVER}\n`);
+
+const rows = CASES.map(c => {
+  process.stdout.write(`  ${c.label} ...`);
+  const text   = c.fn();
+  const chars  = text.length;
+  const tokens = tokEst(text, c.prose ?? false);
+  process.stdout.write(` ${tokens.toLocaleString()} tokens\n`);
+  return { label: c.label, chars, tokens };
+});
+
+const baseline = rows[0].tokens;
+
+const W = { label: 50, chars: 10, tokens: 9, pct: 14 };
+const rule = '─'.repeat(W.label + W.chars + W.tokens + W.pct + 4);
+
+console.log('\n' + rule);
+console.log(
+  'Tool call'.padEnd(W.label) +
+  'Chars'.padStart(W.chars) +
+  'Tokens'.padStart(W.tokens) +
+  'vs baseline'.padStart(W.pct)
+);
+console.log(rule);
+
+rows.forEach((r, i) => {
+  const pct = i === 0
+    ? '(baseline)'
+    : `−${Math.round((1 - r.tokens / baseline) * 100)} %`;
+  console.log(
+    r.label.padEnd(W.label) +
+    r.chars.toLocaleString().padStart(W.chars) +
+    r.tokens.toLocaleString().padStart(W.tokens) +
+    pct.padStart(W.pct)
+  );
+});
+
+console.log(rule);
+console.log(`\nBaseline: ${baseline.toLocaleString()} tokens  (${rows[0].chars.toLocaleString()} chars)\n`);