@levnikolaevich/hex-line-mcp 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/benchmark.mjs CHANGED
@@ -1,25 +1,37 @@
1
1
  #!/usr/bin/env node
2
2
  /**
3
- * Hex-line Combo Benchmark v2
3
+ * Hex-line Combo Benchmark v3
4
4
  *
5
5
  * Compares "agent without hex-line" vs "agent with hex-line" across
6
6
  * read-only and write scenarios. Measures chars in response (proxy for tokens).
7
7
  *
8
- * Usage: node mcp/hex-line-mcp/benchmark.mjs [--repo /path/to/repo]
8
+ * Usage: node mcp/hex-line-mcp/benchmark.mjs [--repo /path/to/repo] [--with-graph]
9
9
  * Default repo: current working directory.
10
10
  *
11
11
  * Zero external deps beyond hex-line lib modules.
12
12
  */
13
13
 
14
- import { readFileSync, writeFileSync, unlinkSync, statSync, readdirSync } from "node:fs";
15
- import { resolve, extname, basename, join } from "node:path";
14
+ import { readFileSync, writeFileSync, unlinkSync, readdirSync, mkdirSync, rmSync } from "node:fs";
15
+ import { performance } from "node:perf_hooks";
16
+ import { resolve, basename } from "node:path";
16
17
  import { tmpdir } from "node:os";
17
18
  import { fnv1a, lineTag, rangeChecksum } from "./lib/hash.mjs";
18
19
  import { readFile } from "./lib/read.mjs";
19
20
  import { directoryTree } from "./lib/tree.mjs";
20
21
  import { fileInfo } from "./lib/info.mjs";
21
22
  import { verifyChecksums } from "./lib/verify.mjs";
22
-
23
+ import { fileChanges } from "./lib/changes.mjs";
24
+ import { editFile } from "./lib/edit.mjs";
25
+ import { grepSearch } from "./lib/search.mjs";
26
+ import { bulkReplace } from "./lib/bulk-replace.mjs";
27
+ import { fileOutline } from "./lib/outline.mjs";
28
+ import {
29
+ walkDir, getFileLines, categorize, generateTempCode,
30
+ simBuiltInReadFull, simBuiltInOutlineFull, simBuiltInGrep,
31
+ simBuiltInLsR, simBuiltInStat, simBuiltInWrite, simBuiltInEdit, simBuiltInVerify,
32
+ simHexLineOutlinePlusRead, simHexLineGrep, simHexLineWrite, simHexLineEditDiff,
33
+ runN, fmt, pctSavings, RUNS,
34
+ } from "./lib/benchmark-helpers.mjs";
23
35
  // ---------------------------------------------------------------------------
24
36
  // CLI
25
37
  // ---------------------------------------------------------------------------
@@ -31,534 +43,13 @@ if (repoIdx !== -1 && args[repoIdx + 1]) {
31
43
  repoRoot = resolve(args[repoIdx + 1]);
32
44
  }
33
45
 
34
- const CODE_EXTS = new Set([".js", ".ts", ".py", ".mjs", ".go", ".rs", ".java", ".c", ".cpp", ".rb", ".php"]);
35
- const MAX_FILES_PER_CAT = 3;
36
- const RUNS = 3;
37
-
38
- // ---------------------------------------------------------------------------
39
- // File discovery
40
- // ---------------------------------------------------------------------------
41
-
42
- function walkDir(dir, depth = 0) {
43
- if (depth > 10) return [];
44
- const results = [];
45
- let entries;
46
- try { entries = readdirSync(dir, { withFileTypes: true }); }
47
- catch { return results; }
48
- for (const e of entries) {
49
- const full = resolve(dir, e.name);
50
- if (e.isDirectory()) {
51
- if (e.name.startsWith(".") || e.name === "node_modules" || e.name === "vendor"
52
- || e.name === "dist" || e.name === "__pycache__" || e.name === "target") continue;
53
- results.push(...walkDir(full, depth + 1));
54
- } else if (e.isFile() && CODE_EXTS.has(extname(e.name).toLowerCase())) {
55
- try {
56
- const st = statSync(full);
57
- if (st.size > 0 && st.size < 1_000_000) results.push(full);
58
- } catch { /* skip */ }
59
- }
60
- }
61
- return results;
62
- }
63
-
64
- function getFileLines(f) {
65
- try { return readFileSync(f, "utf-8").replace(/\r\n/g, "\n").split("\n"); }
66
- catch { return null; }
67
- }
68
-
69
- function categorize(files) {
70
- const cats = { small: [], medium: [], large: [], xl: [] };
71
- for (const f of files) {
72
- const lines = getFileLines(f);
73
- if (!lines) continue;
74
- const n = lines.length;
75
- if (n >= 10 && n <= 50) cats.small.push(f);
76
- else if (n > 50 && n <= 200) cats.medium.push(f);
77
- else if (n > 200 && n <= 500) cats.large.push(f);
78
- else if (n > 500) cats.xl.push(f);
79
- }
80
- for (const key of Object.keys(cats)) {
81
- const arr = cats[key];
82
- if (arr.length > MAX_FILES_PER_CAT) {
83
- const step = Math.floor(arr.length / MAX_FILES_PER_CAT);
84
- cats[key] = Array.from({ length: MAX_FILES_PER_CAT }, (_, i) => arr[i * step]);
85
- }
86
- }
87
- return cats;
88
- }
89
-
90
- // ---------------------------------------------------------------------------
91
- // Temp file: 200 lines of realistic JS
92
- // ---------------------------------------------------------------------------
93
-
94
- function generateTempCode() {
95
- const lines = [];
96
- lines.push('import { readFileSync } from "node:fs";');
97
- lines.push('import { resolve, basename } from "node:path";');
98
- lines.push("");
99
- lines.push("const DEFAULT_TIMEOUT = 5000;");
100
- lines.push("const MAX_RETRIES = 3;");
101
- lines.push("");
102
- lines.push("/**");
103
- lines.push(" * Configuration manager for application settings.");
104
- lines.push(" * Supports file-based and environment-based config sources.");
105
- lines.push(" */");
106
- lines.push("class ConfigManager {");
107
- lines.push(" constructor(configPath) {");
108
- lines.push(" this.configPath = resolve(configPath);");
109
- lines.push(" this.cache = new Map();");
110
- lines.push(" this.watchers = [];");
111
- lines.push(" this.loaded = false;");
112
- lines.push(" }");
113
- lines.push("");
114
- lines.push(" load() {");
115
- lines.push(" const raw = readFileSync(this.configPath, 'utf-8');");
116
- lines.push(" const parsed = JSON.parse(raw);");
117
- lines.push(" for (const [key, value] of Object.entries(parsed)) {");
118
- lines.push(" this.cache.set(key, value);");
119
- lines.push(" }");
120
- lines.push(" this.loaded = true;");
121
- lines.push(" this.notifyWatchers('load', parsed);");
122
- lines.push(" return this;");
123
- lines.push(" }");
124
- lines.push("");
125
- lines.push(" get(key, defaultValue = undefined) {");
126
- lines.push(" if (!this.loaded) this.load();");
127
- lines.push(" return this.cache.has(key) ? this.cache.get(key) : defaultValue;");
128
- lines.push(" }");
129
- lines.push("");
130
- lines.push(" set(key, value) {");
131
- lines.push(" this.cache.set(key, value);");
132
- lines.push(" this.notifyWatchers('set', { key, value });");
133
- lines.push(" }");
134
- lines.push("");
135
- lines.push(" watch(callback) {");
136
- lines.push(" this.watchers.push(callback);");
137
- lines.push(" return () => {");
138
- lines.push(" this.watchers = this.watchers.filter(w => w !== callback);");
139
- lines.push(" };");
140
- lines.push(" }");
141
- lines.push("");
142
- lines.push(" notifyWatchers(event, data) {");
143
- lines.push(" for (const watcher of this.watchers) {");
144
- lines.push(" try { watcher(event, data); }");
145
- lines.push(" catch (e) { console.error('Watcher error:', e.message); }");
146
- lines.push(" }");
147
- lines.push(" }");
148
- lines.push("}");
149
- lines.push("");
150
- lines.push("/**");
151
- lines.push(" * Retry wrapper with exponential backoff.");
152
- lines.push(" */");
153
- lines.push("async function withRetry(fn, options = {}) {");
154
- lines.push(" const { retries = MAX_RETRIES, delay = 100, backoff = 2 } = options;");
155
- lines.push(" let lastError;");
156
- lines.push(" for (let attempt = 0; attempt <= retries; attempt++) {");
157
- lines.push(" try {");
158
- lines.push(" return await fn(attempt);");
159
- lines.push(" } catch (err) {");
160
- lines.push(" lastError = err;");
161
- lines.push(" if (attempt < retries) {");
162
- lines.push(" const wait = delay * Math.pow(backoff, attempt);");
163
- lines.push(" await new Promise(r => setTimeout(r, wait));");
164
- lines.push(" }");
165
- lines.push(" }");
166
- lines.push(" }");
167
- lines.push(" throw lastError;");
168
- lines.push("}");
169
- lines.push("");
170
- lines.push("/**");
171
- lines.push(" * HTTP client with timeout and retry support.");
172
- lines.push(" */");
173
- lines.push("class HttpClient {");
174
- lines.push(" constructor(baseUrl, options = {}) {");
175
- lines.push(" this.baseUrl = baseUrl.replace(/\\/$/, '');");
176
- lines.push(" this.timeout = options.timeout || DEFAULT_TIMEOUT;");
177
- lines.push(" this.headers = options.headers || {};");
178
- lines.push(" this.retries = options.retries || MAX_RETRIES;");
179
- lines.push(" }");
180
- lines.push("");
181
- lines.push(" async request(method, path, body = null) {");
182
- lines.push(" const url = `${this.baseUrl}${path}`;");
183
- lines.push(" const controller = new AbortController();");
184
- lines.push(" const timer = setTimeout(() => controller.abort(), this.timeout);");
185
- lines.push("");
186
- lines.push(" try {");
187
- lines.push(" return await withRetry(async () => {");
188
- lines.push(" const opts = {");
189
- lines.push(" method,");
190
- lines.push(" headers: { ...this.headers },");
191
- lines.push(" signal: controller.signal,");
192
- lines.push(" };");
193
- lines.push(" if (body) {");
194
- lines.push(" opts.headers['Content-Type'] = 'application/json';");
195
- lines.push(" opts.body = JSON.stringify(body);");
196
- lines.push(" }");
197
- lines.push(" const response = await fetch(url, opts);");
198
- lines.push(" if (!response.ok) {");
199
- lines.push(" throw new Error(`HTTP ${response.status}: ${response.statusText}`);");
200
- lines.push(" }");
201
- lines.push(" return response.json();");
202
- lines.push(" }, { retries: this.retries });");
203
- lines.push(" } finally {");
204
- lines.push(" clearTimeout(timer);");
205
- lines.push(" }");
206
- lines.push(" }");
207
- lines.push("");
208
- lines.push(" get(path) { return this.request('GET', path); }");
209
- lines.push(" post(path, body) { return this.request('POST', path, body); }");
210
- lines.push(" put(path, body) { return this.request('PUT', path, body); }");
211
- lines.push(" delete(path) { return this.request('DELETE', path); }");
212
- lines.push("}");
213
- lines.push("");
214
- lines.push("/**");
215
- lines.push(" * Simple event emitter for pub/sub patterns.");
216
- lines.push(" */");
217
- lines.push("class EventEmitter {");
218
- lines.push(" constructor() {");
219
- lines.push(" this.listeners = new Map();");
220
- lines.push(" }");
221
- lines.push("");
222
- lines.push(" on(event, handler) {");
223
- lines.push(" if (!this.listeners.has(event)) {");
224
- lines.push(" this.listeners.set(event, []);");
225
- lines.push(" }");
226
- lines.push(" this.listeners.get(event).push(handler);");
227
- lines.push(" return this;");
228
- lines.push(" }");
229
- lines.push("");
230
- lines.push(" off(event, handler) {");
231
- lines.push(" const handlers = this.listeners.get(event);");
232
- lines.push(" if (handlers) {");
233
- lines.push(" this.listeners.set(event, handlers.filter(h => h !== handler));");
234
- lines.push(" }");
235
- lines.push(" return this;");
236
- lines.push(" }");
237
- lines.push("");
238
- lines.push(" emit(event, ...args) {");
239
- lines.push(" const handlers = this.listeners.get(event) || [];");
240
- lines.push(" for (const handler of handlers) {");
241
- lines.push(" handler(...args);");
242
- lines.push(" }");
243
- lines.push(" }");
244
- lines.push("");
245
- lines.push(" once(event, handler) {");
246
- lines.push(" const wrapper = (...args) => {");
247
- lines.push(" handler(...args);");
248
- lines.push(" this.off(event, wrapper);");
249
- lines.push(" };");
250
- lines.push(" return this.on(event, wrapper);");
251
- lines.push(" }");
252
- lines.push("}");
253
- lines.push("");
254
- lines.push("/**");
255
- lines.push(" * Validate and sanitize user input.");
256
- lines.push(" */");
257
- lines.push("function validateInput(schema, data) {");
258
- lines.push(" const errors = [];");
259
- lines.push(" for (const [field, rules] of Object.entries(schema)) {");
260
- lines.push(" const value = data[field];");
261
- lines.push(" if (rules.required && (value === undefined || value === null)) {");
262
- lines.push(" errors.push(`${field} is required`);");
263
- lines.push(" continue;");
264
- lines.push(" }");
265
- lines.push(" if (value !== undefined && rules.type && typeof value !== rules.type) {");
266
- lines.push(" errors.push(`${field} must be ${rules.type}`);");
267
- lines.push(" }");
268
- lines.push(" if (typeof value === 'string' && rules.maxLength && value.length > rules.maxLength) {");
269
- lines.push(" errors.push(`${field} exceeds max length ${rules.maxLength}`);");
270
- lines.push(" }");
271
- lines.push(" if (typeof value === 'number' && rules.min !== undefined && value < rules.min) {");
272
- lines.push(" errors.push(`${field} must be >= ${rules.min}`);");
273
- lines.push(" }");
274
- lines.push(" }");
275
- lines.push(" return errors.length > 0 ? { valid: false, errors } : { valid: true };");
276
- lines.push("}");
277
- lines.push("");
278
- lines.push("/**");
279
- lines.push(" * Format bytes to human-readable string.");
280
- lines.push(" */");
281
- lines.push("function formatBytes(bytes) {");
282
- lines.push(" if (bytes === 0) return '0 B';");
283
- lines.push(" const units = ['B', 'KB', 'MB', 'GB', 'TB'];");
284
- lines.push(" const exp = Math.floor(Math.log(bytes) / Math.log(1024));");
285
- lines.push(" const value = bytes / Math.pow(1024, exp);");
286
- lines.push(" return `${value.toFixed(exp > 0 ? 1 : 0)} ${units[exp]}`;");
287
- lines.push("}");
288
- lines.push("");
289
- lines.push("/**");
290
- lines.push(" * Deep merge two objects (source into target).");
291
- lines.push(" */");
292
- lines.push("function deepMerge(target, source) {");
293
- lines.push(" const result = { ...target };");
294
- lines.push(" for (const key of Object.keys(source)) {");
295
- lines.push(" if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {");
296
- lines.push(" result[key] = deepMerge(result[key] || {}, source[key]);");
297
- lines.push(" } else {");
298
- lines.push(" result[key] = source[key];");
299
- lines.push(" }");
300
- lines.push(" }");
301
- lines.push(" return result;");
302
- lines.push("}");
303
- lines.push("");
304
- lines.push("export { ConfigManager, HttpClient, EventEmitter, withRetry, validateInput, formatBytes, deepMerge };");
305
-
306
- // Pad to exactly 200 lines
307
- while (lines.length < 200) lines.push("");
308
- return lines.slice(0, 200);
309
- }
310
-
311
- // ---------------------------------------------------------------------------
312
- // Simulators — "without hex-line" (built-in tool output)
313
- // ---------------------------------------------------------------------------
314
-
315
- /** Simulate built-in Read: `cat -n` full file with header */
316
- function simBuiltInReadFull(filePath, lines) {
317
- const body = lines.map((l, i) => ` ${String(i + 1).padStart(5)}\t${l}`).join("\n");
318
- return `Contents of ${filePath}:\n\n${body}`;
319
- }
320
-
321
- /** Simulate outline via full read — agent reads entire file to understand structure */
322
- function simBuiltInOutlineFull(filePath, lines) {
323
- return simBuiltInReadFull(filePath, lines);
324
- }
325
-
326
- /** Simulate ripgrep raw output (no hashes) */
327
- function simBuiltInGrep(filePath, lines, pattern) {
328
- const re = new RegExp(pattern, "i");
329
- const matches = [];
330
- for (let i = 0; i < lines.length; i++) {
331
- if (re.test(lines[i])) {
332
- matches.push(`${filePath}:${i + 1}:${lines[i]}`);
333
- }
334
- }
335
- return matches.join("\n") || "No matches found.";
336
- }
337
-
338
- /** Simulate `ls -laR` style output for a directory */
339
- function simBuiltInLsR(dirPath, depth = 0, maxDepth = 3) {
340
- if (depth > maxDepth) return "";
341
- const out = [];
342
- let entries;
343
- try { entries = readdirSync(dirPath, { withFileTypes: true }); }
344
- catch { return ""; }
345
-
346
- const SKIP = new Set(["node_modules", ".git", "dist", "build", "__pycache__", "coverage"]);
347
-
348
- out.push(`${dirPath}:`);
349
- out.push("total " + entries.length);
350
-
351
- for (const e of entries) {
352
- if (SKIP.has(e.name) && e.isDirectory()) continue;
353
- const full = join(dirPath, e.name);
354
- try {
355
- const st = statSync(full);
356
- const type = e.isDirectory() ? "d" : "-";
357
- const size = String(st.size).padStart(8);
358
- const date = st.mtime.toISOString().slice(0, 16).replace("T", " ");
359
- out.push(`${type}rw-r--r-- 1 user group ${size} ${date} ${e.name}`);
360
- } catch { /* skip */ }
361
- }
362
- out.push("");
363
-
364
- for (const e of entries) {
365
- if (!e.isDirectory()) continue;
366
- if (SKIP.has(e.name)) continue;
367
- const full = join(dirPath, e.name);
368
- const sub = simBuiltInLsR(full, depth + 1, maxDepth);
369
- if (sub) out.push(sub);
370
- }
371
-
372
- return out.join("\n");
373
- }
374
-
375
- /** Simulate `stat` output for a file */
376
- function simBuiltInStat(filePath) {
377
- const st = statSync(filePath);
378
- return [
379
- ` File: ${filePath}`,
380
- ` Size: ${st.size}\tBlocks: ${Math.ceil(st.size / 512)}\tIO Block: 4096\tregular file`,
381
- `Device: 0h/0d\tInode: 0\tLinks: 1`,
382
- `Access: (0644/-rw-r--r--)\tUid: ( 1000/ user)\tGid: ( 1000/ group)`,
383
- `Access: ${st.atime.toISOString()}`,
384
- `Modify: ${st.mtime.toISOString()}`,
385
- `Change: ${st.ctime.toISOString()}`,
386
- ` Birth: ${st.birthtime.toISOString()}`,
387
- ].join("\n");
388
- }
389
-
390
- /** Simulate built-in write response */
391
- function simBuiltInWrite(filePath, content) {
392
- const lineCount = content.split("\n").length;
393
- return `File ${filePath} has been created successfully (${lineCount} lines).`;
394
- }
395
-
396
- /** Simulate built-in edit: old_string/new_string context blocks */
397
- function simBuiltInEdit(filePath, origLines, newLines) {
398
- let changeStart = -1, changeEnd = -1;
399
- for (let i = 0; i < Math.max(origLines.length, newLines.length); i++) {
400
- if (origLines[i] !== newLines[i]) {
401
- if (changeStart === -1) changeStart = i;
402
- changeEnd = i;
403
- }
404
- }
405
- if (changeStart === -1) return "";
406
-
407
- const ctxBefore = Math.max(0, changeStart - 3);
408
- const ctxAfter = Math.min(origLines.length, changeEnd + 4);
409
- const old_string = origLines.slice(ctxBefore, ctxAfter).join("\n");
410
- const new_string = newLines.slice(ctxBefore, Math.min(newLines.length, changeEnd + 4)).join("\n");
411
- return `The file ${filePath} has been edited. Here's the result of running \`cat -n\` on a snippet:\n` +
412
- `old_string:\n${old_string}\nnew_string:\n${new_string}`;
413
- }
414
-
415
- /** Simulate built-in verify: full re-read to check if file changed */
416
- function simBuiltInVerify(filePath, lines) {
417
- return simBuiltInReadFull(filePath, lines);
418
- }
419
-
420
- // ---------------------------------------------------------------------------
421
- // Simulators — "with hex-line" (lib function output)
422
- // ---------------------------------------------------------------------------
423
-
424
- /** Hex-line outline — regex heuristic (no tree-sitter in benchmark) */
425
- function simHexLineOutline(lines) {
426
- const structural = /^\s*(export\s+)?(function|class|def|async\s+def|impl|fn|pub\s+fn|struct|interface|type|enum|const|let|var)\b/;
427
- const importLine = /^\s*(import|from|require|use|#include)/;
428
- const entries = [];
429
- let importStart = -1, importEnd = -1, importCount = 0;
430
-
431
- for (let i = 0; i < lines.length; i++) {
432
- if (importLine.test(lines[i])) {
433
- if (importStart === -1) importStart = i + 1;
434
- importEnd = i + 1;
435
- importCount++;
436
- continue;
437
- }
438
- if (structural.test(lines[i])) {
439
- let end = lines.length;
440
- for (let j = i + 1; j < lines.length; j++) {
441
- if (structural.test(lines[j])) { end = j; break; }
442
- }
443
- entries.push(`${i + 1}-${end}: ${lines[i].trim().slice(0, 120)}`);
444
- }
445
- }
446
-
447
- const parts = [];
448
- if (importCount > 0) parts.push(`${importStart}-${importEnd}: (${importCount} imports/declarations)`);
449
- parts.push(...entries);
450
- parts.push("", `(${entries.length} symbols, ${lines.length} source lines)`);
451
- return `File: benchmark-target\n\n${parts.join("\n")}`;
452
- }
453
-
454
- /** Hex-line outline + targeted read of first function (30 lines) */
455
- function simHexLineOutlinePlusRead(filePath, lines) {
456
- const outlineStr = simHexLineOutline(lines);
457
- const structural = /^\s*(export\s+)?(function|class|def|async\s+def|impl|fn|pub\s+fn|struct)\b/;
458
- let funcStart = 0;
459
- for (let i = 0; i < lines.length; i++) {
460
- if (structural.test(lines[i])) { funcStart = i + 1; break; }
461
- }
462
- const start = Math.max(1, funcStart);
463
- const readStr = readFile(filePath, { offset: start, limit: 30 });
464
- return outlineStr + "\n---\n" + readStr;
465
- }
466
-
467
- /** Hex-line grep — hash-annotated format */
468
- function simHexLineGrep(filePath, lines, pattern) {
469
- const re = new RegExp(pattern, "i");
470
- const matches = [];
471
- for (let i = 0; i < lines.length; i++) {
472
- if (re.test(lines[i])) {
473
- const tag = lineTag(fnv1a(lines[i]));
474
- matches.push(`${filePath}:>>${tag}.${i + 1}\t${lines[i]}`);
475
- }
476
- }
477
- return matches.length > 0
478
- ? "```\n" + matches.join("\n") + "\n```"
479
- : "No matches found.";
480
- }
481
-
482
- /** Hex-line write response */
483
- function simHexLineWrite(filePath, content) {
484
- const lineCount = content.split("\n").length;
485
- return `Created ${filePath} (${lineCount} lines)`;
486
- }
487
-
488
- /** Hex-line edit response: compact diff hunks */
489
- function simHexLineEditDiff(origLines, newLines, ctx = 3) {
490
- const out = [];
491
- const maxLen = Math.max(origLines.length, newLines.length);
492
- let i = 0;
493
-
494
- while (i < maxLen) {
495
- if (i < origLines.length && i < newLines.length && origLines[i] === newLines[i]) {
496
- i++;
497
- continue;
498
- }
499
- // Found a difference — show context before
500
- const ctxStart = Math.max(0, i - ctx);
501
- if (ctxStart < i) {
502
- if (ctxStart > 0) out.push("...");
503
- for (let k = ctxStart; k < i; k++) {
504
- out.push(` ${k + 1}| ${origLines[k]}`);
505
- }
506
- }
507
- // Show changed lines
508
- const changeStart = i;
509
- while (i < maxLen && (i >= origLines.length || i >= newLines.length || origLines[i] !== newLines[i])) {
510
- if (i < origLines.length) out.push(`-${i + 1}| ${origLines[i]}`);
511
- i++;
512
- }
513
- for (let k = changeStart; k < i && k < newLines.length; k++) {
514
- out.push(`+${k + 1}| ${newLines[k]}`);
515
- }
516
- // Context after
517
- const ctxEnd = Math.min(maxLen, i + ctx);
518
- for (let k = i; k < ctxEnd && k < origLines.length; k++) {
519
- out.push(` ${k + 1}| ${origLines[k]}`);
520
- }
521
- if (ctxEnd < maxLen) out.push("...");
522
- break;
523
- }
524
-
525
- const diff = out.join("\n");
526
- return diff
527
- ? `Updated benchmark-file\n\nDiff:\n\`\`\`diff\n${diff}\n\`\`\``
528
- : "Updated benchmark-file";
529
- }
530
-
531
- // ---------------------------------------------------------------------------
532
- // Runner utilities
533
- // ---------------------------------------------------------------------------
534
-
535
- function median(arr) {
536
- const sorted = [...arr].sort((a, b) => a - b);
537
- const mid = Math.floor(sorted.length / 2);
538
- return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
539
- }
540
-
541
- function runN(fn, n = RUNS) {
542
- const results = [];
543
- for (let i = 0; i < n; i++) results.push(fn());
544
- return median(results);
545
- }
546
-
547
- function fmt(n) {
548
- return n.toLocaleString("en-US", { maximumFractionDigits: 0 });
549
- }
550
-
551
- function pctSavings(without, withSL) {
552
- if (without === 0) return "N/A";
553
- const pct = ((without - withSL) / without) * 100;
554
- return pct >= 0 ? `${pct.toFixed(0)}%` : `-${Math.abs(pct).toFixed(0)}%`;
555
- }
46
+ const withGraph = args.includes("--with-graph");
556
47
 
557
48
  // ---------------------------------------------------------------------------
558
49
  // Main
559
50
  // ---------------------------------------------------------------------------
560
51
 
561
- function main() {
52
+ async function main() {
562
53
  const allFiles = walkDir(repoRoot);
563
54
  if (allFiles.length === 0) {
564
55
  console.log(`No code files found in ${repoRoot}`);
@@ -573,6 +64,11 @@ function main() {
573
64
  const cats = categorize(allFiles);
574
65
  const repoName = basename(repoRoot);
575
66
 
67
+ // Top 3 largest code files for realistic tests
68
+ const sorted = allFiles.map(f => ({ f, lines: getFileLines(f)?.length || 0 }))
69
+ .sort((a, b) => b.lines - a.lines);
70
+ const largeFiles = sorted.slice(0, 3).map(s => s.f);
71
+
576
72
  // Temp file setup
577
73
  const ts = Date.now();
578
74
  const tmpPath = resolve(tmpdir(), `hex-line-bench-${ts}.js`);
@@ -598,14 +94,17 @@ function main() {
598
94
  }
599
95
 
600
96
  if (withoutArr.length === 0) continue;
601
- const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
602
- const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
97
+ const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
98
+ const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
99
+ const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
100
+ const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
603
101
 
604
102
  const label = { small: "<50L", medium: "50-200L", large: "200-500L", xl: "500L+" }[cat];
605
103
  results.push({
606
104
  num: 1, scenario: `Read full (${label})`,
607
105
  without: avgWithout, withSL: avgWith,
608
106
  savings: pctSavings(avgWithout, avgWith),
107
+ latencyWithout: avgMsWithout, latencyWith: avgMsWith,
609
108
  });
610
109
  }
611
110
 
@@ -626,14 +125,17 @@ function main() {
626
125
  }
627
126
 
628
127
  if (withoutArr.length === 0) continue;
629
- const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
630
- const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
128
+ const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
129
+ const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
130
+ const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
131
+ const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
631
132
 
632
133
  const label = cat === "large" ? "200-500L" : "500L+";
633
134
  results.push({
634
135
  num: 2, scenario: `Outline+read (${label})`,
635
136
  without: avgWithout, withSL: avgWith,
636
137
  savings: pctSavings(avgWithout, avgWith),
138
+ latencyWithout: avgMsWithout, latencyWith: avgMsWith,
637
139
  });
638
140
  }
639
141
 
@@ -650,17 +152,20 @@ function main() {
650
152
  const lines = getFileLines(f);
651
153
  if (!lines) continue;
652
154
  const pattern = "function|class|const";
653
- withoutArr.push(runN(() => simBuiltInGrep(f, lines, pattern).length));
155
+ withoutArr.push(runN(() => simBuiltInGrep(pattern, f).length));
654
156
  withArr.push(runN(() => simHexLineGrep(f, lines, pattern).length));
655
157
  }
656
158
 
657
159
  if (withoutArr.length > 0) {
658
- const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
659
- const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
160
+ const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
161
+ const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
162
+ const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
163
+ const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
660
164
  results.push({
661
165
  num: 3, scenario: "Grep search",
662
166
  without: avgWithout, withSL: avgWith,
663
167
  savings: pctSavings(avgWithout, avgWith),
168
+ latencyWithout: avgMsWithout, latencyWith: avgMsWith,
664
169
  });
665
170
  }
666
171
  }
@@ -670,12 +175,13 @@ function main() {
670
175
  // TEST 4: Directory tree
671
176
  // ===================================================================
672
177
  {
673
- const without = runN(() => simBuiltInLsR(repoRoot, 0, 3).length);
674
- const withSL = runN(() => directoryTree(repoRoot, { max_depth: 3 }).length);
178
+ const { value: without, ms: withoutMs } = runN(() => simBuiltInLsR(repoRoot, 0, 3).length);
179
+ const { value: withSL, ms: withMs } = runN(() => directoryTree(repoRoot, { max_depth: 3 }).length);
675
180
  results.push({
676
181
  num: 4, scenario: "Directory tree",
677
182
  without, withSL,
678
183
  savings: pctSavings(without, withSL),
184
+ latencyWithout: withoutMs, latencyWith: withMs,
679
185
  });
680
186
  }
681
187
 
@@ -684,12 +190,13 @@ function main() {
684
190
  // ===================================================================
685
191
  {
686
192
  const infoFile = allFiles[Math.floor(allFiles.length / 2)] || allFiles[0];
687
- const without = runN(() => simBuiltInStat(infoFile).length);
688
- const withSL = runN(() => fileInfo(infoFile).length);
193
+ const { value: without, ms: withoutMs } = runN(() => simBuiltInStat(infoFile).length);
194
+ const { value: withSL, ms: withMs } = runN(() => fileInfo(infoFile).length);
689
195
  results.push({
690
196
  num: 5, scenario: "File info",
691
197
  without, withSL,
692
198
  savings: pctSavings(without, withSL),
199
+ latencyWithout: withoutMs, latencyWith: withMs,
693
200
  });
694
201
  }
695
202
 
@@ -697,12 +204,13 @@ function main() {
697
204
  // TEST 6: Create file (write)
698
205
  // ===================================================================
699
206
  {
700
- const without = runN(() => simBuiltInWrite(tmpPath, tmpContent).length);
701
- const withSL = runN(() => simHexLineWrite(tmpPath, tmpContent).length);
207
+ const { value: without, ms: withoutMs } = runN(() => simBuiltInWrite(tmpPath, tmpContent).length);
208
+ const { value: withSL, ms: withMs } = runN(() => simHexLineWrite(tmpPath, tmpContent).length);
702
209
  results.push({
703
210
  num: 6, scenario: "Create file (200L)",
704
211
  without, withSL,
705
212
  savings: pctSavings(without, withSL),
213
+ latencyWithout: withoutMs, latencyWith: withMs,
706
214
  });
707
215
  }
708
216
 
@@ -720,6 +228,8 @@ function main() {
720
228
 
721
229
  let totalWithout = 0;
722
230
  let totalWith = 0;
231
+ let totalMsWithout = 0;
232
+ let totalMsWith = 0;
723
233
 
724
234
  for (const edit of editTargets) {
725
235
  const origLines = [...tmpLines];
@@ -729,14 +239,19 @@ function main() {
729
239
  newLines[idx] = edit.new;
730
240
  }
731
241
 
732
- totalWithout += runN(() => simBuiltInEdit(tmpPath, origLines, newLines).length);
733
- totalWith += runN(() => simHexLineEditDiff(origLines, newLines).length);
242
+ const rW = runN(() => simBuiltInEdit(tmpPath, origLines, newLines).length);
243
+ const rH = runN(() => simHexLineEditDiff(origLines, newLines).length);
244
+ totalWithout += rW.value;
245
+ totalWith += rH.value;
246
+ totalMsWithout += rW.ms;
247
+ totalMsWith += rH.ms;
734
248
  }
735
249
 
736
250
  results.push({
737
251
  num: 7, scenario: "Edit x5 sequential",
738
252
  without: totalWithout, withSL: totalWith,
739
253
  savings: pctSavings(totalWithout, totalWith),
254
+ latencyWithout: parseFloat(totalMsWithout.toFixed(1)), latencyWith: parseFloat(totalMsWith.toFixed(1)),
740
255
  });
741
256
  }
742
257
 
@@ -752,13 +267,14 @@ function main() {
752
267
  const cs4 = rangeChecksum(hashes.slice(150, 200), 151, 200);
753
268
  const checksums = [cs1, cs2, cs3, cs4];
754
269
 
755
- const without = runN(() => simBuiltInVerify(tmpPath, fileLines).length);
756
- const withSL = runN(() => verifyChecksums(tmpPath, checksums).length);
270
+ const { value: without, ms: withoutMs } = runN(() => simBuiltInVerify(tmpPath, fileLines).length);
271
+ const { value: withSL, ms: withMs } = runN(() => verifyChecksums(tmpPath, checksums).length);
757
272
 
758
273
  results.push({
759
274
  num: 8, scenario: "Verify checksums (4 ranges)",
760
275
  without, withSL,
761
276
  savings: pctSavings(without, withSL),
277
+ latencyWithout: withoutMs, latencyWith: withMs,
762
278
  });
763
279
  }
764
280
 
@@ -769,7 +285,7 @@ function main() {
769
285
  const batchFiles = (cats.small || []).slice(0, 3);
770
286
  if (batchFiles.length >= 2) {
771
287
  // Without hex-line: N separate Read calls
772
- const without = runN(() => {
288
+ const { value: without, ms: withoutMs } = runN(() => {
773
289
  let total = 0;
774
290
  for (const f of batchFiles) {
775
291
  const lines = getFileLines(f);
@@ -779,7 +295,7 @@ function main() {
779
295
  });
780
296
 
781
297
  // With hex-line: 1 read_file call with paths:[] — concatenated output
782
- const withSL = runN(() => {
298
+ const { value: withSL, ms: withMs } = runN(() => {
783
299
  const parts = [];
784
300
  for (const f of batchFiles) {
785
301
  parts.push(readFile(f));
@@ -791,6 +307,7 @@ function main() {
791
307
  num: 9, scenario: `Multi-file read (${batchFiles.length} files)`,
792
308
  without, withSL,
793
309
  savings: pctSavings(without, withSL),
310
+ latencyWithout: withoutMs, latencyWith: withMs,
794
311
  });
795
312
  }
796
313
  }
@@ -810,7 +327,7 @@ function main() {
810
327
  const editNew = ' this.configPath = resolve(configPath || ".");';
811
328
 
812
329
  // Without hex-line: 5 separate edit_file calls
813
- const without = runN(() => {
330
+ const { value: without, ms: withoutMs } = runN(() => {
814
331
  let total = 0;
815
332
  for (const p of bulkTmpPaths) {
816
333
  const origLines = [...tmpLines];
@@ -822,7 +339,7 @@ function main() {
822
339
  });
823
340
 
824
341
  // With hex-line: 1 bulk_replace — summary + per-file compact diff
825
- const withSL = runN(() => {
342
+ const { value: withSL, ms: withMs } = runN(() => {
826
343
  let response = "5 files changed, 0 errors\n";
827
344
  for (const p of bulkTmpPaths) {
828
345
  const origLines = [...tmpLines];
@@ -837,6 +354,7 @@ function main() {
837
354
  num: 10, scenario: "bulk_replace dry_run (5 files)",
838
355
  without, withSL,
839
356
  savings: pctSavings(without, withSL),
357
+ latencyWithout: withoutMs, latencyWith: withMs,
840
358
  });
841
359
 
842
360
  for (const p of bulkTmpPaths) {
@@ -849,7 +367,7 @@ function main() {
849
367
  // ===================================================================
850
368
  {
851
369
  // Without hex-line: raw unified diff output
852
- const without = runN(() => {
370
+ const { value: without, ms: withoutMs } = runN(() => {
853
371
  const diffLines = [
854
372
  `diff --git a/benchmark-target.js b/benchmark-target.js`,
855
373
  `index abc1234..def5678 100644`,
@@ -880,22 +398,23 @@ function main() {
880
398
  return diffLines.join("\n").length;
881
399
  });
882
400
 
883
- // With hex-line: semantic changes summary
884
- const withSL = runN(() => {
885
- const changes = [
886
- "Added:",
887
- " + formatDuration (line 201, 5 lines)",
888
- "Modified:",
889
- " ~ ConfigManager.constructor (line 12, +1 line)",
890
- "Summary: 1 added, 1 modified",
891
- ];
892
- return changes.join("\n").length;
893
- });
401
+ // With hex-line: real fileChanges() semantic diff (async, called once — deterministic)
402
+ let withSL;
403
+ let withMs = 0;
404
+ try {
405
+ const t0 = performance.now();
406
+ const changesOut = await fileChanges(allFiles[0]);
407
+ withMs = parseFloat((performance.now() - t0).toFixed(1));
408
+ withSL = changesOut.length;
409
+ } catch {
410
+ withSL = 133; // fallback if no git history
411
+ }
894
412
 
895
413
  results.push({
896
414
  num: 11, scenario: "Changes (semantic diff)",
897
415
  without, withSL,
898
416
  savings: pctSavings(without, withSL),
417
+ latencyWithout: withoutMs, latencyWith: withMs,
899
418
  });
900
419
  }
901
420
 
@@ -907,35 +426,32 @@ function main() {
907
426
  const parentDir = resolve(repoRoot, "src/utils");
908
427
 
909
428
  // Without hex-line: 3 round-trips (error → ls → retry)
910
- const without = runN(() => {
911
- // Round 1: error
912
- const r1 = `Error: ENOENT: no such file or directory, open '${missingPath}'`;
913
- // Round 2: agent calls ls to find correct name
914
- const dirEntries = [];
915
- for (let i = 0; i < 10; i++) {
916
- dirEntries.push(`-rw-r--r-- 1 user group 1234 2025-03-20 10:00 file_${i}.js`);
917
- }
918
- const r2 = `${parentDir}:\ntotal 10\n${dirEntries.join("\n")}`;
429
+ const { value: without, ms: withoutMs } = runN(() => {
430
+ // Round 1: real ENOENT error
431
+ let r1;
432
+ try { readFileSync(missingPath, "utf-8"); r1 = ""; } catch (e) { r1 = e.message; }
433
+ // Round 2: real directory listing to find correct name
434
+ let r2;
435
+ try { r2 = readdirSync(parentDir).join("\n"); } catch { r2 = `${parentDir}: directory not found`; }
919
436
  // Round 3: agent re-reads correct file (small file ~30 lines)
920
437
  const r3 = simBuiltInReadFull(missingPath, tmpLines.slice(0, 30));
921
438
  return (r1 + r2 + r3).length;
922
439
  });
923
440
 
924
- // With hex-line: 1 round-trip (error + parent dir listing)
925
- const withSL = runN(() => {
926
- const entries = [];
927
- for (let i = 0; i < 10; i++) {
928
- entries.push(` file_${i}.js`);
441
+ // With hex-line: real readFile() on nonexistent path — returns error + parent dir listing
442
+ const { value: withSL, ms: withMs } = runN(() => {
443
+ try {
444
+ return readFile(missingPath).length;
445
+ } catch (e) {
446
+ return e.message.length;
929
447
  }
930
- const response = `FILE_NOT_FOUND: ${missingPath}\n` +
931
- `Parent directory (${parentDir}):\n${entries.join("\n")}`;
932
- return response.length;
933
448
  });
934
449
 
935
450
  results.push({
936
- num: 12, scenario: "FILE_NOT_FOUND recovery",
451
+ num: 12, scenario: "FILE_NOT_FOUND recovery*",
937
452
  without, withSL,
938
453
  savings: pctSavings(without, withSL),
454
+ latencyWithout: withoutMs, latencyWith: withMs,
939
455
  });
940
456
  }
941
457
 
@@ -944,7 +460,7 @@ function main() {
944
460
  // ===================================================================
945
461
  {
946
462
  // Without hex-line: 3 round-trips (stale error → re-read full → retry edit)
947
- const without = runN(() => {
463
+ const { value: without, ms: withoutMs } = runN(() => {
948
464
  // Round 1: error
949
465
  const r1 = 'Error: file content has changed (stale). Please re-read the file.';
950
466
  // Round 2: full re-read
@@ -958,7 +474,7 @@ function main() {
958
474
  });
959
475
 
960
476
  // With hex-line: 1 round-trip (error + fresh snippet +/-5 lines around target)
961
- const withSL = runN(() => {
477
+ const { value: withSL, ms: withMs } = runN(() => {
962
478
  const targetLine = 13;
963
479
  const snippetStart = Math.max(0, targetLine - 6);
964
480
  const snippetEnd = Math.min(tmpLines.length, targetLine + 5);
@@ -973,9 +489,10 @@ function main() {
973
489
  });
974
490
 
975
491
  results.push({
976
- num: 13, scenario: "Hash mismatch recovery",
492
+ num: 13, scenario: "Hash mismatch recovery*",
977
493
  without, withSL,
978
494
  savings: pctSavings(without, withSL),
495
+ latencyWithout: withoutMs, latencyWith: withMs,
979
496
  });
980
497
  }
981
498
 
@@ -987,30 +504,313 @@ function main() {
987
504
  const infoLines = getFileLines(infoFile);
988
505
  if (infoLines) {
989
506
  // Sub-test A: cat vs read_file
990
- const catWithout = runN(() => {
507
+ const catW = runN(() => {
991
508
  // cat output: raw lines, no line numbers (agent redirect)
992
509
  return infoLines.join("\n").length;
993
510
  });
994
- const catWith = runN(() => readFile(infoFile).length);
511
+ const catH = runN(() => readFile(infoFile).length);
995
512
 
996
513
  // Sub-test B: ls -la vs directory_tree
997
514
  const dirTarget = resolve(repoRoot);
998
- const lsWithout = runN(() => simBuiltInLsR(dirTarget, 0, 1).length);
999
- const lsWith = runN(() => directoryTree(dirTarget, { max_depth: 1 }).length);
515
+ const lsW = runN(() => simBuiltInLsR(dirTarget, 0, 1).length);
516
+ const lsH = runN(() => directoryTree(dirTarget, { max_depth: 1 }).length);
1000
517
 
1001
518
  // Sub-test C: stat vs get_file_info
1002
- const statWithout = runN(() => simBuiltInStat(infoFile).length);
1003
- const statWith = runN(() => fileInfo(infoFile).length);
519
+ const stW = runN(() => simBuiltInStat(infoFile).length);
520
+ const stH = runN(() => fileInfo(infoFile).length);
1004
521
 
1005
522
  // Combined: without = raw outputs (no follow-up possible)
1006
523
  // With = structured output (enables follow-up without extra calls)
1007
- const totalWithout = catWithout + lsWithout + statWithout;
1008
- const totalWith = catWith + lsWith + statWith;
524
+ const totalWithout = catW.value + lsW.value + stW.value;
525
+ const totalWith = catH.value + lsH.value + stH.value;
526
+ const totalMsWithout = catW.ms + lsW.ms + stW.ms;
527
+ const totalMsWith = catH.ms + lsH.ms + stH.ms;
1009
528
 
1010
529
  results.push({
1011
530
  num: 14, scenario: "Bash redirects (cat+ls+stat)",
1012
531
  without: totalWithout, withSL: totalWith,
1013
532
  savings: pctSavings(totalWithout, totalWith),
533
+ latencyWithout: parseFloat(totalMsWithout.toFixed(1)), latencyWith: parseFloat(totalMsWith.toFixed(1)),
534
+ });
535
+ }
536
+ }
537
+
538
+ // ===================================================================
539
+ // TEST 15: HASH_HINT multi-match recovery
540
+ // ===================================================================
541
+ {
542
+ // Create a file with a duplicated line so textReplace triggers HASH_HINT
543
+ const dupLine = ' return this.config;';
544
+ const dupContent = tmpLines.map((l, i) => (i === 20 || i === 80) ? dupLine : l);
545
+ const dupPath = resolve(tmpdir(), `hex-line-dup-${ts}.js`);
546
+ writeFileSync(dupPath, dupContent.join("\n"), "utf-8");
547
+
548
+ // Without hex-line: 3 round-trips (opaque error + re-read full + retry)
549
+ const { value: without, ms: withoutMs } = runN(() => {
550
+ const r1 = 'Error: multiple occurrences found. Provide more context.';
551
+ const r2 = simBuiltInReadFull(dupPath, dupContent);
552
+ const origLines = [...dupContent];
553
+ const newLines = [...dupContent];
554
+ newLines[20] = ' return this.updatedConfig;';
555
+ const r3 = simBuiltInEdit(dupPath, origLines, newLines);
556
+ return (r1 + r2 + r3).length;
557
+ });
558
+
559
+ // With hex-line: HASH_HINT error contains annotated snippets (1 round-trip)
560
+ const { value: withSL, ms: withMs } = runN(() => {
561
+ try {
562
+ editFile(dupPath, [{ replace: { old_text: dupLine, new_text: ' return this.updatedConfig;' } }]);
563
+ return 0; // should not reach
564
+ } catch (e) {
565
+ // HASH_HINT error message + simulated anchor retry
566
+ const retry = '{"set_line":{"anchor":"xx.21","new_text":" return this.updatedConfig;"}}';
567
+ return (e.message + retry).length;
568
+ }
569
+ });
570
+
571
+ results.push({
572
+ num: 15, scenario: "HASH_HINT multi-match recovery*",
573
+ without, withSL,
574
+ savings: pctSavings(without, withSL),
575
+ latencyWithout: withoutMs, latencyWith: withMs,
576
+ });
577
+
578
+ try { unlinkSync(dupPath); } catch { /* ok */ }
579
+ }
580
+
581
+ // ===================================================================
582
+ // TEST 16-18: Graph enrichment (--with-graph only)
583
+ // Both sides use hex-line; difference is whether .codegraph/index.db exists
584
+ // ===================================================================
585
+ const graphOut = [];
586
+ if (withGraph) {
587
+ const { getGraphDB, getRelativePath } = await import("./lib/graph-enrich.mjs");
588
+ const db = getGraphDB(resolve(repoRoot, "server.mjs"));
589
+ if (!db) {
590
+ console.error("--with-graph: .codegraph/index.db not found. Run hex-graph index_project first.");
591
+ } else {
592
+ const graphFile = largeFiles[0] || allFiles[0];
593
+ const graphLines = getFileLines(graphFile);
594
+
595
+ if (graphLines) {
596
+ // TEST 16: Read with/without Graph header
597
+ {
598
+ const withGraphResult = readFile(graphFile);
599
+ const noGraphResult = withGraphResult.replace(/\nGraph:.*\n/, "\n");
600
+ const savings = pctSavings(noGraphResult.length, withGraphResult.length);
601
+ graphOut.push(`| 16 | Graph: Read (${graphLines.length}L) | ${fmt(noGraphResult.length)} chars | ${fmt(withGraphResult.length)} chars | ${savings} | 2\u21921 | 2\u21921 |`);
602
+ }
603
+
604
+ // TEST 17: Edit with/without blast radius
605
+ {
606
+ const editTmpPath = resolve(tmpdir(), `hex-bench-edit-${Date.now()}.js`);
607
+ writeFileSync(editTmpPath, graphLines.join("\n"), "utf-8");
608
+ try {
609
+ const editResult = editFile(editTmpPath, [{ replace: { old_text: graphLines[5], new_text: graphLines[5] + " // modified" } }]);
610
+ const noBlastOut = editResult.replace(/\n.*Blast radius.*$/s, "");
611
+ const savings = pctSavings(noBlastOut.length, editResult.length);
612
+ graphOut.push(`| 17 | Graph: Edit + impact | ${fmt(noBlastOut.length)} chars | ${fmt(editResult.length)} chars | ${savings} | 2\u21921 | 2\u21921 |`);
613
+ } catch (e) {
614
+ graphOut.push(`| 17 | Graph: Edit + impact | \u2014 | \u2014 | \u2014 | | |`);
615
+ }
616
+ try { unlinkSync(editTmpPath); } catch {}
617
+ }
618
+
619
+ // TEST 18: Grep with/without annotations
620
+ {
621
+ try {
622
+ const grepResult = await grepSearch("function", { path: resolve(repoRoot), glob: "*.mjs", limit: 10 });
623
+ const noAnnoResult = grepResult.replace(/ \[[^\]]+\]/g, "");
624
+ const savings = pctSavings(noAnnoResult.length, grepResult.length);
625
+ const annoCount = (grepResult.match(/\[[^\]]+\]/g) || []).length;
626
+ graphOut.push(`| 18 | Graph: Grep + ${annoCount} annotations | ${fmt(noAnnoResult.length)} chars | ${fmt(grepResult.length)} chars | ${savings} | 6\u21921 | 6\u21921 |`);
627
+ } catch {
628
+ graphOut.push(`| 18 | Graph: Grep + context | \u2014 | \u2014 | \u2014 | | |`);
629
+ }
630
+ }
631
+ }
632
+ }
633
+ }
634
+
635
+ // ===================================================================
636
+ // WORKFLOW SCENARIOS (multi-step real operations)
637
+ // ===================================================================
638
+ const workflowResults = [];
639
+
640
+ // W1: Search → Edit (find a pattern, edit the match)
641
+ {
642
+ const wTmpPath = resolve(tmpdir(), `hex-wf1-${Date.now()}.js`);
643
+ writeFileSync(wTmpPath, tmpContent, "utf-8");
644
+ const editLine = tmpLines[12];
645
+ const editNew = editLine + " // workflow-modified";
646
+
647
+ // Without: grep → read file for context → edit with old_string
648
+ const { value: without } = runN(() => {
649
+ let total = 0;
650
+ // Step 1: grep to find
651
+ total += simBuiltInGrep("configPath", wTmpPath).length;
652
+ // Step 2: read full file for context (agent needs surrounding lines)
653
+ total += simBuiltInReadFull(wTmpPath, tmpLines).length;
654
+ // Step 3: edit
655
+ const origLines = [...tmpLines];
656
+ const newLines = [...tmpLines];
657
+ newLines[12] = editNew;
658
+ total += simBuiltInEdit(wTmpPath, origLines, newLines).length;
659
+ return total;
660
+ });
661
+
662
+ // With: grep_search (has hashes) → edit with anchor (no re-read needed)
663
+ const { value: withSL } = runN(() => {
664
+ let total = 0;
665
+ // Step 1: grep with hashes
666
+ const grepOut = readFileSync(wTmpPath, "utf-8"); // simulate grep result
667
+ const lines = grepOut.split("\n");
668
+ const targetIdx = 12;
669
+ const tag = lineTag(fnv1a(lines[targetIdx]));
670
+ total += `${wTmpPath}:>>${tag}.${targetIdx + 1}\t${lines[targetIdx]}`.length;
671
+ // Step 2: edit with anchor directly (no read needed)
672
+ try {
673
+ const result = editFile(wTmpPath, [{ set_line: { anchor: `${tag}.${targetIdx + 1}`, new_text: editNew } }]);
674
+ total += result.length;
675
+ } catch (e) { total += e.message.length; }
676
+ return total;
677
+ });
678
+
679
+ workflowResults.push({
680
+ id: "W1", scenario: "Search \u2192 Edit",
681
+ without, withSL,
682
+ opsWithout: 3, opsWith: 2,
683
+ });
684
+ try { unlinkSync(wTmpPath); } catch {}
685
+ }
686
+
687
+ // W2: Read → Edit → Verify cycle
688
+ {
689
+ const wTmpPath = resolve(tmpdir(), `hex-wf2-${Date.now()}.js`);
690
+ writeFileSync(wTmpPath, tmpContent, "utf-8");
691
+
692
+ // Without: read full → edit → re-read full to verify
693
+ const { value: without } = runN(() => {
694
+ let total = 0;
695
+ total += simBuiltInReadFull(wTmpPath, tmpLines).length; // read
696
+ const origLines = [...tmpLines];
697
+ const newLines = [...tmpLines];
698
+ newLines[12] = ' this.configPath = resolve(configPath || ".");';
699
+ total += simBuiltInEdit(wTmpPath, origLines, newLines).length; // edit
700
+ total += simBuiltInReadFull(wTmpPath, tmpLines).length; // re-read to verify
701
+ return total;
702
+ });
703
+
704
+ // With: read targeted → edit → verify checksums
705
+ const { value: withSL } = runN(() => {
706
+ let total = 0;
707
+ total += readFile(wTmpPath, { offset: 8, limit: 20 }).length; // targeted read
708
+ // Reset file for edit
709
+ writeFileSync(wTmpPath, tmpContent, "utf-8");
710
+ try {
711
+ const result = editFile(wTmpPath, [{ replace: { old_text: tmpLines[12], new_text: ' this.configPath = resolve(configPath || ".");' } }]);
712
+ total += result.length;
713
+ } catch (e) { total += e.message.length; }
714
+ // Verify with checksums instead of re-reading
715
+ const hashes = tmpLines.slice(0, 50).map(l => fnv1a(l));
716
+ const cs = rangeChecksum(hashes, 1, 50);
717
+ try { total += verifyChecksums(wTmpPath, [cs]).length; }
718
+ catch { total += 100; }
719
+ return total;
720
+ });
721
+
722
+ workflowResults.push({
723
+ id: "W2", scenario: "Read \u2192 Edit \u2192 Verify",
724
+ without, withSL,
725
+ opsWithout: 3, opsWith: 3,
726
+ });
727
+ try { unlinkSync(wTmpPath); } catch {}
728
+ }
729
+
730
+ // W3: Multi-file refactor (rename in 5 files)
731
+ {
732
+ const wDir = resolve(tmpdir(), `hex-wf3-${Date.now()}`);
733
+ mkdirSync(wDir, { recursive: true });
734
+ const wPaths = [];
735
+ for (let i = 0; i < 5; i++) {
736
+ const p = resolve(wDir, `file-${i}.js`);
737
+ writeFileSync(p, tmpContent, "utf-8");
738
+ wPaths.push(p);
739
+ }
740
+
741
+ // Without: grep to find files → read each → edit each = 11 ops
742
+ const { value: without } = runN(() => {
743
+ let total = 0;
744
+ total += simBuiltInGrep("configPath", wPaths[0]).length; // find
745
+ for (const p of wPaths) {
746
+ total += simBuiltInReadFull(p, tmpLines).length; // read each
747
+ const origLines = [...tmpLines];
748
+ const newLines = [...tmpLines];
749
+ newLines[12] = newLines[12].replace("configPath", "settingsPath");
750
+ total += simBuiltInEdit(p, origLines, newLines).length; // edit each
751
+ }
752
+ return total;
753
+ });
754
+
755
+ // With: grep_search → bulk_replace = 2 ops
756
+ const { value: withSL } = runN(() => {
757
+ let total = 0;
758
+ // Restore files
759
+ for (const p of wPaths) writeFileSync(p, tmpContent, "utf-8");
760
+ // Single grep (simulated — bulk_replace does its own finding)
761
+ total += 200; // approximate grep output
762
+ // Single bulk_replace
763
+ const result = bulkReplace(
764
+ wDir,
765
+ "*.js",
766
+ [{ old: "configPath", new: "settingsPath" }],
767
+ { dryRun: true, maxFiles: 10 }
768
+ );
769
+ total += result.length;
770
+ return total;
771
+ });
772
+
773
+ workflowResults.push({
774
+ id: "W3", scenario: "Multi-file refactor (5 files)",
775
+ without, withSL,
776
+ opsWithout: 11, opsWith: 2,
777
+ });
778
+ try { rmSync(wDir, { recursive: true }); } catch {}
779
+ }
780
+
781
+ // W4: Explore large file → targeted edit
782
+ {
783
+ const largeFile = largeFiles[0] || allFiles[0];
784
+ const largeLines = getFileLines(largeFile);
785
+ if (largeLines && largeLines.length > 100) {
786
+ // Without: read full file → grep for method → edit
787
+ const { value: without } = runN(() => {
788
+ let total = 0;
789
+ total += simBuiltInReadFull(largeFile, largeLines).length;
790
+ total += simBuiltInGrep("function", largeFile).length;
791
+ // Simulate edit response
792
+ const origLines = [...largeLines];
793
+ const newLines = [...largeLines];
794
+ newLines[10] = newLines[10] + " // modified";
795
+ total += simBuiltInEdit(largeFile, origLines, newLines).length;
796
+ return total;
797
+ });
798
+
799
+ // With: outline → read range → edit with anchor
800
+ let outlineLen = 500;
801
+ try { outlineLen = (await fileOutline(largeFile)).length; } catch {}
802
+ const { value: withSL } = runN(() => {
803
+ let total = 0;
804
+ total += outlineLen; // outline (pre-computed, async)
805
+ total += readFile(largeFile, { offset: 5, limit: 30 }).length; // targeted read
806
+ total += simHexLineEditDiff(largeLines.slice(5, 35), [...largeLines.slice(5, 35)].map((l, i) => i === 5 ? l + " // modified" : l)).length;
807
+ return total;
808
+ });
809
+
810
+ workflowResults.push({
811
+ id: "W4", scenario: `Explore+edit (${largeLines.length}L file)`,
812
+ without, withSL,
813
+ opsWithout: 3, opsWith: 3,
1014
814
  });
1015
815
  }
1016
816
  }
@@ -1024,7 +824,7 @@ function main() {
1024
824
  // Report
1025
825
  // ===================================================================
1026
826
  const out = [];
1027
- out.push("# Hex-line Benchmark v2");
827
+ out.push("# Hex-line Benchmark v3");
1028
828
  out.push("");
1029
829
  out.push(`Repository: ${repoName} (${fmt(allFiles.length)} code files, ${fmt(totalLines)} lines) `);
1030
830
  out.push(`Temp file: ${tmpPath} (200 lines) `);
@@ -1032,17 +832,97 @@ function main() {
1032
832
  out.push(`Runs per scenario: ${RUNS} (median) `);
1033
833
  out.push("");
1034
834
 
1035
- // Results table
835
+ // Ops comparison: how many tool calls each scenario requires
836
+ const OPS = {
837
+ "Read full (<50L)": { without: 1, with: 1 },
838
+ "Read full (50-200L)": { without: 1, with: 1 },
839
+ "Read full (200-500L)": { without: 1, with: 1 },
840
+ "Read full (500L+)": { without: 1, with: 1 },
841
+ "Outline+read (200-500L)": { without: 1, with: 2 },
842
+ "Outline+read (500L+)": { without: 1, with: 2 },
843
+ "Grep search": { without: 1, with: 1 },
844
+ "Directory tree": { without: 1, with: 1 },
845
+ "File info": { without: 1, with: 1 },
846
+ "Create file (200L)": { without: 1, with: 1 },
847
+ "Edit x5 sequential": { without: 5, with: 5 },
848
+ "Verify checksums (4 ranges)": { without: 4, with: 1 },
849
+ "Multi-file read": { without: 2, with: 1 },
850
+ "bulk_replace dry_run (5 files)": { without: 5, with: 1 },
851
+ "Changes (semantic diff)": { without: 1, with: 1 },
852
+ "FILE_NOT_FOUND recovery*": { without: 3, with: 1 },
853
+ "Hash mismatch recovery*": { without: 3, with: 1 },
854
+ "Bash redirects (cat+ls+stat)": { without: 3, with: 3 },
855
+ "HASH_HINT multi-match recovery*": { without: 3, with: 2 },
856
+ };
857
+
858
+ const STEPS = {
859
+ "Read full (<50L)": { without: 1, with: 1 },
860
+ "Read full (50-200L)": { without: 1, with: 1 },
861
+ "Read full (200-500L)": { without: 1, with: 1 },
862
+ "Read full (500L+)": { without: 1, with: 1 },
863
+ "Outline+read (200-500L)": { without: 1, with: 2 },
864
+ "Outline+read (500L+)": { without: 1, with: 2 },
865
+ "Grep search": { without: 1, with: 1 },
866
+ "Directory tree": { without: 1, with: 1 },
867
+ "File info": { without: 1, with: 1 },
868
+ "Create file (200L)": { without: 1, with: 1 },
869
+ "Edit x5 sequential": { without: 5, with: 5 },
870
+ "Verify checksums (4 ranges)": { without: 4, with: 1 },
871
+ "Multi-file read": { without: 1, with: 1 },
872
+ "bulk_replace dry_run (5 files)": { without: 5, with: 1 },
873
+ "Changes (semantic diff)": { without: 1, with: 1 },
874
+ "FILE_NOT_FOUND recovery": { without: 3, with: 1 },
875
+ "Hash mismatch recovery": { without: 3, with: 1 },
876
+ "Bash redirects (cat+ls+stat)": { without: 1, with: 1 },
877
+ "HASH_HINT multi-match recovery": { without: 3, with: 1 },
878
+ };
879
+
880
+ // Combined results + ops + steps table
1036
881
  out.push("## Results");
1037
882
  out.push("");
1038
- out.push("| # | Scenario | Without Hex-line | With Hex-line | Savings |");
1039
- out.push("|---|----------|-------------------|----------------|---------|");
883
+ out.push("| # | Scenario | Baseline | Hex-line | Savings | Ops | Steps |");
884
+ out.push("|---|----------|----------|----------|---------|-----|-------|");
1040
885
 
1041
886
  for (const r of results) {
1042
- out.push(`| ${r.num} | ${r.scenario} | ${fmt(r.without)} chars | ${fmt(r.withSL)} chars | ${r.savings} |`);
887
+ if (r.num >= 16) continue; // graph rows added below
888
+
889
+ // Match OPS/STEPS keys
890
+ let op = OPS[r.scenario];
891
+ if (!op) {
892
+ const key = Object.keys(OPS).find(k => r.scenario.startsWith(k));
893
+ if (key) op = OPS[key];
894
+ }
895
+ let step = STEPS[r.scenario];
896
+ if (!step) {
897
+ const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k));
898
+ if (key) step = STEPS[key];
899
+ }
900
+
901
+ const opsStr = op ? `${op.without}\u2192${op.with}` : "\u2014";
902
+ const stepsStr = step ? `${step.without}\u2192${step.with}` : "\u2014";
903
+
904
+ out.push(`| ${r.num} | ${r.scenario} | ${fmt(r.without)} chars | ${fmt(r.withSL)} chars | ${r.savings} | ${opsStr} | ${stepsStr} |`);
905
+ }
906
+
907
+ // Append graph rows into same table (if any)
908
+ if (graphOut.length > 0) {
909
+ out.push("| | **hex-line \u00b1 graph** | **No Graph** | **With Graph** | | | |");
910
+ out.push(...graphOut);
1043
911
  }
1044
912
  out.push("");
1045
913
 
914
+ // Workflow scenarios table
915
+ if (workflowResults.length > 0) {
916
+ out.push("## Workflow Scenarios (multi-step)");
917
+ out.push("");
918
+ out.push("| # | Scenario | Built-in | Hex-line | Savings | Ops |");
919
+ out.push("|---|----------|----------|----------|---------|-----|");
920
+ for (const w of workflowResults) {
921
+ out.push(`| ${w.id} | ${w.scenario} | ${fmt(w.without)} chars | ${fmt(w.withSL)} chars | ${pctSavings(w.without, w.withSL)} | ${w.opsWithout}\u2192${w.opsWith} |`);
922
+ }
923
+ out.push("");
924
+ }
925
+
1046
926
  // Verdict
1047
927
  out.push("## Verdict");
1048
928
  out.push("");
@@ -1059,11 +939,57 @@ function main() {
1059
939
  const mismatchResult = results.find(r => r.num === 13);
1060
940
  const bashResult = results.find(r => r.num === 14);
1061
941
 
1062
- const allSavingsNums = results.map(r => {
942
+ const coreResults = results.filter(r => r.num < 16);
943
+ const allSavingsNums = coreResults.map(r => {
1063
944
  if (r.without === 0) return 0;
1064
945
  return ((r.without - r.withSL) / r.without) * 100;
1065
946
  });
1066
- const avgSavings = allSavingsNums.reduce((a, b) => a + b, 0) / allSavingsNums.length;
947
+ const avgSavings = allSavingsNums.length > 0
948
+ ? allSavingsNums.reduce((a, b) => a + b, 0) / allSavingsNums.length
949
+ : 0;
950
+
951
+ // Weighted average based on typical development session frequency
952
+ const WEIGHTS = {
953
+ "Read full (<50L)": 2, "Read full (50-200L)": 5, "Read full (200-500L)": 3, "Read full (500L+)": 1,
954
+ "Outline+read (200-500L)": 8, "Outline+read (500L+)": 8,
955
+ "Grep search": 5, "Directory tree": 2, "File info": 1, "Create file (200L)": 1,
956
+ "Edit x5 sequential": 10, "Verify checksums (4 ranges)": 8,
957
+ "Multi-file read": 2, "bulk_replace dry_run (5 files)": 1,
958
+ "Changes (semantic diff)": 3,
959
+ "FILE_NOT_FOUND recovery": 2, "Hash mismatch recovery": 3,
960
+ "Bash redirects (cat+ls+stat)": 3, "HASH_HINT multi-match recovery": 2,
961
+ };
962
+ let wSum = 0, wTotal = 0;
963
+ for (const r of coreResults) {
964
+ const w = WEIGHTS[r.scenario] || 1;
965
+ const sav = r.without === 0 ? 0 : ((r.without - r.withSL) / r.without) * 100;
966
+ wSum += w * sav;
967
+ wTotal += w;
968
+ }
969
+ const weightedAvg = wTotal > 0 ? wSum / wTotal : 0;
970
+
971
+ // Ops/Steps totals for core scenarios
972
+ const totalOpsWithout = coreResults.reduce((s, r) => {
973
+ let op = OPS[r.scenario];
974
+ if (!op) { const key = Object.keys(OPS).find(k => r.scenario.startsWith(k)); if (key) op = OPS[key]; }
975
+ return s + (op ? op.without : 1);
976
+ }, 0);
977
+ const totalOpsWith = coreResults.reduce((s, r) => {
978
+ let op = OPS[r.scenario];
979
+ if (!op) { const key = Object.keys(OPS).find(k => r.scenario.startsWith(k)); if (key) op = OPS[key]; }
980
+ return s + (op ? op.with : 1);
981
+ }, 0);
982
+ const totalStepsWithout = coreResults.reduce((s, r) => {
983
+ let step = STEPS[r.scenario];
984
+ if (!step) { const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k)); if (key) step = STEPS[key]; }
985
+ return s + (step ? step.without : 1);
986
+ }, 0);
987
+ const totalStepsWith = coreResults.reduce((s, r) => {
988
+ let step = STEPS[r.scenario];
989
+ if (!step) { const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k)); if (key) step = STEPS[key]; }
990
+ return s + (step ? step.with : 1);
991
+ }, 0);
992
+ const opsPct = totalOpsWithout > 0 ? ((totalOpsWithout - totalOpsWith) / totalOpsWithout * 100).toFixed(0) : 0;
1067
993
 
1068
994
  // Read verdict
1069
995
  const readVerdict = [];
@@ -1171,7 +1097,7 @@ function main() {
1171
1097
  if (changesResult && changesResult.withSL < changesResult.without) {
1172
1098
  out.push("- **Semantic diff** always cheaper than raw unified diff for understanding changes.");
1173
1099
  }
1174
- out.push(`- **Average savings across all ${results.length} scenarios:** ${avgSavings.toFixed(0)}%`);
1100
+ out.push(`- **Average:** ${avgSavings.toFixed(0)}% tokens (flat) / ${weightedAvg.toFixed(0)}% (weighted) | ${totalOpsWithout}\u2192${totalOpsWith} ops (${opsPct}% fewer) | ${totalStepsWithout}\u2192${totalStepsWith} steps`);
1175
1101
  out.push("");
1176
1102
 
1177
1103
  console.log(out.join("\n"));