@levnikolaevich/hex-line-mcp 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -19
- package/benchmark.mjs +524 -598
- package/hook.mjs +62 -19
- package/lib/benchmark-helpers.mjs +541 -0
- package/lib/bulk-replace.mjs +8 -0
- package/lib/coerce.mjs +5 -0
- package/lib/edit.mjs +71 -31
- package/lib/read.mjs +44 -19
- package/lib/setup.mjs +134 -16
- package/lib/tree.mjs +84 -9
- package/output-style.md +27 -0
- package/package.json +3 -2
- package/server.mjs +23 -24
package/benchmark.mjs
CHANGED
|
@@ -1,25 +1,37 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* Hex-line Combo Benchmark
|
|
3
|
+
* Hex-line Combo Benchmark v3
|
|
4
4
|
*
|
|
5
5
|
* Compares "agent without hex-line" vs "agent with hex-line" across
|
|
6
6
|
* read-only and write scenarios. Measures chars in response (proxy for tokens).
|
|
7
7
|
*
|
|
8
|
-
* Usage: node mcp/hex-line-mcp/benchmark.mjs [--repo /path/to/repo]
|
|
8
|
+
* Usage: node mcp/hex-line-mcp/benchmark.mjs [--repo /path/to/repo] [--with-graph]
|
|
9
9
|
* Default repo: current working directory.
|
|
10
10
|
*
|
|
11
11
|
* Zero external deps beyond hex-line lib modules.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import { readFileSync, writeFileSync, unlinkSync,
|
|
15
|
-
import {
|
|
14
|
+
import { readFileSync, writeFileSync, unlinkSync, readdirSync, mkdirSync, rmSync } from "node:fs";
|
|
15
|
+
import { performance } from "node:perf_hooks";
|
|
16
|
+
import { resolve, basename } from "node:path";
|
|
16
17
|
import { tmpdir } from "node:os";
|
|
17
18
|
import { fnv1a, lineTag, rangeChecksum } from "./lib/hash.mjs";
|
|
18
19
|
import { readFile } from "./lib/read.mjs";
|
|
19
20
|
import { directoryTree } from "./lib/tree.mjs";
|
|
20
21
|
import { fileInfo } from "./lib/info.mjs";
|
|
21
22
|
import { verifyChecksums } from "./lib/verify.mjs";
|
|
22
|
-
|
|
23
|
+
import { fileChanges } from "./lib/changes.mjs";
|
|
24
|
+
import { editFile } from "./lib/edit.mjs";
|
|
25
|
+
import { grepSearch } from "./lib/search.mjs";
|
|
26
|
+
import { bulkReplace } from "./lib/bulk-replace.mjs";
|
|
27
|
+
import { fileOutline } from "./lib/outline.mjs";
|
|
28
|
+
import {
|
|
29
|
+
walkDir, getFileLines, categorize, generateTempCode,
|
|
30
|
+
simBuiltInReadFull, simBuiltInOutlineFull, simBuiltInGrep,
|
|
31
|
+
simBuiltInLsR, simBuiltInStat, simBuiltInWrite, simBuiltInEdit, simBuiltInVerify,
|
|
32
|
+
simHexLineOutlinePlusRead, simHexLineGrep, simHexLineWrite, simHexLineEditDiff,
|
|
33
|
+
runN, fmt, pctSavings, RUNS,
|
|
34
|
+
} from "./lib/benchmark-helpers.mjs";
|
|
23
35
|
// ---------------------------------------------------------------------------
|
|
24
36
|
// CLI
|
|
25
37
|
// ---------------------------------------------------------------------------
|
|
@@ -31,534 +43,13 @@ if (repoIdx !== -1 && args[repoIdx + 1]) {
|
|
|
31
43
|
repoRoot = resolve(args[repoIdx + 1]);
|
|
32
44
|
}
|
|
33
45
|
|
|
34
|
-
const
|
|
35
|
-
const MAX_FILES_PER_CAT = 3;
|
|
36
|
-
const RUNS = 3;
|
|
37
|
-
|
|
38
|
-
// ---------------------------------------------------------------------------
|
|
39
|
-
// File discovery
|
|
40
|
-
// ---------------------------------------------------------------------------
|
|
41
|
-
|
|
42
|
-
function walkDir(dir, depth = 0) {
|
|
43
|
-
if (depth > 10) return [];
|
|
44
|
-
const results = [];
|
|
45
|
-
let entries;
|
|
46
|
-
try { entries = readdirSync(dir, { withFileTypes: true }); }
|
|
47
|
-
catch { return results; }
|
|
48
|
-
for (const e of entries) {
|
|
49
|
-
const full = resolve(dir, e.name);
|
|
50
|
-
if (e.isDirectory()) {
|
|
51
|
-
if (e.name.startsWith(".") || e.name === "node_modules" || e.name === "vendor"
|
|
52
|
-
|| e.name === "dist" || e.name === "__pycache__" || e.name === "target") continue;
|
|
53
|
-
results.push(...walkDir(full, depth + 1));
|
|
54
|
-
} else if (e.isFile() && CODE_EXTS.has(extname(e.name).toLowerCase())) {
|
|
55
|
-
try {
|
|
56
|
-
const st = statSync(full);
|
|
57
|
-
if (st.size > 0 && st.size < 1_000_000) results.push(full);
|
|
58
|
-
} catch { /* skip */ }
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
return results;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function getFileLines(f) {
|
|
65
|
-
try { return readFileSync(f, "utf-8").replace(/\r\n/g, "\n").split("\n"); }
|
|
66
|
-
catch { return null; }
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function categorize(files) {
|
|
70
|
-
const cats = { small: [], medium: [], large: [], xl: [] };
|
|
71
|
-
for (const f of files) {
|
|
72
|
-
const lines = getFileLines(f);
|
|
73
|
-
if (!lines) continue;
|
|
74
|
-
const n = lines.length;
|
|
75
|
-
if (n >= 10 && n <= 50) cats.small.push(f);
|
|
76
|
-
else if (n > 50 && n <= 200) cats.medium.push(f);
|
|
77
|
-
else if (n > 200 && n <= 500) cats.large.push(f);
|
|
78
|
-
else if (n > 500) cats.xl.push(f);
|
|
79
|
-
}
|
|
80
|
-
for (const key of Object.keys(cats)) {
|
|
81
|
-
const arr = cats[key];
|
|
82
|
-
if (arr.length > MAX_FILES_PER_CAT) {
|
|
83
|
-
const step = Math.floor(arr.length / MAX_FILES_PER_CAT);
|
|
84
|
-
cats[key] = Array.from({ length: MAX_FILES_PER_CAT }, (_, i) => arr[i * step]);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
return cats;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
// ---------------------------------------------------------------------------
|
|
91
|
-
// Temp file: 200 lines of realistic JS
|
|
92
|
-
// ---------------------------------------------------------------------------
|
|
93
|
-
|
|
94
|
-
function generateTempCode() {
|
|
95
|
-
const lines = [];
|
|
96
|
-
lines.push('import { readFileSync } from "node:fs";');
|
|
97
|
-
lines.push('import { resolve, basename } from "node:path";');
|
|
98
|
-
lines.push("");
|
|
99
|
-
lines.push("const DEFAULT_TIMEOUT = 5000;");
|
|
100
|
-
lines.push("const MAX_RETRIES = 3;");
|
|
101
|
-
lines.push("");
|
|
102
|
-
lines.push("/**");
|
|
103
|
-
lines.push(" * Configuration manager for application settings.");
|
|
104
|
-
lines.push(" * Supports file-based and environment-based config sources.");
|
|
105
|
-
lines.push(" */");
|
|
106
|
-
lines.push("class ConfigManager {");
|
|
107
|
-
lines.push(" constructor(configPath) {");
|
|
108
|
-
lines.push(" this.configPath = resolve(configPath);");
|
|
109
|
-
lines.push(" this.cache = new Map();");
|
|
110
|
-
lines.push(" this.watchers = [];");
|
|
111
|
-
lines.push(" this.loaded = false;");
|
|
112
|
-
lines.push(" }");
|
|
113
|
-
lines.push("");
|
|
114
|
-
lines.push(" load() {");
|
|
115
|
-
lines.push(" const raw = readFileSync(this.configPath, 'utf-8');");
|
|
116
|
-
lines.push(" const parsed = JSON.parse(raw);");
|
|
117
|
-
lines.push(" for (const [key, value] of Object.entries(parsed)) {");
|
|
118
|
-
lines.push(" this.cache.set(key, value);");
|
|
119
|
-
lines.push(" }");
|
|
120
|
-
lines.push(" this.loaded = true;");
|
|
121
|
-
lines.push(" this.notifyWatchers('load', parsed);");
|
|
122
|
-
lines.push(" return this;");
|
|
123
|
-
lines.push(" }");
|
|
124
|
-
lines.push("");
|
|
125
|
-
lines.push(" get(key, defaultValue = undefined) {");
|
|
126
|
-
lines.push(" if (!this.loaded) this.load();");
|
|
127
|
-
lines.push(" return this.cache.has(key) ? this.cache.get(key) : defaultValue;");
|
|
128
|
-
lines.push(" }");
|
|
129
|
-
lines.push("");
|
|
130
|
-
lines.push(" set(key, value) {");
|
|
131
|
-
lines.push(" this.cache.set(key, value);");
|
|
132
|
-
lines.push(" this.notifyWatchers('set', { key, value });");
|
|
133
|
-
lines.push(" }");
|
|
134
|
-
lines.push("");
|
|
135
|
-
lines.push(" watch(callback) {");
|
|
136
|
-
lines.push(" this.watchers.push(callback);");
|
|
137
|
-
lines.push(" return () => {");
|
|
138
|
-
lines.push(" this.watchers = this.watchers.filter(w => w !== callback);");
|
|
139
|
-
lines.push(" };");
|
|
140
|
-
lines.push(" }");
|
|
141
|
-
lines.push("");
|
|
142
|
-
lines.push(" notifyWatchers(event, data) {");
|
|
143
|
-
lines.push(" for (const watcher of this.watchers) {");
|
|
144
|
-
lines.push(" try { watcher(event, data); }");
|
|
145
|
-
lines.push(" catch (e) { console.error('Watcher error:', e.message); }");
|
|
146
|
-
lines.push(" }");
|
|
147
|
-
lines.push(" }");
|
|
148
|
-
lines.push("}");
|
|
149
|
-
lines.push("");
|
|
150
|
-
lines.push("/**");
|
|
151
|
-
lines.push(" * Retry wrapper with exponential backoff.");
|
|
152
|
-
lines.push(" */");
|
|
153
|
-
lines.push("async function withRetry(fn, options = {}) {");
|
|
154
|
-
lines.push(" const { retries = MAX_RETRIES, delay = 100, backoff = 2 } = options;");
|
|
155
|
-
lines.push(" let lastError;");
|
|
156
|
-
lines.push(" for (let attempt = 0; attempt <= retries; attempt++) {");
|
|
157
|
-
lines.push(" try {");
|
|
158
|
-
lines.push(" return await fn(attempt);");
|
|
159
|
-
lines.push(" } catch (err) {");
|
|
160
|
-
lines.push(" lastError = err;");
|
|
161
|
-
lines.push(" if (attempt < retries) {");
|
|
162
|
-
lines.push(" const wait = delay * Math.pow(backoff, attempt);");
|
|
163
|
-
lines.push(" await new Promise(r => setTimeout(r, wait));");
|
|
164
|
-
lines.push(" }");
|
|
165
|
-
lines.push(" }");
|
|
166
|
-
lines.push(" }");
|
|
167
|
-
lines.push(" throw lastError;");
|
|
168
|
-
lines.push("}");
|
|
169
|
-
lines.push("");
|
|
170
|
-
lines.push("/**");
|
|
171
|
-
lines.push(" * HTTP client with timeout and retry support.");
|
|
172
|
-
lines.push(" */");
|
|
173
|
-
lines.push("class HttpClient {");
|
|
174
|
-
lines.push(" constructor(baseUrl, options = {}) {");
|
|
175
|
-
lines.push(" this.baseUrl = baseUrl.replace(/\\/$/, '');");
|
|
176
|
-
lines.push(" this.timeout = options.timeout || DEFAULT_TIMEOUT;");
|
|
177
|
-
lines.push(" this.headers = options.headers || {};");
|
|
178
|
-
lines.push(" this.retries = options.retries || MAX_RETRIES;");
|
|
179
|
-
lines.push(" }");
|
|
180
|
-
lines.push("");
|
|
181
|
-
lines.push(" async request(method, path, body = null) {");
|
|
182
|
-
lines.push(" const url = `${this.baseUrl}${path}`;");
|
|
183
|
-
lines.push(" const controller = new AbortController();");
|
|
184
|
-
lines.push(" const timer = setTimeout(() => controller.abort(), this.timeout);");
|
|
185
|
-
lines.push("");
|
|
186
|
-
lines.push(" try {");
|
|
187
|
-
lines.push(" return await withRetry(async () => {");
|
|
188
|
-
lines.push(" const opts = {");
|
|
189
|
-
lines.push(" method,");
|
|
190
|
-
lines.push(" headers: { ...this.headers },");
|
|
191
|
-
lines.push(" signal: controller.signal,");
|
|
192
|
-
lines.push(" };");
|
|
193
|
-
lines.push(" if (body) {");
|
|
194
|
-
lines.push(" opts.headers['Content-Type'] = 'application/json';");
|
|
195
|
-
lines.push(" opts.body = JSON.stringify(body);");
|
|
196
|
-
lines.push(" }");
|
|
197
|
-
lines.push(" const response = await fetch(url, opts);");
|
|
198
|
-
lines.push(" if (!response.ok) {");
|
|
199
|
-
lines.push(" throw new Error(`HTTP ${response.status}: ${response.statusText}`);");
|
|
200
|
-
lines.push(" }");
|
|
201
|
-
lines.push(" return response.json();");
|
|
202
|
-
lines.push(" }, { retries: this.retries });");
|
|
203
|
-
lines.push(" } finally {");
|
|
204
|
-
lines.push(" clearTimeout(timer);");
|
|
205
|
-
lines.push(" }");
|
|
206
|
-
lines.push(" }");
|
|
207
|
-
lines.push("");
|
|
208
|
-
lines.push(" get(path) { return this.request('GET', path); }");
|
|
209
|
-
lines.push(" post(path, body) { return this.request('POST', path, body); }");
|
|
210
|
-
lines.push(" put(path, body) { return this.request('PUT', path, body); }");
|
|
211
|
-
lines.push(" delete(path) { return this.request('DELETE', path); }");
|
|
212
|
-
lines.push("}");
|
|
213
|
-
lines.push("");
|
|
214
|
-
lines.push("/**");
|
|
215
|
-
lines.push(" * Simple event emitter for pub/sub patterns.");
|
|
216
|
-
lines.push(" */");
|
|
217
|
-
lines.push("class EventEmitter {");
|
|
218
|
-
lines.push(" constructor() {");
|
|
219
|
-
lines.push(" this.listeners = new Map();");
|
|
220
|
-
lines.push(" }");
|
|
221
|
-
lines.push("");
|
|
222
|
-
lines.push(" on(event, handler) {");
|
|
223
|
-
lines.push(" if (!this.listeners.has(event)) {");
|
|
224
|
-
lines.push(" this.listeners.set(event, []);");
|
|
225
|
-
lines.push(" }");
|
|
226
|
-
lines.push(" this.listeners.get(event).push(handler);");
|
|
227
|
-
lines.push(" return this;");
|
|
228
|
-
lines.push(" }");
|
|
229
|
-
lines.push("");
|
|
230
|
-
lines.push(" off(event, handler) {");
|
|
231
|
-
lines.push(" const handlers = this.listeners.get(event);");
|
|
232
|
-
lines.push(" if (handlers) {");
|
|
233
|
-
lines.push(" this.listeners.set(event, handlers.filter(h => h !== handler));");
|
|
234
|
-
lines.push(" }");
|
|
235
|
-
lines.push(" return this;");
|
|
236
|
-
lines.push(" }");
|
|
237
|
-
lines.push("");
|
|
238
|
-
lines.push(" emit(event, ...args) {");
|
|
239
|
-
lines.push(" const handlers = this.listeners.get(event) || [];");
|
|
240
|
-
lines.push(" for (const handler of handlers) {");
|
|
241
|
-
lines.push(" handler(...args);");
|
|
242
|
-
lines.push(" }");
|
|
243
|
-
lines.push(" }");
|
|
244
|
-
lines.push("");
|
|
245
|
-
lines.push(" once(event, handler) {");
|
|
246
|
-
lines.push(" const wrapper = (...args) => {");
|
|
247
|
-
lines.push(" handler(...args);");
|
|
248
|
-
lines.push(" this.off(event, wrapper);");
|
|
249
|
-
lines.push(" };");
|
|
250
|
-
lines.push(" return this.on(event, wrapper);");
|
|
251
|
-
lines.push(" }");
|
|
252
|
-
lines.push("}");
|
|
253
|
-
lines.push("");
|
|
254
|
-
lines.push("/**");
|
|
255
|
-
lines.push(" * Validate and sanitize user input.");
|
|
256
|
-
lines.push(" */");
|
|
257
|
-
lines.push("function validateInput(schema, data) {");
|
|
258
|
-
lines.push(" const errors = [];");
|
|
259
|
-
lines.push(" for (const [field, rules] of Object.entries(schema)) {");
|
|
260
|
-
lines.push(" const value = data[field];");
|
|
261
|
-
lines.push(" if (rules.required && (value === undefined || value === null)) {");
|
|
262
|
-
lines.push(" errors.push(`${field} is required`);");
|
|
263
|
-
lines.push(" continue;");
|
|
264
|
-
lines.push(" }");
|
|
265
|
-
lines.push(" if (value !== undefined && rules.type && typeof value !== rules.type) {");
|
|
266
|
-
lines.push(" errors.push(`${field} must be ${rules.type}`);");
|
|
267
|
-
lines.push(" }");
|
|
268
|
-
lines.push(" if (typeof value === 'string' && rules.maxLength && value.length > rules.maxLength) {");
|
|
269
|
-
lines.push(" errors.push(`${field} exceeds max length ${rules.maxLength}`);");
|
|
270
|
-
lines.push(" }");
|
|
271
|
-
lines.push(" if (typeof value === 'number' && rules.min !== undefined && value < rules.min) {");
|
|
272
|
-
lines.push(" errors.push(`${field} must be >= ${rules.min}`);");
|
|
273
|
-
lines.push(" }");
|
|
274
|
-
lines.push(" }");
|
|
275
|
-
lines.push(" return errors.length > 0 ? { valid: false, errors } : { valid: true };");
|
|
276
|
-
lines.push("}");
|
|
277
|
-
lines.push("");
|
|
278
|
-
lines.push("/**");
|
|
279
|
-
lines.push(" * Format bytes to human-readable string.");
|
|
280
|
-
lines.push(" */");
|
|
281
|
-
lines.push("function formatBytes(bytes) {");
|
|
282
|
-
lines.push(" if (bytes === 0) return '0 B';");
|
|
283
|
-
lines.push(" const units = ['B', 'KB', 'MB', 'GB', 'TB'];");
|
|
284
|
-
lines.push(" const exp = Math.floor(Math.log(bytes) / Math.log(1024));");
|
|
285
|
-
lines.push(" const value = bytes / Math.pow(1024, exp);");
|
|
286
|
-
lines.push(" return `${value.toFixed(exp > 0 ? 1 : 0)} ${units[exp]}`;");
|
|
287
|
-
lines.push("}");
|
|
288
|
-
lines.push("");
|
|
289
|
-
lines.push("/**");
|
|
290
|
-
lines.push(" * Deep merge two objects (source into target).");
|
|
291
|
-
lines.push(" */");
|
|
292
|
-
lines.push("function deepMerge(target, source) {");
|
|
293
|
-
lines.push(" const result = { ...target };");
|
|
294
|
-
lines.push(" for (const key of Object.keys(source)) {");
|
|
295
|
-
lines.push(" if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {");
|
|
296
|
-
lines.push(" result[key] = deepMerge(result[key] || {}, source[key]);");
|
|
297
|
-
lines.push(" } else {");
|
|
298
|
-
lines.push(" result[key] = source[key];");
|
|
299
|
-
lines.push(" }");
|
|
300
|
-
lines.push(" }");
|
|
301
|
-
lines.push(" return result;");
|
|
302
|
-
lines.push("}");
|
|
303
|
-
lines.push("");
|
|
304
|
-
lines.push("export { ConfigManager, HttpClient, EventEmitter, withRetry, validateInput, formatBytes, deepMerge };");
|
|
305
|
-
|
|
306
|
-
// Pad to exactly 200 lines
|
|
307
|
-
while (lines.length < 200) lines.push("");
|
|
308
|
-
return lines.slice(0, 200);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
// ---------------------------------------------------------------------------
|
|
312
|
-
// Simulators — "without hex-line" (built-in tool output)
|
|
313
|
-
// ---------------------------------------------------------------------------
|
|
314
|
-
|
|
315
|
-
/** Simulate built-in Read: `cat -n` full file with header */
|
|
316
|
-
function simBuiltInReadFull(filePath, lines) {
|
|
317
|
-
const body = lines.map((l, i) => ` ${String(i + 1).padStart(5)}\t${l}`).join("\n");
|
|
318
|
-
return `Contents of ${filePath}:\n\n${body}`;
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
/** Simulate outline via full read — agent reads entire file to understand structure */
|
|
322
|
-
function simBuiltInOutlineFull(filePath, lines) {
|
|
323
|
-
return simBuiltInReadFull(filePath, lines);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
/** Simulate ripgrep raw output (no hashes) */
|
|
327
|
-
function simBuiltInGrep(filePath, lines, pattern) {
|
|
328
|
-
const re = new RegExp(pattern, "i");
|
|
329
|
-
const matches = [];
|
|
330
|
-
for (let i = 0; i < lines.length; i++) {
|
|
331
|
-
if (re.test(lines[i])) {
|
|
332
|
-
matches.push(`${filePath}:${i + 1}:${lines[i]}`);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
return matches.join("\n") || "No matches found.";
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
/** Simulate `ls -laR` style output for a directory */
|
|
339
|
-
function simBuiltInLsR(dirPath, depth = 0, maxDepth = 3) {
|
|
340
|
-
if (depth > maxDepth) return "";
|
|
341
|
-
const out = [];
|
|
342
|
-
let entries;
|
|
343
|
-
try { entries = readdirSync(dirPath, { withFileTypes: true }); }
|
|
344
|
-
catch { return ""; }
|
|
345
|
-
|
|
346
|
-
const SKIP = new Set(["node_modules", ".git", "dist", "build", "__pycache__", "coverage"]);
|
|
347
|
-
|
|
348
|
-
out.push(`${dirPath}:`);
|
|
349
|
-
out.push("total " + entries.length);
|
|
350
|
-
|
|
351
|
-
for (const e of entries) {
|
|
352
|
-
if (SKIP.has(e.name) && e.isDirectory()) continue;
|
|
353
|
-
const full = join(dirPath, e.name);
|
|
354
|
-
try {
|
|
355
|
-
const st = statSync(full);
|
|
356
|
-
const type = e.isDirectory() ? "d" : "-";
|
|
357
|
-
const size = String(st.size).padStart(8);
|
|
358
|
-
const date = st.mtime.toISOString().slice(0, 16).replace("T", " ");
|
|
359
|
-
out.push(`${type}rw-r--r-- 1 user group ${size} ${date} ${e.name}`);
|
|
360
|
-
} catch { /* skip */ }
|
|
361
|
-
}
|
|
362
|
-
out.push("");
|
|
363
|
-
|
|
364
|
-
for (const e of entries) {
|
|
365
|
-
if (!e.isDirectory()) continue;
|
|
366
|
-
if (SKIP.has(e.name)) continue;
|
|
367
|
-
const full = join(dirPath, e.name);
|
|
368
|
-
const sub = simBuiltInLsR(full, depth + 1, maxDepth);
|
|
369
|
-
if (sub) out.push(sub);
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
return out.join("\n");
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
/** Simulate `stat` output for a file */
|
|
376
|
-
function simBuiltInStat(filePath) {
|
|
377
|
-
const st = statSync(filePath);
|
|
378
|
-
return [
|
|
379
|
-
` File: ${filePath}`,
|
|
380
|
-
` Size: ${st.size}\tBlocks: ${Math.ceil(st.size / 512)}\tIO Block: 4096\tregular file`,
|
|
381
|
-
`Device: 0h/0d\tInode: 0\tLinks: 1`,
|
|
382
|
-
`Access: (0644/-rw-r--r--)\tUid: ( 1000/ user)\tGid: ( 1000/ group)`,
|
|
383
|
-
`Access: ${st.atime.toISOString()}`,
|
|
384
|
-
`Modify: ${st.mtime.toISOString()}`,
|
|
385
|
-
`Change: ${st.ctime.toISOString()}`,
|
|
386
|
-
` Birth: ${st.birthtime.toISOString()}`,
|
|
387
|
-
].join("\n");
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
/** Simulate built-in write response */
|
|
391
|
-
function simBuiltInWrite(filePath, content) {
|
|
392
|
-
const lineCount = content.split("\n").length;
|
|
393
|
-
return `File ${filePath} has been created successfully (${lineCount} lines).`;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
/** Simulate built-in edit: old_string/new_string context blocks */
|
|
397
|
-
function simBuiltInEdit(filePath, origLines, newLines) {
|
|
398
|
-
let changeStart = -1, changeEnd = -1;
|
|
399
|
-
for (let i = 0; i < Math.max(origLines.length, newLines.length); i++) {
|
|
400
|
-
if (origLines[i] !== newLines[i]) {
|
|
401
|
-
if (changeStart === -1) changeStart = i;
|
|
402
|
-
changeEnd = i;
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
if (changeStart === -1) return "";
|
|
406
|
-
|
|
407
|
-
const ctxBefore = Math.max(0, changeStart - 3);
|
|
408
|
-
const ctxAfter = Math.min(origLines.length, changeEnd + 4);
|
|
409
|
-
const old_string = origLines.slice(ctxBefore, ctxAfter).join("\n");
|
|
410
|
-
const new_string = newLines.slice(ctxBefore, Math.min(newLines.length, changeEnd + 4)).join("\n");
|
|
411
|
-
return `The file ${filePath} has been edited. Here's the result of running \`cat -n\` on a snippet:\n` +
|
|
412
|
-
`old_string:\n${old_string}\nnew_string:\n${new_string}`;
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
/** Simulate built-in verify: full re-read to check if file changed */
|
|
416
|
-
function simBuiltInVerify(filePath, lines) {
|
|
417
|
-
return simBuiltInReadFull(filePath, lines);
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
// ---------------------------------------------------------------------------
|
|
421
|
-
// Simulators — "with hex-line" (lib function output)
|
|
422
|
-
// ---------------------------------------------------------------------------
|
|
423
|
-
|
|
424
|
-
/** Hex-line outline — regex heuristic (no tree-sitter in benchmark) */
|
|
425
|
-
function simHexLineOutline(lines) {
|
|
426
|
-
const structural = /^\s*(export\s+)?(function|class|def|async\s+def|impl|fn|pub\s+fn|struct|interface|type|enum|const|let|var)\b/;
|
|
427
|
-
const importLine = /^\s*(import|from|require|use|#include)/;
|
|
428
|
-
const entries = [];
|
|
429
|
-
let importStart = -1, importEnd = -1, importCount = 0;
|
|
430
|
-
|
|
431
|
-
for (let i = 0; i < lines.length; i++) {
|
|
432
|
-
if (importLine.test(lines[i])) {
|
|
433
|
-
if (importStart === -1) importStart = i + 1;
|
|
434
|
-
importEnd = i + 1;
|
|
435
|
-
importCount++;
|
|
436
|
-
continue;
|
|
437
|
-
}
|
|
438
|
-
if (structural.test(lines[i])) {
|
|
439
|
-
let end = lines.length;
|
|
440
|
-
for (let j = i + 1; j < lines.length; j++) {
|
|
441
|
-
if (structural.test(lines[j])) { end = j; break; }
|
|
442
|
-
}
|
|
443
|
-
entries.push(`${i + 1}-${end}: ${lines[i].trim().slice(0, 120)}`);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
const parts = [];
|
|
448
|
-
if (importCount > 0) parts.push(`${importStart}-${importEnd}: (${importCount} imports/declarations)`);
|
|
449
|
-
parts.push(...entries);
|
|
450
|
-
parts.push("", `(${entries.length} symbols, ${lines.length} source lines)`);
|
|
451
|
-
return `File: benchmark-target\n\n${parts.join("\n")}`;
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
/** Hex-line outline + targeted read of first function (30 lines) */
|
|
455
|
-
function simHexLineOutlinePlusRead(filePath, lines) {
|
|
456
|
-
const outlineStr = simHexLineOutline(lines);
|
|
457
|
-
const structural = /^\s*(export\s+)?(function|class|def|async\s+def|impl|fn|pub\s+fn|struct)\b/;
|
|
458
|
-
let funcStart = 0;
|
|
459
|
-
for (let i = 0; i < lines.length; i++) {
|
|
460
|
-
if (structural.test(lines[i])) { funcStart = i + 1; break; }
|
|
461
|
-
}
|
|
462
|
-
const start = Math.max(1, funcStart);
|
|
463
|
-
const readStr = readFile(filePath, { offset: start, limit: 30 });
|
|
464
|
-
return outlineStr + "\n---\n" + readStr;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
/** Hex-line grep — hash-annotated format */
|
|
468
|
-
function simHexLineGrep(filePath, lines, pattern) {
|
|
469
|
-
const re = new RegExp(pattern, "i");
|
|
470
|
-
const matches = [];
|
|
471
|
-
for (let i = 0; i < lines.length; i++) {
|
|
472
|
-
if (re.test(lines[i])) {
|
|
473
|
-
const tag = lineTag(fnv1a(lines[i]));
|
|
474
|
-
matches.push(`${filePath}:>>${tag}.${i + 1}\t${lines[i]}`);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
return matches.length > 0
|
|
478
|
-
? "```\n" + matches.join("\n") + "\n```"
|
|
479
|
-
: "No matches found.";
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
/** Hex-line write response */
|
|
483
|
-
function simHexLineWrite(filePath, content) {
|
|
484
|
-
const lineCount = content.split("\n").length;
|
|
485
|
-
return `Created ${filePath} (${lineCount} lines)`;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
/** Hex-line edit response: compact diff hunks */
|
|
489
|
-
function simHexLineEditDiff(origLines, newLines, ctx = 3) {
|
|
490
|
-
const out = [];
|
|
491
|
-
const maxLen = Math.max(origLines.length, newLines.length);
|
|
492
|
-
let i = 0;
|
|
493
|
-
|
|
494
|
-
while (i < maxLen) {
|
|
495
|
-
if (i < origLines.length && i < newLines.length && origLines[i] === newLines[i]) {
|
|
496
|
-
i++;
|
|
497
|
-
continue;
|
|
498
|
-
}
|
|
499
|
-
// Found a difference — show context before
|
|
500
|
-
const ctxStart = Math.max(0, i - ctx);
|
|
501
|
-
if (ctxStart < i) {
|
|
502
|
-
if (ctxStart > 0) out.push("...");
|
|
503
|
-
for (let k = ctxStart; k < i; k++) {
|
|
504
|
-
out.push(` ${k + 1}| ${origLines[k]}`);
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
// Show changed lines
|
|
508
|
-
const changeStart = i;
|
|
509
|
-
while (i < maxLen && (i >= origLines.length || i >= newLines.length || origLines[i] !== newLines[i])) {
|
|
510
|
-
if (i < origLines.length) out.push(`-${i + 1}| ${origLines[i]}`);
|
|
511
|
-
i++;
|
|
512
|
-
}
|
|
513
|
-
for (let k = changeStart; k < i && k < newLines.length; k++) {
|
|
514
|
-
out.push(`+${k + 1}| ${newLines[k]}`);
|
|
515
|
-
}
|
|
516
|
-
// Context after
|
|
517
|
-
const ctxEnd = Math.min(maxLen, i + ctx);
|
|
518
|
-
for (let k = i; k < ctxEnd && k < origLines.length; k++) {
|
|
519
|
-
out.push(` ${k + 1}| ${origLines[k]}`);
|
|
520
|
-
}
|
|
521
|
-
if (ctxEnd < maxLen) out.push("...");
|
|
522
|
-
break;
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
const diff = out.join("\n");
|
|
526
|
-
return diff
|
|
527
|
-
? `Updated benchmark-file\n\nDiff:\n\`\`\`diff\n${diff}\n\`\`\``
|
|
528
|
-
: "Updated benchmark-file";
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
// ---------------------------------------------------------------------------
|
|
532
|
-
// Runner utilities
|
|
533
|
-
// ---------------------------------------------------------------------------
|
|
534
|
-
|
|
535
|
-
function median(arr) {
|
|
536
|
-
const sorted = [...arr].sort((a, b) => a - b);
|
|
537
|
-
const mid = Math.floor(sorted.length / 2);
|
|
538
|
-
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
function runN(fn, n = RUNS) {
|
|
542
|
-
const results = [];
|
|
543
|
-
for (let i = 0; i < n; i++) results.push(fn());
|
|
544
|
-
return median(results);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function fmt(n) {
|
|
548
|
-
return n.toLocaleString("en-US", { maximumFractionDigits: 0 });
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
function pctSavings(without, withSL) {
|
|
552
|
-
if (without === 0) return "N/A";
|
|
553
|
-
const pct = ((without - withSL) / without) * 100;
|
|
554
|
-
return pct >= 0 ? `${pct.toFixed(0)}%` : `-${Math.abs(pct).toFixed(0)}%`;
|
|
555
|
-
}
|
|
46
|
+
const withGraph = args.includes("--with-graph");
|
|
556
47
|
|
|
557
48
|
// ---------------------------------------------------------------------------
|
|
558
49
|
// Main
|
|
559
50
|
// ---------------------------------------------------------------------------
|
|
560
51
|
|
|
561
|
-
function main() {
|
|
52
|
+
async function main() {
|
|
562
53
|
const allFiles = walkDir(repoRoot);
|
|
563
54
|
if (allFiles.length === 0) {
|
|
564
55
|
console.log(`No code files found in ${repoRoot}`);
|
|
@@ -573,6 +64,11 @@ function main() {
|
|
|
573
64
|
const cats = categorize(allFiles);
|
|
574
65
|
const repoName = basename(repoRoot);
|
|
575
66
|
|
|
67
|
+
// Top 3 largest code files for realistic tests
|
|
68
|
+
const sorted = allFiles.map(f => ({ f, lines: getFileLines(f)?.length || 0 }))
|
|
69
|
+
.sort((a, b) => b.lines - a.lines);
|
|
70
|
+
const largeFiles = sorted.slice(0, 3).map(s => s.f);
|
|
71
|
+
|
|
576
72
|
// Temp file setup
|
|
577
73
|
const ts = Date.now();
|
|
578
74
|
const tmpPath = resolve(tmpdir(), `hex-line-bench-${ts}.js`);
|
|
@@ -598,14 +94,17 @@ function main() {
|
|
|
598
94
|
}
|
|
599
95
|
|
|
600
96
|
if (withoutArr.length === 0) continue;
|
|
601
|
-
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
|
|
602
|
-
const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
|
|
97
|
+
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
|
|
98
|
+
const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
|
|
99
|
+
const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
|
|
100
|
+
const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
|
|
603
101
|
|
|
604
102
|
const label = { small: "<50L", medium: "50-200L", large: "200-500L", xl: "500L+" }[cat];
|
|
605
103
|
results.push({
|
|
606
104
|
num: 1, scenario: `Read full (${label})`,
|
|
607
105
|
without: avgWithout, withSL: avgWith,
|
|
608
106
|
savings: pctSavings(avgWithout, avgWith),
|
|
107
|
+
latencyWithout: avgMsWithout, latencyWith: avgMsWith,
|
|
609
108
|
});
|
|
610
109
|
}
|
|
611
110
|
|
|
@@ -626,14 +125,17 @@ function main() {
|
|
|
626
125
|
}
|
|
627
126
|
|
|
628
127
|
if (withoutArr.length === 0) continue;
|
|
629
|
-
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
|
|
630
|
-
const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
|
|
128
|
+
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
|
|
129
|
+
const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
|
|
130
|
+
const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
|
|
131
|
+
const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
|
|
631
132
|
|
|
632
133
|
const label = cat === "large" ? "200-500L" : "500L+";
|
|
633
134
|
results.push({
|
|
634
135
|
num: 2, scenario: `Outline+read (${label})`,
|
|
635
136
|
without: avgWithout, withSL: avgWith,
|
|
636
137
|
savings: pctSavings(avgWithout, avgWith),
|
|
138
|
+
latencyWithout: avgMsWithout, latencyWith: avgMsWith,
|
|
637
139
|
});
|
|
638
140
|
}
|
|
639
141
|
|
|
@@ -650,17 +152,20 @@ function main() {
|
|
|
650
152
|
const lines = getFileLines(f);
|
|
651
153
|
if (!lines) continue;
|
|
652
154
|
const pattern = "function|class|const";
|
|
653
|
-
withoutArr.push(runN(() => simBuiltInGrep(
|
|
155
|
+
withoutArr.push(runN(() => simBuiltInGrep(pattern, f).length));
|
|
654
156
|
withArr.push(runN(() => simHexLineGrep(f, lines, pattern).length));
|
|
655
157
|
}
|
|
656
158
|
|
|
657
159
|
if (withoutArr.length > 0) {
|
|
658
|
-
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b, 0) / withoutArr.length);
|
|
659
|
-
const avgWith = Math.round(withArr.reduce((a, b) => a + b, 0) / withArr.length);
|
|
160
|
+
const avgWithout = Math.round(withoutArr.reduce((a, b) => a + b.value, 0) / withoutArr.length);
|
|
161
|
+
const avgWith = Math.round(withArr.reduce((a, b) => a + b.value, 0) / withArr.length);
|
|
162
|
+
const avgMsWithout = parseFloat((withoutArr.reduce((a, b) => a + b.ms, 0) / withoutArr.length).toFixed(1));
|
|
163
|
+
const avgMsWith = parseFloat((withArr.reduce((a, b) => a + b.ms, 0) / withArr.length).toFixed(1));
|
|
660
164
|
results.push({
|
|
661
165
|
num: 3, scenario: "Grep search",
|
|
662
166
|
without: avgWithout, withSL: avgWith,
|
|
663
167
|
savings: pctSavings(avgWithout, avgWith),
|
|
168
|
+
latencyWithout: avgMsWithout, latencyWith: avgMsWith,
|
|
664
169
|
});
|
|
665
170
|
}
|
|
666
171
|
}
|
|
@@ -670,12 +175,13 @@ function main() {
|
|
|
670
175
|
// TEST 4: Directory tree
|
|
671
176
|
// ===================================================================
|
|
672
177
|
{
|
|
673
|
-
const without = runN(() => simBuiltInLsR(repoRoot, 0, 3).length);
|
|
674
|
-
const withSL = runN(() => directoryTree(repoRoot, { max_depth: 3 }).length);
|
|
178
|
+
const { value: without, ms: withoutMs } = runN(() => simBuiltInLsR(repoRoot, 0, 3).length);
|
|
179
|
+
const { value: withSL, ms: withMs } = runN(() => directoryTree(repoRoot, { max_depth: 3 }).length);
|
|
675
180
|
results.push({
|
|
676
181
|
num: 4, scenario: "Directory tree",
|
|
677
182
|
without, withSL,
|
|
678
183
|
savings: pctSavings(without, withSL),
|
|
184
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
679
185
|
});
|
|
680
186
|
}
|
|
681
187
|
|
|
@@ -684,12 +190,13 @@ function main() {
|
|
|
684
190
|
// ===================================================================
|
|
685
191
|
{
|
|
686
192
|
const infoFile = allFiles[Math.floor(allFiles.length / 2)] || allFiles[0];
|
|
687
|
-
const without = runN(() => simBuiltInStat(infoFile).length);
|
|
688
|
-
const withSL = runN(() => fileInfo(infoFile).length);
|
|
193
|
+
const { value: without, ms: withoutMs } = runN(() => simBuiltInStat(infoFile).length);
|
|
194
|
+
const { value: withSL, ms: withMs } = runN(() => fileInfo(infoFile).length);
|
|
689
195
|
results.push({
|
|
690
196
|
num: 5, scenario: "File info",
|
|
691
197
|
without, withSL,
|
|
692
198
|
savings: pctSavings(without, withSL),
|
|
199
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
693
200
|
});
|
|
694
201
|
}
|
|
695
202
|
|
|
@@ -697,12 +204,13 @@ function main() {
|
|
|
697
204
|
// TEST 6: Create file (write)
|
|
698
205
|
// ===================================================================
|
|
699
206
|
{
|
|
700
|
-
const without = runN(() => simBuiltInWrite(tmpPath, tmpContent).length);
|
|
701
|
-
const withSL = runN(() => simHexLineWrite(tmpPath, tmpContent).length);
|
|
207
|
+
const { value: without, ms: withoutMs } = runN(() => simBuiltInWrite(tmpPath, tmpContent).length);
|
|
208
|
+
const { value: withSL, ms: withMs } = runN(() => simHexLineWrite(tmpPath, tmpContent).length);
|
|
702
209
|
results.push({
|
|
703
210
|
num: 6, scenario: "Create file (200L)",
|
|
704
211
|
without, withSL,
|
|
705
212
|
savings: pctSavings(without, withSL),
|
|
213
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
706
214
|
});
|
|
707
215
|
}
|
|
708
216
|
|
|
@@ -720,6 +228,8 @@ function main() {
|
|
|
720
228
|
|
|
721
229
|
let totalWithout = 0;
|
|
722
230
|
let totalWith = 0;
|
|
231
|
+
let totalMsWithout = 0;
|
|
232
|
+
let totalMsWith = 0;
|
|
723
233
|
|
|
724
234
|
for (const edit of editTargets) {
|
|
725
235
|
const origLines = [...tmpLines];
|
|
@@ -729,14 +239,19 @@ function main() {
|
|
|
729
239
|
newLines[idx] = edit.new;
|
|
730
240
|
}
|
|
731
241
|
|
|
732
|
-
|
|
733
|
-
|
|
242
|
+
const rW = runN(() => simBuiltInEdit(tmpPath, origLines, newLines).length);
|
|
243
|
+
const rH = runN(() => simHexLineEditDiff(origLines, newLines).length);
|
|
244
|
+
totalWithout += rW.value;
|
|
245
|
+
totalWith += rH.value;
|
|
246
|
+
totalMsWithout += rW.ms;
|
|
247
|
+
totalMsWith += rH.ms;
|
|
734
248
|
}
|
|
735
249
|
|
|
736
250
|
results.push({
|
|
737
251
|
num: 7, scenario: "Edit x5 sequential",
|
|
738
252
|
without: totalWithout, withSL: totalWith,
|
|
739
253
|
savings: pctSavings(totalWithout, totalWith),
|
|
254
|
+
latencyWithout: parseFloat(totalMsWithout.toFixed(1)), latencyWith: parseFloat(totalMsWith.toFixed(1)),
|
|
740
255
|
});
|
|
741
256
|
}
|
|
742
257
|
|
|
@@ -752,13 +267,14 @@ function main() {
|
|
|
752
267
|
const cs4 = rangeChecksum(hashes.slice(150, 200), 151, 200);
|
|
753
268
|
const checksums = [cs1, cs2, cs3, cs4];
|
|
754
269
|
|
|
755
|
-
const without = runN(() => simBuiltInVerify(tmpPath, fileLines).length);
|
|
756
|
-
const withSL = runN(() => verifyChecksums(tmpPath, checksums).length);
|
|
270
|
+
const { value: without, ms: withoutMs } = runN(() => simBuiltInVerify(tmpPath, fileLines).length);
|
|
271
|
+
const { value: withSL, ms: withMs } = runN(() => verifyChecksums(tmpPath, checksums).length);
|
|
757
272
|
|
|
758
273
|
results.push({
|
|
759
274
|
num: 8, scenario: "Verify checksums (4 ranges)",
|
|
760
275
|
without, withSL,
|
|
761
276
|
savings: pctSavings(without, withSL),
|
|
277
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
762
278
|
});
|
|
763
279
|
}
|
|
764
280
|
|
|
@@ -769,7 +285,7 @@ function main() {
|
|
|
769
285
|
const batchFiles = (cats.small || []).slice(0, 3);
|
|
770
286
|
if (batchFiles.length >= 2) {
|
|
771
287
|
// Without hex-line: N separate Read calls
|
|
772
|
-
const without = runN(() => {
|
|
288
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
773
289
|
let total = 0;
|
|
774
290
|
for (const f of batchFiles) {
|
|
775
291
|
const lines = getFileLines(f);
|
|
@@ -779,7 +295,7 @@ function main() {
|
|
|
779
295
|
});
|
|
780
296
|
|
|
781
297
|
// With hex-line: 1 read_file call with paths:[] — concatenated output
|
|
782
|
-
const withSL = runN(() => {
|
|
298
|
+
const { value: withSL, ms: withMs } = runN(() => {
|
|
783
299
|
const parts = [];
|
|
784
300
|
for (const f of batchFiles) {
|
|
785
301
|
parts.push(readFile(f));
|
|
@@ -791,6 +307,7 @@ function main() {
|
|
|
791
307
|
num: 9, scenario: `Multi-file read (${batchFiles.length} files)`,
|
|
792
308
|
without, withSL,
|
|
793
309
|
savings: pctSavings(without, withSL),
|
|
310
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
794
311
|
});
|
|
795
312
|
}
|
|
796
313
|
}
|
|
@@ -810,7 +327,7 @@ function main() {
|
|
|
810
327
|
const editNew = ' this.configPath = resolve(configPath || ".");';
|
|
811
328
|
|
|
812
329
|
// Without hex-line: 5 separate edit_file calls
|
|
813
|
-
const without = runN(() => {
|
|
330
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
814
331
|
let total = 0;
|
|
815
332
|
for (const p of bulkTmpPaths) {
|
|
816
333
|
const origLines = [...tmpLines];
|
|
@@ -822,7 +339,7 @@ function main() {
|
|
|
822
339
|
});
|
|
823
340
|
|
|
824
341
|
// With hex-line: 1 bulk_replace — summary + per-file compact diff
|
|
825
|
-
const withSL = runN(() => {
|
|
342
|
+
const { value: withSL, ms: withMs } = runN(() => {
|
|
826
343
|
let response = "5 files changed, 0 errors\n";
|
|
827
344
|
for (const p of bulkTmpPaths) {
|
|
828
345
|
const origLines = [...tmpLines];
|
|
@@ -837,6 +354,7 @@ function main() {
|
|
|
837
354
|
num: 10, scenario: "bulk_replace dry_run (5 files)",
|
|
838
355
|
without, withSL,
|
|
839
356
|
savings: pctSavings(without, withSL),
|
|
357
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
840
358
|
});
|
|
841
359
|
|
|
842
360
|
for (const p of bulkTmpPaths) {
|
|
@@ -849,7 +367,7 @@ function main() {
|
|
|
849
367
|
// ===================================================================
|
|
850
368
|
{
|
|
851
369
|
// Without hex-line: raw unified diff output
|
|
852
|
-
const without = runN(() => {
|
|
370
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
853
371
|
const diffLines = [
|
|
854
372
|
`diff --git a/benchmark-target.js b/benchmark-target.js`,
|
|
855
373
|
`index abc1234..def5678 100644`,
|
|
@@ -880,22 +398,23 @@ function main() {
|
|
|
880
398
|
return diffLines.join("\n").length;
|
|
881
399
|
});
|
|
882
400
|
|
|
883
|
-
// With hex-line: semantic
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
}
|
|
401
|
+
// With hex-line: real fileChanges() semantic diff (async, called once — deterministic)
|
|
402
|
+
let withSL;
|
|
403
|
+
let withMs = 0;
|
|
404
|
+
try {
|
|
405
|
+
const t0 = performance.now();
|
|
406
|
+
const changesOut = await fileChanges(allFiles[0]);
|
|
407
|
+
withMs = parseFloat((performance.now() - t0).toFixed(1));
|
|
408
|
+
withSL = changesOut.length;
|
|
409
|
+
} catch {
|
|
410
|
+
withSL = 133; // fallback if no git history
|
|
411
|
+
}
|
|
894
412
|
|
|
895
413
|
results.push({
|
|
896
414
|
num: 11, scenario: "Changes (semantic diff)",
|
|
897
415
|
without, withSL,
|
|
898
416
|
savings: pctSavings(without, withSL),
|
|
417
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
899
418
|
});
|
|
900
419
|
}
|
|
901
420
|
|
|
@@ -907,35 +426,32 @@ function main() {
|
|
|
907
426
|
const parentDir = resolve(repoRoot, "src/utils");
|
|
908
427
|
|
|
909
428
|
// Without hex-line: 3 round-trips (error → ls → retry)
|
|
910
|
-
const without = runN(() => {
|
|
911
|
-
// Round 1: error
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
}
|
|
918
|
-
const r2 = `${parentDir}:\ntotal 10\n${dirEntries.join("\n")}`;
|
|
429
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
430
|
+
// Round 1: real ENOENT error
|
|
431
|
+
let r1;
|
|
432
|
+
try { readFileSync(missingPath, "utf-8"); r1 = ""; } catch (e) { r1 = e.message; }
|
|
433
|
+
// Round 2: real directory listing to find correct name
|
|
434
|
+
let r2;
|
|
435
|
+
try { r2 = readdirSync(parentDir).join("\n"); } catch { r2 = `${parentDir}: directory not found`; }
|
|
919
436
|
// Round 3: agent re-reads correct file (small file ~30 lines)
|
|
920
437
|
const r3 = simBuiltInReadFull(missingPath, tmpLines.slice(0, 30));
|
|
921
438
|
return (r1 + r2 + r3).length;
|
|
922
439
|
});
|
|
923
440
|
|
|
924
|
-
// With hex-line:
|
|
925
|
-
const withSL = runN(() => {
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
441
|
+
// With hex-line: real readFile() on nonexistent path — returns error + parent dir listing
|
|
442
|
+
const { value: withSL, ms: withMs } = runN(() => {
|
|
443
|
+
try {
|
|
444
|
+
return readFile(missingPath).length;
|
|
445
|
+
} catch (e) {
|
|
446
|
+
return e.message.length;
|
|
929
447
|
}
|
|
930
|
-
const response = `FILE_NOT_FOUND: ${missingPath}\n` +
|
|
931
|
-
`Parent directory (${parentDir}):\n${entries.join("\n")}`;
|
|
932
|
-
return response.length;
|
|
933
448
|
});
|
|
934
449
|
|
|
935
450
|
results.push({
|
|
936
|
-
num: 12, scenario: "FILE_NOT_FOUND recovery",
|
|
451
|
+
num: 12, scenario: "FILE_NOT_FOUND recovery*",
|
|
937
452
|
without, withSL,
|
|
938
453
|
savings: pctSavings(without, withSL),
|
|
454
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
939
455
|
});
|
|
940
456
|
}
|
|
941
457
|
|
|
@@ -944,7 +460,7 @@ function main() {
|
|
|
944
460
|
// ===================================================================
|
|
945
461
|
{
|
|
946
462
|
// Without hex-line: 3 round-trips (stale error → re-read full → retry edit)
|
|
947
|
-
const without = runN(() => {
|
|
463
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
948
464
|
// Round 1: error
|
|
949
465
|
const r1 = 'Error: file content has changed (stale). Please re-read the file.';
|
|
950
466
|
// Round 2: full re-read
|
|
@@ -958,7 +474,7 @@ function main() {
|
|
|
958
474
|
});
|
|
959
475
|
|
|
960
476
|
// With hex-line: 1 round-trip (error + fresh snippet +/-5 lines around target)
|
|
961
|
-
const withSL = runN(() => {
|
|
477
|
+
const { value: withSL, ms: withMs } = runN(() => {
|
|
962
478
|
const targetLine = 13;
|
|
963
479
|
const snippetStart = Math.max(0, targetLine - 6);
|
|
964
480
|
const snippetEnd = Math.min(tmpLines.length, targetLine + 5);
|
|
@@ -973,9 +489,10 @@ function main() {
|
|
|
973
489
|
});
|
|
974
490
|
|
|
975
491
|
results.push({
|
|
976
|
-
num: 13, scenario: "Hash mismatch recovery",
|
|
492
|
+
num: 13, scenario: "Hash mismatch recovery*",
|
|
977
493
|
without, withSL,
|
|
978
494
|
savings: pctSavings(without, withSL),
|
|
495
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
979
496
|
});
|
|
980
497
|
}
|
|
981
498
|
|
|
@@ -987,30 +504,313 @@ function main() {
|
|
|
987
504
|
const infoLines = getFileLines(infoFile);
|
|
988
505
|
if (infoLines) {
|
|
989
506
|
// Sub-test A: cat vs read_file
|
|
990
|
-
const
|
|
507
|
+
const catW = runN(() => {
|
|
991
508
|
// cat output: raw lines, no line numbers (agent redirect)
|
|
992
509
|
return infoLines.join("\n").length;
|
|
993
510
|
});
|
|
994
|
-
const
|
|
511
|
+
const catH = runN(() => readFile(infoFile).length);
|
|
995
512
|
|
|
996
513
|
// Sub-test B: ls -la vs directory_tree
|
|
997
514
|
const dirTarget = resolve(repoRoot);
|
|
998
|
-
const
|
|
999
|
-
const
|
|
515
|
+
const lsW = runN(() => simBuiltInLsR(dirTarget, 0, 1).length);
|
|
516
|
+
const lsH = runN(() => directoryTree(dirTarget, { max_depth: 1 }).length);
|
|
1000
517
|
|
|
1001
518
|
// Sub-test C: stat vs get_file_info
|
|
1002
|
-
const
|
|
1003
|
-
const
|
|
519
|
+
const stW = runN(() => simBuiltInStat(infoFile).length);
|
|
520
|
+
const stH = runN(() => fileInfo(infoFile).length);
|
|
1004
521
|
|
|
1005
522
|
// Combined: without = raw outputs (no follow-up possible)
|
|
1006
523
|
// With = structured output (enables follow-up without extra calls)
|
|
1007
|
-
const totalWithout =
|
|
1008
|
-
const totalWith =
|
|
524
|
+
const totalWithout = catW.value + lsW.value + stW.value;
|
|
525
|
+
const totalWith = catH.value + lsH.value + stH.value;
|
|
526
|
+
const totalMsWithout = catW.ms + lsW.ms + stW.ms;
|
|
527
|
+
const totalMsWith = catH.ms + lsH.ms + stH.ms;
|
|
1009
528
|
|
|
1010
529
|
results.push({
|
|
1011
530
|
num: 14, scenario: "Bash redirects (cat+ls+stat)",
|
|
1012
531
|
without: totalWithout, withSL: totalWith,
|
|
1013
532
|
savings: pctSavings(totalWithout, totalWith),
|
|
533
|
+
latencyWithout: parseFloat(totalMsWithout.toFixed(1)), latencyWith: parseFloat(totalMsWith.toFixed(1)),
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// ===================================================================
|
|
539
|
+
// TEST 15: HASH_HINT multi-match recovery
|
|
540
|
+
// ===================================================================
|
|
541
|
+
{
|
|
542
|
+
// Create a file with a duplicated line so textReplace triggers HASH_HINT
|
|
543
|
+
const dupLine = ' return this.config;';
|
|
544
|
+
const dupContent = tmpLines.map((l, i) => (i === 20 || i === 80) ? dupLine : l);
|
|
545
|
+
const dupPath = resolve(tmpdir(), `hex-line-dup-${ts}.js`);
|
|
546
|
+
writeFileSync(dupPath, dupContent.join("\n"), "utf-8");
|
|
547
|
+
|
|
548
|
+
// Without hex-line: 3 round-trips (opaque error + re-read full + retry)
|
|
549
|
+
const { value: without, ms: withoutMs } = runN(() => {
|
|
550
|
+
const r1 = 'Error: multiple occurrences found. Provide more context.';
|
|
551
|
+
const r2 = simBuiltInReadFull(dupPath, dupContent);
|
|
552
|
+
const origLines = [...dupContent];
|
|
553
|
+
const newLines = [...dupContent];
|
|
554
|
+
newLines[20] = ' return this.updatedConfig;';
|
|
555
|
+
const r3 = simBuiltInEdit(dupPath, origLines, newLines);
|
|
556
|
+
return (r1 + r2 + r3).length;
|
|
557
|
+
});
|
|
558
|
+
|
|
559
|
+
// With hex-line: HASH_HINT error contains annotated snippets (1 round-trip)
|
|
560
|
+
const { value: withSL, ms: withMs } = runN(() => {
|
|
561
|
+
try {
|
|
562
|
+
editFile(dupPath, [{ replace: { old_text: dupLine, new_text: ' return this.updatedConfig;' } }]);
|
|
563
|
+
return 0; // should not reach
|
|
564
|
+
} catch (e) {
|
|
565
|
+
// HASH_HINT error message + simulated anchor retry
|
|
566
|
+
const retry = '{"set_line":{"anchor":"xx.21","new_text":" return this.updatedConfig;"}}';
|
|
567
|
+
return (e.message + retry).length;
|
|
568
|
+
}
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
results.push({
|
|
572
|
+
num: 15, scenario: "HASH_HINT multi-match recovery*",
|
|
573
|
+
without, withSL,
|
|
574
|
+
savings: pctSavings(without, withSL),
|
|
575
|
+
latencyWithout: withoutMs, latencyWith: withMs,
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
try { unlinkSync(dupPath); } catch { /* ok */ }
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// ===================================================================
|
|
582
|
+
// TEST 16-18: Graph enrichment (--with-graph only)
|
|
583
|
+
// Both sides use hex-line; difference is whether .codegraph/index.db exists
|
|
584
|
+
// ===================================================================
|
|
585
|
+
const graphOut = [];
|
|
586
|
+
if (withGraph) {
|
|
587
|
+
const { getGraphDB, getRelativePath } = await import("./lib/graph-enrich.mjs");
|
|
588
|
+
const db = getGraphDB(resolve(repoRoot, "server.mjs"));
|
|
589
|
+
if (!db) {
|
|
590
|
+
console.error("--with-graph: .codegraph/index.db not found. Run hex-graph index_project first.");
|
|
591
|
+
} else {
|
|
592
|
+
const graphFile = largeFiles[0] || allFiles[0];
|
|
593
|
+
const graphLines = getFileLines(graphFile);
|
|
594
|
+
|
|
595
|
+
if (graphLines) {
|
|
596
|
+
// TEST 16: Read with/without Graph header
|
|
597
|
+
{
|
|
598
|
+
const withGraphResult = readFile(graphFile);
|
|
599
|
+
const noGraphResult = withGraphResult.replace(/\nGraph:.*\n/, "\n");
|
|
600
|
+
const savings = pctSavings(noGraphResult.length, withGraphResult.length);
|
|
601
|
+
graphOut.push(`| 16 | Graph: Read (${graphLines.length}L) | ${fmt(noGraphResult.length)} chars | ${fmt(withGraphResult.length)} chars | ${savings} | 2\u21921 | 2\u21921 |`);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// TEST 17: Edit with/without blast radius
|
|
605
|
+
{
|
|
606
|
+
const editTmpPath = resolve(tmpdir(), `hex-bench-edit-${Date.now()}.js`);
|
|
607
|
+
writeFileSync(editTmpPath, graphLines.join("\n"), "utf-8");
|
|
608
|
+
try {
|
|
609
|
+
const editResult = editFile(editTmpPath, [{ replace: { old_text: graphLines[5], new_text: graphLines[5] + " // modified" } }]);
|
|
610
|
+
const noBlastOut = editResult.replace(/\n.*Blast radius.*$/s, "");
|
|
611
|
+
const savings = pctSavings(noBlastOut.length, editResult.length);
|
|
612
|
+
graphOut.push(`| 17 | Graph: Edit + impact | ${fmt(noBlastOut.length)} chars | ${fmt(editResult.length)} chars | ${savings} | 2\u21921 | 2\u21921 |`);
|
|
613
|
+
} catch (e) {
|
|
614
|
+
graphOut.push(`| 17 | Graph: Edit + impact | \u2014 | \u2014 | \u2014 | | |`);
|
|
615
|
+
}
|
|
616
|
+
try { unlinkSync(editTmpPath); } catch {}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
// TEST 18: Grep with/without annotations
|
|
620
|
+
{
|
|
621
|
+
try {
|
|
622
|
+
const grepResult = await grepSearch("function", { path: resolve(repoRoot), glob: "*.mjs", limit: 10 });
|
|
623
|
+
const noAnnoResult = grepResult.replace(/ \[[^\]]+\]/g, "");
|
|
624
|
+
const savings = pctSavings(noAnnoResult.length, grepResult.length);
|
|
625
|
+
const annoCount = (grepResult.match(/\[[^\]]+\]/g) || []).length;
|
|
626
|
+
graphOut.push(`| 18 | Graph: Grep + ${annoCount} annotations | ${fmt(noAnnoResult.length)} chars | ${fmt(grepResult.length)} chars | ${savings} | 6\u21921 | 6\u21921 |`);
|
|
627
|
+
} catch {
|
|
628
|
+
graphOut.push(`| 18 | Graph: Grep + context | \u2014 | \u2014 | \u2014 | | |`);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// ===================================================================
|
|
636
|
+
// WORKFLOW SCENARIOS (multi-step real operations)
|
|
637
|
+
// ===================================================================
|
|
638
|
+
const workflowResults = [];
|
|
639
|
+
|
|
640
|
+
// W1: Search → Edit (find a pattern, edit the match)
|
|
641
|
+
{
|
|
642
|
+
const wTmpPath = resolve(tmpdir(), `hex-wf1-${Date.now()}.js`);
|
|
643
|
+
writeFileSync(wTmpPath, tmpContent, "utf-8");
|
|
644
|
+
const editLine = tmpLines[12];
|
|
645
|
+
const editNew = editLine + " // workflow-modified";
|
|
646
|
+
|
|
647
|
+
// Without: grep → read file for context → edit with old_string
|
|
648
|
+
const { value: without } = runN(() => {
|
|
649
|
+
let total = 0;
|
|
650
|
+
// Step 1: grep to find
|
|
651
|
+
total += simBuiltInGrep("configPath", wTmpPath).length;
|
|
652
|
+
// Step 2: read full file for context (agent needs surrounding lines)
|
|
653
|
+
total += simBuiltInReadFull(wTmpPath, tmpLines).length;
|
|
654
|
+
// Step 3: edit
|
|
655
|
+
const origLines = [...tmpLines];
|
|
656
|
+
const newLines = [...tmpLines];
|
|
657
|
+
newLines[12] = editNew;
|
|
658
|
+
total += simBuiltInEdit(wTmpPath, origLines, newLines).length;
|
|
659
|
+
return total;
|
|
660
|
+
});
|
|
661
|
+
|
|
662
|
+
// With: grep_search (has hashes) → edit with anchor (no re-read needed)
|
|
663
|
+
const { value: withSL } = runN(() => {
|
|
664
|
+
let total = 0;
|
|
665
|
+
// Step 1: grep with hashes
|
|
666
|
+
const grepOut = readFileSync(wTmpPath, "utf-8"); // simulate grep result
|
|
667
|
+
const lines = grepOut.split("\n");
|
|
668
|
+
const targetIdx = 12;
|
|
669
|
+
const tag = lineTag(fnv1a(lines[targetIdx]));
|
|
670
|
+
total += `${wTmpPath}:>>${tag}.${targetIdx + 1}\t${lines[targetIdx]}`.length;
|
|
671
|
+
// Step 2: edit with anchor directly (no read needed)
|
|
672
|
+
try {
|
|
673
|
+
const result = editFile(wTmpPath, [{ set_line: { anchor: `${tag}.${targetIdx + 1}`, new_text: editNew } }]);
|
|
674
|
+
total += result.length;
|
|
675
|
+
} catch (e) { total += e.message.length; }
|
|
676
|
+
return total;
|
|
677
|
+
});
|
|
678
|
+
|
|
679
|
+
workflowResults.push({
|
|
680
|
+
id: "W1", scenario: "Search \u2192 Edit",
|
|
681
|
+
without, withSL,
|
|
682
|
+
opsWithout: 3, opsWith: 2,
|
|
683
|
+
});
|
|
684
|
+
try { unlinkSync(wTmpPath); } catch {}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// W2: Read → Edit → Verify cycle
|
|
688
|
+
{
|
|
689
|
+
const wTmpPath = resolve(tmpdir(), `hex-wf2-${Date.now()}.js`);
|
|
690
|
+
writeFileSync(wTmpPath, tmpContent, "utf-8");
|
|
691
|
+
|
|
692
|
+
// Without: read full → edit → re-read full to verify
|
|
693
|
+
const { value: without } = runN(() => {
|
|
694
|
+
let total = 0;
|
|
695
|
+
total += simBuiltInReadFull(wTmpPath, tmpLines).length; // read
|
|
696
|
+
const origLines = [...tmpLines];
|
|
697
|
+
const newLines = [...tmpLines];
|
|
698
|
+
newLines[12] = ' this.configPath = resolve(configPath || ".");';
|
|
699
|
+
total += simBuiltInEdit(wTmpPath, origLines, newLines).length; // edit
|
|
700
|
+
total += simBuiltInReadFull(wTmpPath, tmpLines).length; // re-read to verify
|
|
701
|
+
return total;
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
// With: read targeted → edit → verify checksums
|
|
705
|
+
const { value: withSL } = runN(() => {
|
|
706
|
+
let total = 0;
|
|
707
|
+
total += readFile(wTmpPath, { offset: 8, limit: 20 }).length; // targeted read
|
|
708
|
+
// Reset file for edit
|
|
709
|
+
writeFileSync(wTmpPath, tmpContent, "utf-8");
|
|
710
|
+
try {
|
|
711
|
+
const result = editFile(wTmpPath, [{ replace: { old_text: tmpLines[12], new_text: ' this.configPath = resolve(configPath || ".");' } }]);
|
|
712
|
+
total += result.length;
|
|
713
|
+
} catch (e) { total += e.message.length; }
|
|
714
|
+
// Verify with checksums instead of re-reading
|
|
715
|
+
const hashes = tmpLines.slice(0, 50).map(l => fnv1a(l));
|
|
716
|
+
const cs = rangeChecksum(hashes, 1, 50);
|
|
717
|
+
try { total += verifyChecksums(wTmpPath, [cs]).length; }
|
|
718
|
+
catch { total += 100; }
|
|
719
|
+
return total;
|
|
720
|
+
});
|
|
721
|
+
|
|
722
|
+
workflowResults.push({
|
|
723
|
+
id: "W2", scenario: "Read \u2192 Edit \u2192 Verify",
|
|
724
|
+
without, withSL,
|
|
725
|
+
opsWithout: 3, opsWith: 3,
|
|
726
|
+
});
|
|
727
|
+
try { unlinkSync(wTmpPath); } catch {}
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
// W3: Multi-file refactor (rename in 5 files)
|
|
731
|
+
{
|
|
732
|
+
const wDir = resolve(tmpdir(), `hex-wf3-${Date.now()}`);
|
|
733
|
+
mkdirSync(wDir, { recursive: true });
|
|
734
|
+
const wPaths = [];
|
|
735
|
+
for (let i = 0; i < 5; i++) {
|
|
736
|
+
const p = resolve(wDir, `file-${i}.js`);
|
|
737
|
+
writeFileSync(p, tmpContent, "utf-8");
|
|
738
|
+
wPaths.push(p);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// Without: grep to find files → read each → edit each = 11 ops
|
|
742
|
+
const { value: without } = runN(() => {
|
|
743
|
+
let total = 0;
|
|
744
|
+
total += simBuiltInGrep("configPath", wPaths[0]).length; // find
|
|
745
|
+
for (const p of wPaths) {
|
|
746
|
+
total += simBuiltInReadFull(p, tmpLines).length; // read each
|
|
747
|
+
const origLines = [...tmpLines];
|
|
748
|
+
const newLines = [...tmpLines];
|
|
749
|
+
newLines[12] = newLines[12].replace("configPath", "settingsPath");
|
|
750
|
+
total += simBuiltInEdit(p, origLines, newLines).length; // edit each
|
|
751
|
+
}
|
|
752
|
+
return total;
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
// With: grep_search → bulk_replace = 2 ops
|
|
756
|
+
const { value: withSL } = runN(() => {
|
|
757
|
+
let total = 0;
|
|
758
|
+
// Restore files
|
|
759
|
+
for (const p of wPaths) writeFileSync(p, tmpContent, "utf-8");
|
|
760
|
+
// Single grep (simulated — bulk_replace does its own finding)
|
|
761
|
+
total += 200; // approximate grep output
|
|
762
|
+
// Single bulk_replace
|
|
763
|
+
const result = bulkReplace(
|
|
764
|
+
wDir,
|
|
765
|
+
"*.js",
|
|
766
|
+
[{ old: "configPath", new: "settingsPath" }],
|
|
767
|
+
{ dryRun: true, maxFiles: 10 }
|
|
768
|
+
);
|
|
769
|
+
total += result.length;
|
|
770
|
+
return total;
|
|
771
|
+
});
|
|
772
|
+
|
|
773
|
+
workflowResults.push({
|
|
774
|
+
id: "W3", scenario: "Multi-file refactor (5 files)",
|
|
775
|
+
without, withSL,
|
|
776
|
+
opsWithout: 11, opsWith: 2,
|
|
777
|
+
});
|
|
778
|
+
try { rmSync(wDir, { recursive: true }); } catch {}
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// W4: Explore large file → targeted edit
|
|
782
|
+
{
|
|
783
|
+
const largeFile = largeFiles[0] || allFiles[0];
|
|
784
|
+
const largeLines = getFileLines(largeFile);
|
|
785
|
+
if (largeLines && largeLines.length > 100) {
|
|
786
|
+
// Without: read full file → grep for method → edit
|
|
787
|
+
const { value: without } = runN(() => {
|
|
788
|
+
let total = 0;
|
|
789
|
+
total += simBuiltInReadFull(largeFile, largeLines).length;
|
|
790
|
+
total += simBuiltInGrep("function", largeFile).length;
|
|
791
|
+
// Simulate edit response
|
|
792
|
+
const origLines = [...largeLines];
|
|
793
|
+
const newLines = [...largeLines];
|
|
794
|
+
newLines[10] = newLines[10] + " // modified";
|
|
795
|
+
total += simBuiltInEdit(largeFile, origLines, newLines).length;
|
|
796
|
+
return total;
|
|
797
|
+
});
|
|
798
|
+
|
|
799
|
+
// With: outline → read range → edit with anchor
|
|
800
|
+
let outlineLen = 500;
|
|
801
|
+
try { outlineLen = (await fileOutline(largeFile)).length; } catch {}
|
|
802
|
+
const { value: withSL } = runN(() => {
|
|
803
|
+
let total = 0;
|
|
804
|
+
total += outlineLen; // outline (pre-computed, async)
|
|
805
|
+
total += readFile(largeFile, { offset: 5, limit: 30 }).length; // targeted read
|
|
806
|
+
total += simHexLineEditDiff(largeLines.slice(5, 35), [...largeLines.slice(5, 35)].map((l, i) => i === 5 ? l + " // modified" : l)).length;
|
|
807
|
+
return total;
|
|
808
|
+
});
|
|
809
|
+
|
|
810
|
+
workflowResults.push({
|
|
811
|
+
id: "W4", scenario: `Explore+edit (${largeLines.length}L file)`,
|
|
812
|
+
without, withSL,
|
|
813
|
+
opsWithout: 3, opsWith: 3,
|
|
1014
814
|
});
|
|
1015
815
|
}
|
|
1016
816
|
}
|
|
@@ -1024,7 +824,7 @@ function main() {
|
|
|
1024
824
|
// Report
|
|
1025
825
|
// ===================================================================
|
|
1026
826
|
const out = [];
|
|
1027
|
-
out.push("# Hex-line Benchmark
|
|
827
|
+
out.push("# Hex-line Benchmark v3");
|
|
1028
828
|
out.push("");
|
|
1029
829
|
out.push(`Repository: ${repoName} (${fmt(allFiles.length)} code files, ${fmt(totalLines)} lines) `);
|
|
1030
830
|
out.push(`Temp file: ${tmpPath} (200 lines) `);
|
|
@@ -1032,17 +832,97 @@ function main() {
|
|
|
1032
832
|
out.push(`Runs per scenario: ${RUNS} (median) `);
|
|
1033
833
|
out.push("");
|
|
1034
834
|
|
|
1035
|
-
//
|
|
835
|
+
// Ops comparison: how many tool calls each scenario requires
|
|
836
|
+
const OPS = {
|
|
837
|
+
"Read full (<50L)": { without: 1, with: 1 },
|
|
838
|
+
"Read full (50-200L)": { without: 1, with: 1 },
|
|
839
|
+
"Read full (200-500L)": { without: 1, with: 1 },
|
|
840
|
+
"Read full (500L+)": { without: 1, with: 1 },
|
|
841
|
+
"Outline+read (200-500L)": { without: 1, with: 2 },
|
|
842
|
+
"Outline+read (500L+)": { without: 1, with: 2 },
|
|
843
|
+
"Grep search": { without: 1, with: 1 },
|
|
844
|
+
"Directory tree": { without: 1, with: 1 },
|
|
845
|
+
"File info": { without: 1, with: 1 },
|
|
846
|
+
"Create file (200L)": { without: 1, with: 1 },
|
|
847
|
+
"Edit x5 sequential": { without: 5, with: 5 },
|
|
848
|
+
"Verify checksums (4 ranges)": { without: 4, with: 1 },
|
|
849
|
+
"Multi-file read": { without: 2, with: 1 },
|
|
850
|
+
"bulk_replace dry_run (5 files)": { without: 5, with: 1 },
|
|
851
|
+
"Changes (semantic diff)": { without: 1, with: 1 },
|
|
852
|
+
"FILE_NOT_FOUND recovery*": { without: 3, with: 1 },
|
|
853
|
+
"Hash mismatch recovery*": { without: 3, with: 1 },
|
|
854
|
+
"Bash redirects (cat+ls+stat)": { without: 3, with: 3 },
|
|
855
|
+
"HASH_HINT multi-match recovery*": { without: 3, with: 2 },
|
|
856
|
+
};
|
|
857
|
+
|
|
858
|
+
const STEPS = {
|
|
859
|
+
"Read full (<50L)": { without: 1, with: 1 },
|
|
860
|
+
"Read full (50-200L)": { without: 1, with: 1 },
|
|
861
|
+
"Read full (200-500L)": { without: 1, with: 1 },
|
|
862
|
+
"Read full (500L+)": { without: 1, with: 1 },
|
|
863
|
+
"Outline+read (200-500L)": { without: 1, with: 2 },
|
|
864
|
+
"Outline+read (500L+)": { without: 1, with: 2 },
|
|
865
|
+
"Grep search": { without: 1, with: 1 },
|
|
866
|
+
"Directory tree": { without: 1, with: 1 },
|
|
867
|
+
"File info": { without: 1, with: 1 },
|
|
868
|
+
"Create file (200L)": { without: 1, with: 1 },
|
|
869
|
+
"Edit x5 sequential": { without: 5, with: 5 },
|
|
870
|
+
"Verify checksums (4 ranges)": { without: 4, with: 1 },
|
|
871
|
+
"Multi-file read": { without: 1, with: 1 },
|
|
872
|
+
"bulk_replace dry_run (5 files)": { without: 5, with: 1 },
|
|
873
|
+
"Changes (semantic diff)": { without: 1, with: 1 },
|
|
874
|
+
"FILE_NOT_FOUND recovery": { without: 3, with: 1 },
|
|
875
|
+
"Hash mismatch recovery": { without: 3, with: 1 },
|
|
876
|
+
"Bash redirects (cat+ls+stat)": { without: 1, with: 1 },
|
|
877
|
+
"HASH_HINT multi-match recovery": { without: 3, with: 1 },
|
|
878
|
+
};
|
|
879
|
+
|
|
880
|
+
// Combined results + ops + steps table
|
|
1036
881
|
out.push("## Results");
|
|
1037
882
|
out.push("");
|
|
1038
|
-
out.push("| # | Scenario |
|
|
1039
|
-
out.push("
|
|
883
|
+
out.push("| # | Scenario | Baseline | Hex-line | Savings | Ops | Steps |");
|
|
884
|
+
out.push("|---|----------|----------|----------|---------|-----|-------|");
|
|
1040
885
|
|
|
1041
886
|
for (const r of results) {
|
|
1042
|
-
|
|
887
|
+
if (r.num >= 16) continue; // graph rows added below
|
|
888
|
+
|
|
889
|
+
// Match OPS/STEPS keys
|
|
890
|
+
let op = OPS[r.scenario];
|
|
891
|
+
if (!op) {
|
|
892
|
+
const key = Object.keys(OPS).find(k => r.scenario.startsWith(k));
|
|
893
|
+
if (key) op = OPS[key];
|
|
894
|
+
}
|
|
895
|
+
let step = STEPS[r.scenario];
|
|
896
|
+
if (!step) {
|
|
897
|
+
const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k));
|
|
898
|
+
if (key) step = STEPS[key];
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
const opsStr = op ? `${op.without}\u2192${op.with}` : "\u2014";
|
|
902
|
+
const stepsStr = step ? `${step.without}\u2192${step.with}` : "\u2014";
|
|
903
|
+
|
|
904
|
+
out.push(`| ${r.num} | ${r.scenario} | ${fmt(r.without)} chars | ${fmt(r.withSL)} chars | ${r.savings} | ${opsStr} | ${stepsStr} |`);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
// Append graph rows into same table (if any)
|
|
908
|
+
if (graphOut.length > 0) {
|
|
909
|
+
out.push("| | **hex-line \u00b1 graph** | **No Graph** | **With Graph** | | | |");
|
|
910
|
+
out.push(...graphOut);
|
|
1043
911
|
}
|
|
1044
912
|
out.push("");
|
|
1045
913
|
|
|
914
|
+
// Workflow scenarios table
|
|
915
|
+
if (workflowResults.length > 0) {
|
|
916
|
+
out.push("## Workflow Scenarios (multi-step)");
|
|
917
|
+
out.push("");
|
|
918
|
+
out.push("| # | Scenario | Built-in | Hex-line | Savings | Ops |");
|
|
919
|
+
out.push("|---|----------|----------|----------|---------|-----|");
|
|
920
|
+
for (const w of workflowResults) {
|
|
921
|
+
out.push(`| ${w.id} | ${w.scenario} | ${fmt(w.without)} chars | ${fmt(w.withSL)} chars | ${pctSavings(w.without, w.withSL)} | ${w.opsWithout}\u2192${w.opsWith} |`);
|
|
922
|
+
}
|
|
923
|
+
out.push("");
|
|
924
|
+
}
|
|
925
|
+
|
|
1046
926
|
// Verdict
|
|
1047
927
|
out.push("## Verdict");
|
|
1048
928
|
out.push("");
|
|
@@ -1059,11 +939,57 @@ function main() {
|
|
|
1059
939
|
const mismatchResult = results.find(r => r.num === 13);
|
|
1060
940
|
const bashResult = results.find(r => r.num === 14);
|
|
1061
941
|
|
|
1062
|
-
const
|
|
942
|
+
const coreResults = results.filter(r => r.num < 16);
|
|
943
|
+
const allSavingsNums = coreResults.map(r => {
|
|
1063
944
|
if (r.without === 0) return 0;
|
|
1064
945
|
return ((r.without - r.withSL) / r.without) * 100;
|
|
1065
946
|
});
|
|
1066
|
-
const avgSavings = allSavingsNums.
|
|
947
|
+
const avgSavings = allSavingsNums.length > 0
|
|
948
|
+
? allSavingsNums.reduce((a, b) => a + b, 0) / allSavingsNums.length
|
|
949
|
+
: 0;
|
|
950
|
+
|
|
951
|
+
// Weighted average based on typical development session frequency
|
|
952
|
+
const WEIGHTS = {
|
|
953
|
+
"Read full (<50L)": 2, "Read full (50-200L)": 5, "Read full (200-500L)": 3, "Read full (500L+)": 1,
|
|
954
|
+
"Outline+read (200-500L)": 8, "Outline+read (500L+)": 8,
|
|
955
|
+
"Grep search": 5, "Directory tree": 2, "File info": 1, "Create file (200L)": 1,
|
|
956
|
+
"Edit x5 sequential": 10, "Verify checksums (4 ranges)": 8,
|
|
957
|
+
"Multi-file read": 2, "bulk_replace dry_run (5 files)": 1,
|
|
958
|
+
"Changes (semantic diff)": 3,
|
|
959
|
+
"FILE_NOT_FOUND recovery": 2, "Hash mismatch recovery": 3,
|
|
960
|
+
"Bash redirects (cat+ls+stat)": 3, "HASH_HINT multi-match recovery": 2,
|
|
961
|
+
};
|
|
962
|
+
let wSum = 0, wTotal = 0;
|
|
963
|
+
for (const r of coreResults) {
|
|
964
|
+
const w = WEIGHTS[r.scenario] || 1;
|
|
965
|
+
const sav = r.without === 0 ? 0 : ((r.without - r.withSL) / r.without) * 100;
|
|
966
|
+
wSum += w * sav;
|
|
967
|
+
wTotal += w;
|
|
968
|
+
}
|
|
969
|
+
const weightedAvg = wTotal > 0 ? wSum / wTotal : 0;
|
|
970
|
+
|
|
971
|
+
// Ops/Steps totals for core scenarios
|
|
972
|
+
const totalOpsWithout = coreResults.reduce((s, r) => {
|
|
973
|
+
let op = OPS[r.scenario];
|
|
974
|
+
if (!op) { const key = Object.keys(OPS).find(k => r.scenario.startsWith(k)); if (key) op = OPS[key]; }
|
|
975
|
+
return s + (op ? op.without : 1);
|
|
976
|
+
}, 0);
|
|
977
|
+
const totalOpsWith = coreResults.reduce((s, r) => {
|
|
978
|
+
let op = OPS[r.scenario];
|
|
979
|
+
if (!op) { const key = Object.keys(OPS).find(k => r.scenario.startsWith(k)); if (key) op = OPS[key]; }
|
|
980
|
+
return s + (op ? op.with : 1);
|
|
981
|
+
}, 0);
|
|
982
|
+
const totalStepsWithout = coreResults.reduce((s, r) => {
|
|
983
|
+
let step = STEPS[r.scenario];
|
|
984
|
+
if (!step) { const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k)); if (key) step = STEPS[key]; }
|
|
985
|
+
return s + (step ? step.without : 1);
|
|
986
|
+
}, 0);
|
|
987
|
+
const totalStepsWith = coreResults.reduce((s, r) => {
|
|
988
|
+
let step = STEPS[r.scenario];
|
|
989
|
+
if (!step) { const key = Object.keys(STEPS).find(k => r.scenario.startsWith(k)); if (key) step = STEPS[key]; }
|
|
990
|
+
return s + (step ? step.with : 1);
|
|
991
|
+
}, 0);
|
|
992
|
+
const opsPct = totalOpsWithout > 0 ? ((totalOpsWithout - totalOpsWith) / totalOpsWithout * 100).toFixed(0) : 0;
|
|
1067
993
|
|
|
1068
994
|
// Read verdict
|
|
1069
995
|
const readVerdict = [];
|
|
@@ -1171,7 +1097,7 @@ function main() {
|
|
|
1171
1097
|
if (changesResult && changesResult.withSL < changesResult.without) {
|
|
1172
1098
|
out.push("- **Semantic diff** always cheaper than raw unified diff for understanding changes.");
|
|
1173
1099
|
}
|
|
1174
|
-
out.push(`- **Average
|
|
1100
|
+
out.push(`- **Average:** ${avgSavings.toFixed(0)}% tokens (flat) / ${weightedAvg.toFixed(0)}% (weighted) | ${totalOpsWithout}\u2192${totalOpsWith} ops (${opsPct}% fewer) | ${totalStepsWithout}\u2192${totalStepsWith} steps`);
|
|
1175
1101
|
out.push("");
|
|
1176
1102
|
|
|
1177
1103
|
console.log(out.join("\n"));
|