mixdog 0.9.162 → 0.9.163
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/search-rg-parity.mjs +132 -0
- package/src/runtime/agent/orchestrator/tools/builtin/binary-file.mjs +8 -8
- package/src/runtime/agent/orchestrator/tools/builtin/cache-layers.mjs +43 -54
- package/src/runtime/agent/orchestrator/tools/builtin/lib/grep-context-expander.mjs +24 -36
- package/src/runtime/agent/orchestrator/tools/builtin/lib/grep-pattern-fanout.mjs +28 -4
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +7 -25
- package/src/runtime/agent/orchestrator/tools/builtin/local-search-telemetry.mjs +12 -0
- package/src/runtime/agent/orchestrator/tools/builtin/native-search-client.mjs +3 -4
- package/src/runtime/agent/orchestrator/tools/builtin/native-search-runner.mjs +2 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-range-index.mjs +40 -23
- package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +19 -18
- package/src/runtime/agent/orchestrator/tools/builtin/read-source-windows.mjs +81 -0
- package/src/runtime/agent/orchestrator/tools/builtin/read-streaming.mjs +81 -65
- package/src/runtime/agent/orchestrator/tools/builtin/search-glob-tool.mjs +13 -5
- package/src/runtime/agent/orchestrator/tools/builtin/search-grep-tool.mjs +7 -6
- package/src/runtime/agent/orchestrator/tools/code-graph/build.mjs +1 -0
- package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +16 -1
- package/src/runtime/agent/orchestrator/tools/code-graph/graph-binary.mjs +3 -0
- package/src/runtime/agent/orchestrator/tools/code-graph/graph-model.mjs +2 -0
- package/src/runtime/agent/orchestrator/tools/graph-manifest.json +11 -11
package/package.json
CHANGED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join, resolve } from 'node:path';
|
|
6
|
+
import { createInterface } from 'node:readline';
|
|
7
|
+
import { performance } from 'node:perf_hooks';
|
|
8
|
+
|
|
9
|
+
const project = process.cwd();
|
|
10
|
+
const deadlineMs = Number(process.env.MIXDOG_PARITY_DEADLINE_MS || 15000);
|
|
11
|
+
assert.ok(Number.isSafeInteger(deadlineMs) && deadlineMs > 0);
|
|
12
|
+
const scratch = await mkdtemp(join(tmpdir(), 'mixdog-rg-parity-'));
|
|
13
|
+
const fixture = join(scratch, 'fixture');
|
|
14
|
+
await mkdir(join(fixture, '.git'), { recursive: true });
|
|
15
|
+
await mkdir(join(fixture, '.cache'), { recursive: true });
|
|
16
|
+
await writeFile(join(fixture, '.gitignore'), '*.mjs\n');
|
|
17
|
+
for (const name of ['keep.mjs', 'drop.mjs', '.cache/hit.mjs', 'other.RS']) {
|
|
18
|
+
await writeFile(join(fixture, name), 'before\nneedle\nafter\n');
|
|
19
|
+
}
|
|
20
|
+
const server = spawn(resolve('native/mixdog-graph/target/release/mixdog-graph.exe'),
|
|
21
|
+
[project, '--serve-search'], {
|
|
22
|
+
env: { ...process.env, MIXDOG_DATA_DIR: join(scratch, 'data') },
|
|
23
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
24
|
+
});
|
|
25
|
+
let id = 0;
|
|
26
|
+
const pending = new Map();
|
|
27
|
+
const lines = createInterface({ input: server.stdout });
|
|
28
|
+
lines.on('line', line => {
|
|
29
|
+
const response = JSON.parse(line);
|
|
30
|
+
pending.get(response.id)?.(response);
|
|
31
|
+
});
|
|
32
|
+
server.on('error', error => {
|
|
33
|
+
for (const settle of pending.values()) settle({ error: error.message });
|
|
34
|
+
});
|
|
35
|
+
server.on('exit', code => {
|
|
36
|
+
for (const settle of pending.values()) settle({ error: `native exit ${code}` });
|
|
37
|
+
});
|
|
38
|
+
async function native(cwd, args) {
|
|
39
|
+
const requestId = ++id;
|
|
40
|
+
const started = performance.now();
|
|
41
|
+
const response = await new Promise(resolve => {
|
|
42
|
+
pending.set(requestId, resolve);
|
|
43
|
+
server.stdin.write(`${JSON.stringify({
|
|
44
|
+
id: requestId, cwd, args, limit: 0, offset: 0, deadlineMs,
|
|
45
|
+
})}\n`);
|
|
46
|
+
});
|
|
47
|
+
pending.delete(requestId);
|
|
48
|
+
assert.ok(!response.error && !response.unsupported, JSON.stringify(response));
|
|
49
|
+
return { ...response, ms: performance.now() - started };
|
|
50
|
+
}
|
|
51
|
+
async function rg(cwd, args) {
|
|
52
|
+
const started = performance.now();
|
|
53
|
+
return await new Promise((resolve, reject) => {
|
|
54
|
+
const child = spawn('rg', args, { cwd });
|
|
55
|
+
let stdout = '', stderr = '';
|
|
56
|
+
let timeout = false;
|
|
57
|
+
child.stdout.on('data', chunk => { stdout += chunk; });
|
|
58
|
+
child.stderr.on('data', chunk => { stderr += chunk; });
|
|
59
|
+
const timer = setTimeout(() => { timeout = true; child.kill(); }, deadlineMs);
|
|
60
|
+
child.on('error', reject);
|
|
61
|
+
child.on('close', code => {
|
|
62
|
+
clearTimeout(timer);
|
|
63
|
+
resolve({ lines: stdout.trimEnd().split(/\r?\n/).filter(Boolean),
|
|
64
|
+
complete: !timeout && (code === 0 || code === 1), timeout,
|
|
65
|
+
code, stderr, ms: performance.now() - started });
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
const normalized = rows => rows.map(row => row.replaceAll('\\', '/').replace(/^\.\//, '')).sort();
|
|
70
|
+
async function compare(name, cwd, args) {
|
|
71
|
+
// Run sequentially: concurrent drive walks compete for the same storage
|
|
72
|
+
// and distort the completion time we are trying to measure.
|
|
73
|
+
const actual = await native(cwd, args);
|
|
74
|
+
let repeat;
|
|
75
|
+
if (cwd === 'C:/' && process.argv.includes('--repeat-drive')) {
|
|
76
|
+
repeat = await native(cwd, args);
|
|
77
|
+
assert.deepEqual(normalized(repeat.lines), normalized(actual.lines), `${name}: repeated results`);
|
|
78
|
+
assert.equal(repeat.scanErrors, actual.scanErrors, `${name}: repeated errors`);
|
|
79
|
+
assert.equal(repeat.complete, actual.complete, `${name}: repeated completeness`);
|
|
80
|
+
assert.equal(repeat.timeout, actual.timeout, `${name}: repeated timeout`);
|
|
81
|
+
}
|
|
82
|
+
const expected = await rg(cwd, args);
|
|
83
|
+
const traversalFinished = !actual.timeout && !expected.timeout
|
|
84
|
+
&& [0, 1, 2].includes(expected.code);
|
|
85
|
+
if (cwd !== 'C:/') {
|
|
86
|
+
assert.ok(actual.complete && expected.complete, `${name}: incomplete search`);
|
|
87
|
+
}
|
|
88
|
+
if (traversalFinished) {
|
|
89
|
+
assert.deepEqual(normalized(actual.lines), normalized(expected.lines), name);
|
|
90
|
+
} else if (cwd !== 'C:/') {
|
|
91
|
+
assert.fail(`${name}: incomplete: ${JSON.stringify({ actual, expected })}`);
|
|
92
|
+
}
|
|
93
|
+
console.log(JSON.stringify({ name, nativeMs: +actual.ms.toFixed(1),
|
|
94
|
+
rgMs: +expected.ms.toFixed(1), nativeCount: actual.lines.length,
|
|
95
|
+
rgCount: expected.lines.length, complete: actual.complete && expected.complete,
|
|
96
|
+
traversalFinished, deadlineMs,
|
|
97
|
+
repeatMs: repeat && +repeat.ms.toFixed(1),
|
|
98
|
+
filesScanned: actual.filesScanned, nativeTimeout: actual.timeout,
|
|
99
|
+
scanErrors: actual.scanErrors, rgTimeout: expected.timeout }));
|
|
100
|
+
}
|
|
101
|
+
try {
|
|
102
|
+
const rules = [
|
|
103
|
+
['--glob', '*.mjs'],
|
|
104
|
+
['--glob', '*.mjs', '--glob', '!drop.mjs'],
|
|
105
|
+
['--glob', '!drop.mjs', '--glob', '*.mjs'],
|
|
106
|
+
['--glob', '!**/.cache/**', '--glob', '*.mjs'],
|
|
107
|
+
['--glob', '*.mjs', '--glob', '!**/.cache/**'],
|
|
108
|
+
['--glob', 'keep.mjs', '--iglob', '*.rs'],
|
|
109
|
+
];
|
|
110
|
+
for (let i = 0; !process.argv.includes('--drive-only') && i < rules.length; i++) {
|
|
111
|
+
for (const mode of [['--files'], ['-l', '-e', 'needle'], ['-n', '-H', '-e', 'needle']]) {
|
|
112
|
+
await compare(`fixture-${i}-${mode[0]}`, fixture, ['--hidden', ...mode, ...rules[i], '.']);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
for (const noIgnore of process.argv.includes('--drive-only') ? [] : [false, true]) {
|
|
116
|
+
const filters = ['--hidden', ...(noIgnore ? ['--no-ignore'] : []),
|
|
117
|
+
'--glob', '*.mjs', '--glob', '!**/.git/**', '--glob', '!**/node_modules/**'];
|
|
118
|
+
await compare(`project-files-ignore-${!noIgnore}`, project, ['--files', ...filters, '.']);
|
|
119
|
+
await compare(`project-grep-ignore-${!noIgnore}`, project,
|
|
120
|
+
['-l', '-F', '-e', 'prepareGrepContextSources', ...filters, '.']);
|
|
121
|
+
}
|
|
122
|
+
if (process.argv.includes('--drive') || process.argv.includes('--drive-only')) {
|
|
123
|
+
await compare('drive-grep', 'C:/', ['-l', '-F', '-e', 'prepareGrepContextSources',
|
|
124
|
+
'--hidden', '--glob', '**/grep-context-expander.mjs', '--glob', '!**/.git/**', '.']);
|
|
125
|
+
}
|
|
126
|
+
} finally {
|
|
127
|
+
const exited = new Promise(resolve => server.once('exit', resolve));
|
|
128
|
+
server.stdin.end();
|
|
129
|
+
if (server.exitCode === null) await exited;
|
|
130
|
+
lines.close();
|
|
131
|
+
await rm(scratch, { recursive: true, force: true });
|
|
132
|
+
}
|
|
@@ -77,17 +77,17 @@ export function isBinaryBuffer(buf, fileSize = buf?.length || 0) {
|
|
|
77
77
|
* Returns the head bytes even when the null marker is in the tail so callers
|
|
78
78
|
* can render a hex preview without reopening the file.
|
|
79
79
|
*/
|
|
80
|
-
export async function inspectBinaryFile(fullPath, fileSize = 0, { previewBytes = 256 } = {}) {
|
|
80
|
+
export async function inspectBinaryFile(fullPath, fileSize = 0, { previewBytes = 256, handle = null } = {}) {
|
|
81
81
|
const headBytes = fileSize > 0 ? Math.min(fileSize, HEAD_CAP) : HEAD_CAP;
|
|
82
82
|
let fh;
|
|
83
83
|
try {
|
|
84
|
-
fh = await open(fullPath, 'r');
|
|
84
|
+
fh = handle || await open(fullPath, 'r');
|
|
85
85
|
const headBuf = Buffer.allocUnsafe(Math.max(0, headBytes));
|
|
86
86
|
const { bytesRead: nHead } = await fh.read(headBuf, 0, headBytes, 0);
|
|
87
87
|
const head = headBuf.subarray(0, nHead);
|
|
88
88
|
const preview = head.subarray(0, Math.min(previewBytes, nHead));
|
|
89
|
-
if (nHead === 0 || hasUtf16Bom(head, nHead)) return { isBinary: false, preview };
|
|
90
|
-
if (containsNull(head, nHead)) return { isBinary: true, preview };
|
|
89
|
+
if (nHead === 0 || hasUtf16Bom(head, nHead)) return { isBinary: false, preview, head };
|
|
90
|
+
if (containsNull(head, nHead)) return { isBinary: true, preview, head };
|
|
91
91
|
if (fileSize > headBytes && fileSize > TAIL_SIZE) {
|
|
92
92
|
const tailBuf = Buffer.allocUnsafe(TAIL_SIZE);
|
|
93
93
|
const { bytesRead: nTail } = await fh.read(
|
|
@@ -96,13 +96,13 @@ export async function inspectBinaryFile(fullPath, fileSize = 0, { previewBytes =
|
|
|
96
96
|
TAIL_SIZE,
|
|
97
97
|
fileSize - TAIL_SIZE,
|
|
98
98
|
);
|
|
99
|
-
if (containsNull(tailBuf, nTail)) return { isBinary: true, preview };
|
|
99
|
+
if (containsNull(tailBuf, nTail)) return { isBinary: true, preview, head };
|
|
100
100
|
}
|
|
101
|
-
return { isBinary: false, preview };
|
|
101
|
+
return { isBinary: false, preview, head };
|
|
102
102
|
} catch {
|
|
103
|
-
return { isBinary: false, preview: Buffer.alloc(0) };
|
|
103
|
+
return { isBinary: false, preview: Buffer.alloc(0), head: Buffer.alloc(0) };
|
|
104
104
|
} finally {
|
|
105
|
-
if (fh) { try { await fh.close(); } catch {} }
|
|
105
|
+
if (fh && !handle) { try { await fh.close(); } catch {} }
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
108
|
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { statSync } from 'fs';
|
|
2
2
|
import * as fsPromises from 'fs/promises';
|
|
3
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
4
|
import { isAbsolute, resolve, sep } from 'path';
|
|
4
5
|
import { canonicalCachePath, deleteReadRangeIndexForPath } from './read-range-index.mjs';
|
|
5
6
|
import { resolveAgainstCwd } from './path-utils.mjs';
|
|
6
7
|
|
|
7
8
|
const RESULT_CACHE = new Map(); // key → { ts, value, paths, scopes, readSnapshotMeta, contentPrefixHash, bytes }
|
|
8
9
|
const RESULT_CACHE_INFLIGHT = new Map(); // key → { promise, controller, subscribers, settled }
|
|
10
|
+
const RESULT_CACHE_COMPUTE = new AsyncLocalStorage();
|
|
9
11
|
const RESULT_CACHE_TTL_MS = 30_000;
|
|
10
12
|
const RESULT_CACHE_MAX_ENTRIES = 200;
|
|
11
13
|
const RESULT_CACHE_MAX_BYTES = (() => {
|
|
@@ -119,6 +121,10 @@ export function cacheGet(key) {
|
|
|
119
121
|
}
|
|
120
122
|
|
|
121
123
|
export function cacheSet(key, value, meta = {}) {
|
|
124
|
+
// Invalidation affects future reuse, not the active reader's snapshot.
|
|
125
|
+
// A detached/aborted computation must never repopulate a newer cache.
|
|
126
|
+
const computation = RESULT_CACHE_COMPUTE.getStore();
|
|
127
|
+
if (computation?.invalidated || computation?.controller?.signal?.aborted) return;
|
|
122
128
|
// Replace-in-place: clear old entry's byte accounting before write.
|
|
123
129
|
if (RESULT_CACHE.has(key)) resultCacheDelete(key);
|
|
124
130
|
const bytes = estimateResultBytes(value);
|
|
@@ -156,7 +162,11 @@ async function subscribeResultCacheInFlight(entry, signal) {
|
|
|
156
162
|
entry.subscribers.add(subscriber);
|
|
157
163
|
const release = () => {
|
|
158
164
|
if (!entry.subscribers.delete(subscriber)) return;
|
|
159
|
-
if (!entry.settled && entry.subscribers.size === 0)
|
|
165
|
+
if (!entry.settled && entry.subscribers.size === 0) {
|
|
166
|
+
// Abort listeners run in the abort caller's context, not where
|
|
167
|
+
// they were registered. Keep their cleanup/cache writes guarded.
|
|
168
|
+
RESULT_CACHE_COMPUTE.run(entry, () => entry.controller.abort());
|
|
169
|
+
}
|
|
160
170
|
};
|
|
161
171
|
if (!signal) {
|
|
162
172
|
try { return await entry.promise; }
|
|
@@ -186,58 +196,39 @@ async function subscribeResultCacheInFlight(entry, signal) {
|
|
|
186
196
|
|
|
187
197
|
export async function runResultCacheInFlight(key, compute, options = {}) {
|
|
188
198
|
const subscriberSignal = options?.signal || options?.abortSignal || null;
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
if (entry
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
promise.catch(() => {});
|
|
221
|
-
RESULT_CACHE_INFLIGHT.set(key, entry);
|
|
222
|
-
} else if (Array.isArray(options?.scopes)) {
|
|
223
|
-
entry.scopes = normalizeCacheMetaPaths([...(entry.scopes || []), ...options.scopes]);
|
|
224
|
-
}
|
|
225
|
-
try {
|
|
226
|
-
return await subscribeResultCacheInFlight(entry, subscriberSignal);
|
|
227
|
-
} catch (error) {
|
|
228
|
-
// Watcher/patch invalidation can race an active read. That is a
|
|
229
|
-
// freshness retry, not a user-visible abort: start a new generation
|
|
230
|
-
// unless this subscriber itself was cancelled.
|
|
231
|
-
if (!entry.invalidated || subscriberSignal?.aborted) throw error;
|
|
232
|
-
// Continuous invalidation (watcher churn, eviction broadcasts)
|
|
233
|
-
// must not spin forever: after a few generations run the compute
|
|
234
|
-
// directly — uncached and no longer abortable by invalidation —
|
|
235
|
-
// mirroring the server-side MAX_WALK_RESTARTS cap.
|
|
236
|
-
if (++invalidationRetries >= 3) {
|
|
237
|
-
return await compute({ signal: subscriberSignal });
|
|
238
|
-
}
|
|
239
|
-
}
|
|
199
|
+
subscriberSignal?.throwIfAborted();
|
|
200
|
+
const cached = cacheGet(key);
|
|
201
|
+
if (cached !== null) return cached;
|
|
202
|
+
let entry = RESULT_CACHE_INFLIGHT.get(key);
|
|
203
|
+
if (entry?.controller?.signal?.aborted) {
|
|
204
|
+
if (RESULT_CACHE_INFLIGHT.get(key) === entry) RESULT_CACHE_INFLIGHT.delete(key);
|
|
205
|
+
entry = null;
|
|
206
|
+
}
|
|
207
|
+
if (!entry) {
|
|
208
|
+
const controller = new AbortController();
|
|
209
|
+
entry = {
|
|
210
|
+
promise: null,
|
|
211
|
+
controller,
|
|
212
|
+
subscribers: new Set(),
|
|
213
|
+
settled: false,
|
|
214
|
+
invalidated: false,
|
|
215
|
+
scopes: normalizeCacheMetaPaths(options?.scopes),
|
|
216
|
+
};
|
|
217
|
+
const promise = Promise.resolve()
|
|
218
|
+
.then(() => RESULT_CACHE_COMPUTE.run(entry, () => compute({ signal: controller.signal })))
|
|
219
|
+
.finally(() => {
|
|
220
|
+
entry.settled = true;
|
|
221
|
+
if (RESULT_CACHE_INFLIGHT.get(key) === entry) {
|
|
222
|
+
RESULT_CACHE_INFLIGHT.delete(key);
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
entry.promise = promise;
|
|
226
|
+
promise.catch(() => {});
|
|
227
|
+
RESULT_CACHE_INFLIGHT.set(key, entry);
|
|
228
|
+
} else if (Array.isArray(options?.scopes)) {
|
|
229
|
+
entry.scopes = normalizeCacheMetaPaths([...(entry.scopes || []), ...options.scopes]);
|
|
240
230
|
}
|
|
231
|
+
return subscribeResultCacheInFlight(entry, subscriberSignal);
|
|
241
232
|
}
|
|
242
233
|
|
|
243
234
|
export async function runRawContentInFlight(fullPath, loader = fsPromises.readFile) {
|
|
@@ -503,7 +494,6 @@ function runExtraInvalidationListeners(affectedPaths = null) {
|
|
|
503
494
|
function cacheInvalidateAll() {
|
|
504
495
|
for (const entry of RESULT_CACHE_INFLIGHT.values()) {
|
|
505
496
|
entry.invalidated = true;
|
|
506
|
-
entry.controller?.abort();
|
|
507
497
|
}
|
|
508
498
|
RESULT_CACHE.clear();
|
|
509
499
|
RESULT_CACHE_INFLIGHT.clear();
|
|
@@ -527,7 +517,6 @@ function cacheInvalidatePaths(paths) {
|
|
|
527
517
|
if (scopes.length === 0
|
|
528
518
|
|| scopes.some((scope) => affectedPaths.some((affected) => cachePathsOverlap(scope, affected)))) {
|
|
529
519
|
entry.invalidated = true;
|
|
530
|
-
entry.controller?.abort();
|
|
531
520
|
RESULT_CACHE_INFLIGHT.delete(key);
|
|
532
521
|
}
|
|
533
522
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { createReadStream } from 'node:fs';
|
|
2
1
|
import { isAbsolute, resolve } from 'node:path';
|
|
3
|
-
import {
|
|
2
|
+
import { readSourceWindows } from '../read-source-windows.mjs';
|
|
4
3
|
|
|
5
4
|
import {
|
|
6
5
|
normalizeOutputPath,
|
|
@@ -266,34 +265,7 @@ async function readFileWindows(entry, radius, signal) {
|
|
|
266
265
|
start: Math.max(1, anchor.lineNo - radius),
|
|
267
266
|
end: anchor.lineNo + radius,
|
|
268
267
|
})));
|
|
269
|
-
|
|
270
|
-
if (intervals.length === 0) return lines;
|
|
271
|
-
const input = createReadStream(entry.absolutePath, { encoding: 'utf8' });
|
|
272
|
-
const reader = createInterface({ input, crlfDelay: Infinity });
|
|
273
|
-
let lineNo = 0;
|
|
274
|
-
let intervalIndex = 0;
|
|
275
|
-
const abort = () => input.destroy(Object.assign(new Error('grep context expansion aborted'), { code: 'ABORT_ERR' }));
|
|
276
|
-
if (signal) {
|
|
277
|
-
if (signal.aborted) abort();
|
|
278
|
-
else signal.addEventListener('abort', abort, { once: true });
|
|
279
|
-
}
|
|
280
|
-
try {
|
|
281
|
-
for await (const line of reader) {
|
|
282
|
-
lineNo++;
|
|
283
|
-
while (intervalIndex < intervals.length && lineNo > intervals[intervalIndex].end) intervalIndex++;
|
|
284
|
-
if (intervalIndex >= intervals.length) {
|
|
285
|
-
input.destroy();
|
|
286
|
-
break;
|
|
287
|
-
}
|
|
288
|
-
const interval = intervals[intervalIndex];
|
|
289
|
-
if (lineNo >= interval.start && lineNo <= interval.end) lines.set(lineNo, line);
|
|
290
|
-
}
|
|
291
|
-
return lines;
|
|
292
|
-
} finally {
|
|
293
|
-
if (signal) signal.removeEventListener('abort', abort);
|
|
294
|
-
reader.close();
|
|
295
|
-
input.destroy();
|
|
296
|
-
}
|
|
268
|
+
return readSourceWindows(entry.absolutePath, intervals, { signal });
|
|
297
269
|
}
|
|
298
270
|
|
|
299
271
|
async function readAnchorSources(anchors, radius, signal) {
|
|
@@ -310,16 +282,30 @@ async function readAnchorSources(anchors, radius, signal) {
|
|
|
310
282
|
groups.get(anchor.absolutePath).anchors.push(anchor);
|
|
311
283
|
}
|
|
312
284
|
const entries = [...groups.values()];
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
entry
|
|
285
|
+
let next = 0;
|
|
286
|
+
await Promise.all(Array.from({ length: Math.min(4, entries.length) }, async () => {
|
|
287
|
+
while (next < entries.length) {
|
|
288
|
+
signal?.throwIfAborted();
|
|
289
|
+
const entry = entries[next++];
|
|
290
|
+
try {
|
|
291
|
+
entry.lines = await readFileWindows(entry, radius, signal);
|
|
292
|
+
} catch (err) {
|
|
293
|
+
signal?.throwIfAborted();
|
|
294
|
+
entry.error = err;
|
|
295
|
+
}
|
|
318
296
|
}
|
|
319
297
|
}));
|
|
320
298
|
return groups;
|
|
321
299
|
}
|
|
322
300
|
|
|
301
|
+
// Share exactly the selected windows across independent pattern sections.
|
|
302
|
+
export async function prepareGrepContextSources(lineGroups, options) {
|
|
303
|
+
const selected = lineGroups.flatMap((lines) => selectAnchors(
|
|
304
|
+
parseAnchors(lines, options), options.headLimit, options.offset,
|
|
305
|
+
).selected);
|
|
306
|
+
return readAnchorSources(selected, Math.max(options.requestedContext || 0, options.maxContext || 0), options.signal);
|
|
307
|
+
}
|
|
308
|
+
|
|
323
309
|
function sourceBlock(anchor, source, radius) {
|
|
324
310
|
if (!source || source.error || !source.lines.has(anchor.lineNo)) {
|
|
325
311
|
return {
|
|
@@ -493,6 +479,7 @@ export async function expandGrepAnchorContextOutput({
|
|
|
493
479
|
caseInsensitive = false,
|
|
494
480
|
charBudget = GREP_CONTEXT_CHAR_BUDGET_DEFAULT,
|
|
495
481
|
signal,
|
|
482
|
+
sources: sharedSources,
|
|
496
483
|
}) {
|
|
497
484
|
const anchors = rankAnchors(parseAnchors(allLines, {
|
|
498
485
|
workDir,
|
|
@@ -523,7 +510,8 @@ export async function expandGrepAnchorContextOutput({
|
|
|
523
510
|
const requested = Math.max(0, Math.floor(Number(requestedContext) || 0));
|
|
524
511
|
const target = Math.max(requested, Math.max(0, Math.floor(Number(maxContext) || 0)));
|
|
525
512
|
const budget = Math.max(512, Math.floor(Number(charBudget) || GREP_CONTEXT_CHAR_BUDGET_DEFAULT));
|
|
526
|
-
|
|
513
|
+
signal?.throwIfAborted();
|
|
514
|
+
const sources = sharedSources || await readAnchorSources(window.selected, target, signal);
|
|
527
515
|
let selected = window.selected;
|
|
528
516
|
let shown = window.shown;
|
|
529
517
|
let omitted = window.omitted;
|
|
@@ -10,7 +10,7 @@ import { buildGrepRgArgs } from '../search-builders.mjs';
|
|
|
10
10
|
import { runRgWindowedLines } from '../native-search-runner.mjs';
|
|
11
11
|
import { statReachable } from '../fs-reachability.mjs';
|
|
12
12
|
import { dedupeFanoutMatchLines, formatGrepOutput } from './grep-output.mjs';
|
|
13
|
-
import { expandGrepAnchorContextOutput } from './grep-context-expander.mjs';
|
|
13
|
+
import { expandGrepAnchorContextOutput, prepareGrepContextSources } from './grep-context-expander.mjs';
|
|
14
14
|
import { markScopedCacheIncomplete } from '../../../session/cache/scoped-cache-outcome.mjs';
|
|
15
15
|
|
|
16
16
|
export async function runGrepPatternFanout({
|
|
@@ -38,6 +38,7 @@ export async function runGrepPatternFanout({
|
|
|
38
38
|
fileType,
|
|
39
39
|
executeGrepTool,
|
|
40
40
|
}) {
|
|
41
|
+
options.signal?.throwIfAborted();
|
|
41
42
|
// ONE rg --files-with-matches pass over ALL patterns can scope the fallback
|
|
42
43
|
// fan-out to candidate files. Start it lazily only after the combined pass
|
|
43
44
|
// declines: the old speculative overlap left a whole-tree bulk scan running
|
|
@@ -80,7 +81,10 @@ export async function runGrepPatternFanout({
|
|
|
80
81
|
{ offset: 0, limit: GREP_FANOUT_PREFILTER_FILE_CAP, summaryLimit: 0, bulkHint: true },
|
|
81
82
|
);
|
|
82
83
|
return pre.complete && !pre.partial ? pre.lines : null;
|
|
83
|
-
} catch {
|
|
84
|
+
} catch (err) {
|
|
85
|
+
options.signal?.throwIfAborted();
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
84
88
|
};
|
|
85
89
|
// Combined single-spawn fan-out: ONE rg run carrying every pattern
|
|
86
90
|
// (-e p1 -e p2 …), then JS-side attribution of each matched line back
|
|
@@ -95,11 +99,16 @@ export async function runGrepPatternFanout({
|
|
|
95
99
|
combined: if (process.env.MIXDOG_GREP_FANOUT_COMBINED !== '0'
|
|
96
100
|
&& !multilineMode
|
|
97
101
|
&& args['-o'] !== true
|
|
102
|
+
&& !(beforeN > 0)
|
|
103
|
+
&& !(afterN > 0)
|
|
98
104
|
&& showLineNumbers) {
|
|
99
105
|
let jsRegexps;
|
|
100
106
|
try {
|
|
101
107
|
jsRegexps = patterns.map((p) => new RegExp(p, caseInsensitive ? 'i' : ''));
|
|
102
|
-
} catch {
|
|
108
|
+
} catch {
|
|
109
|
+
options.signal?.throwIfAborted();
|
|
110
|
+
break combined;
|
|
111
|
+
}
|
|
103
112
|
let preStat;
|
|
104
113
|
try { preStat = await statReachable(grepResolvedPath); } catch { break combined; }
|
|
105
114
|
if (!preStat.isDirectory()) break combined;
|
|
@@ -137,7 +146,10 @@ export async function runGrepPatternFanout({
|
|
|
137
146
|
{ cwd: rgCwd, signal: options.signal },
|
|
138
147
|
{ offset: 0, limit: combinedCap, summaryLimit: 0, bulkHint: combinedBulkHint },
|
|
139
148
|
);
|
|
140
|
-
} catch {
|
|
149
|
+
} catch {
|
|
150
|
+
options.signal?.throwIfAborted();
|
|
151
|
+
break combined;
|
|
152
|
+
}
|
|
141
153
|
// Cap overflow (complete:false without partial) still falls back: the
|
|
142
154
|
// per-pattern rescan restores correct per-pattern windows. Timeout and
|
|
143
155
|
// scan-error partials keep their collected lines instead — the legacy
|
|
@@ -199,6 +211,12 @@ export async function runGrepPatternFanout({
|
|
|
199
211
|
const noMatchBody = (p) => `(no matches) pattern=${JSON.stringify(p)} path=${searchPath}${globStr}; path exists (dir)`;
|
|
200
212
|
const sections = [];
|
|
201
213
|
const noMatchPatterns = [];
|
|
214
|
+
const sources = adaptive ? await prepareGrepContextSources(byPattern, {
|
|
215
|
+
workDir, rgSpawnCwd: rgCwd, grepResolvedPath, searchPath, outputMode,
|
|
216
|
+
filenameOmitted: false, headLimit, offset,
|
|
217
|
+
requestedContext: contextN, maxContext: GREP_AUTO_CONTEXT_LINES,
|
|
218
|
+
signal: options.signal,
|
|
219
|
+
}) : null;
|
|
202
220
|
for (let i = 0; i < patterns.length; i++) {
|
|
203
221
|
const p = patterns[i];
|
|
204
222
|
const linesFor = byPattern[i];
|
|
@@ -224,8 +242,12 @@ export async function runGrepPatternFanout({
|
|
|
224
242
|
caseInsensitive,
|
|
225
243
|
charBudget: perBudget,
|
|
226
244
|
signal: options.signal,
|
|
245
|
+
sources,
|
|
227
246
|
});
|
|
228
247
|
body = ctx.text || noMatchBody(p);
|
|
248
|
+
if (options.scopedCacheOutcome && (ctx.omitted > 0 || !ctx.sourceComplete)) {
|
|
249
|
+
markScopedCacheIncomplete(options.scopedCacheOutcome);
|
|
250
|
+
}
|
|
229
251
|
} else {
|
|
230
252
|
const post = offset > 0 ? linesFor.slice(offset) : linesFor;
|
|
231
253
|
const windowedLines = headLimit === Infinity ? post : post.slice(0, headLimit);
|
|
@@ -267,6 +289,7 @@ export async function runGrepPatternFanout({
|
|
|
267
289
|
// completes under the cap, K patterns cost one repo walk plus K file-list
|
|
268
290
|
// scans instead of K full walks. Zero candidates short-circuits.
|
|
269
291
|
let fanoutCandidateFiles = null;
|
|
292
|
+
options.signal?.throwIfAborted();
|
|
270
293
|
const fanoutPrefilterPromise = process.env.MIXDOG_GREP_FANOUT_PREFILTER !== '0'
|
|
271
294
|
? startFanoutPrefilter()
|
|
272
295
|
: null;
|
|
@@ -295,6 +318,7 @@ export async function runGrepPatternFanout({
|
|
|
295
318
|
try {
|
|
296
319
|
return await executeGrepTool({ ...args, pattern: p }, workDir, executeChildBuiltinTool, readStateScope, subOptions);
|
|
297
320
|
} catch (err) {
|
|
321
|
+
options.signal?.throwIfAborted();
|
|
298
322
|
return `Error: ${err && err.message ? err.message : err}`;
|
|
299
323
|
}
|
|
300
324
|
};
|
|
@@ -52,12 +52,9 @@ function positiveTimeoutEnv(name, fallback) {
|
|
|
52
52
|
const value = Number(process.env[name]);
|
|
53
53
|
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
|
|
54
54
|
}
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
|
|
58
|
-
// whole-filesystem probes across six benchmark tasks, each burning a turn for
|
|
59
|
-
// an error). At this cap the native search returns its ranked partial instead.
|
|
60
|
-
const FIND_FUZZY_TIMEOUT_MS = positiveTimeoutEnv('MIXDOG_FIND_FUZZY_TIMEOUT_MS', 1_500);
|
|
55
|
+
// Prefer a complete answer within the normal search budget. Do not enforce a
|
|
56
|
+
// short response slice that makes ordinary discovery depend on another call.
|
|
57
|
+
const FIND_FUZZY_TIMEOUT_MS = positiveTimeoutEnv('MIXDOG_FIND_FUZZY_TIMEOUT_MS', 17_500);
|
|
61
58
|
// Same contract for name/glob find: bound the walk and the metadata pass, then
|
|
62
59
|
// SAY the result is partial. The old 20s walk plus a 5s stat deadline silently
|
|
63
60
|
// dropped every path whose stat missed the deadline, so a slow scope reported
|
|
@@ -596,26 +593,11 @@ export async function executeTreeTool(args, workDir, options = {}) {
|
|
|
596
593
|
// Partial or timed-out passes say "no fuzzy match YET" and already tell the
|
|
597
594
|
// caller to narrow, so they never pay for the second walk.
|
|
598
595
|
export async function executeFuzzyFindTool(args, workDir, options = {}) {
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
if (!/^\(no fuzzy match for /.test(text)) return result;
|
|
603
|
-
if (args?.include_noise === true || options?._findNoiseWidened === true) return result;
|
|
604
|
-
if (performance.now() - startedAt > FIND_NOISE_WIDEN_BUDGET_MS) return result;
|
|
605
|
-
const widened = String(await runFuzzyFindPass(
|
|
606
|
-
{ ...args, include_noise: true },
|
|
607
|
-
workDir,
|
|
608
|
-
{ ...options, _findNoiseWidened: true },
|
|
609
|
-
) ?? '');
|
|
610
|
-
if (!widened || /^\(no fuzzy match|^Error/.test(widened)) return result;
|
|
611
|
-
const hits = widened.split('\n').filter((line) => line && !line.startsWith('... [') && !line.startsWith('[')).length;
|
|
612
|
-
return `${text}\n[notice] ${hits} match${hits === 1 ? '' : 'es'} exist inside dependency/cache trees the default scan skips`
|
|
613
|
-
+ ' — pass include_noise:true to list them.';
|
|
596
|
+
// A complete miss answers the requested scope. Do not start a second,
|
|
597
|
+
// broader walk merely to offer an optional dependency-tree hint.
|
|
598
|
+
return runFuzzyFindPass(args, workDir, options);
|
|
614
599
|
}
|
|
615
600
|
|
|
616
|
-
/** A miss cheaper than this earns the one widening retry above. */
|
|
617
|
-
const FIND_NOISE_WIDEN_BUDGET_MS = 2_000;
|
|
618
|
-
|
|
619
601
|
async function runFuzzyFindPass(args, workDir, options = {}) {
|
|
620
602
|
const query = String(args.query ?? '').trim();
|
|
621
603
|
if (!query) return 'Error: find requires query.';
|
|
@@ -757,7 +739,7 @@ async function runFuzzyFindPass(args, workDir, options = {}) {
|
|
|
757
739
|
: '';
|
|
758
740
|
return capFindResult([
|
|
759
741
|
`(no fuzzy match yet for "${query}")`,
|
|
760
|
-
`... [native inventory was incomplete${scanErrorNote};
|
|
742
|
+
`... [native inventory was incomplete${scanErrorNote}; narrow path/query for a complete result]`,
|
|
761
743
|
].join('\n'));
|
|
762
744
|
}
|
|
763
745
|
return capFindResult('Error: native fuzzy search did not return a result.');
|
|
@@ -31,6 +31,18 @@ export function recordLocalSearchBackend(backend, durationMs, outcome) {
|
|
|
31
31
|
addNumber(target, `${name}_ms`, durationMs);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
export function recordNativeSearchFailure(error) {
|
|
35
|
+
const target = current();
|
|
36
|
+
if (!target) return;
|
|
37
|
+
if (!Array.isArray(target.native_failures)) target.native_failures = [];
|
|
38
|
+
if (target.native_failures.length < 3) {
|
|
39
|
+
target.native_failures.push({
|
|
40
|
+
code: String(error?.code || ''),
|
|
41
|
+
message: String(error?.message || error).slice(0, 300),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
34
46
|
export function recordNativeSearchTiming(served) {
|
|
35
47
|
const target = current();
|
|
36
48
|
if (!target || !served || typeof served !== 'object') return;
|
|
@@ -34,7 +34,7 @@ const SEARCH_TIMEOUT_RECYCLE_WINDOW_MS = 30_000;
|
|
|
34
34
|
const SEARCH_TIMEOUT_BURST_MS = 1_000;
|
|
35
35
|
const FUZZY_INVENTORY_LEASE_MS = (() => {
|
|
36
36
|
const configured = Number(process.env.MIXDOG_FIND_INVENTORY_LEASE_MS);
|
|
37
|
-
if (!Number.isFinite(configured) || configured < 0) return
|
|
37
|
+
if (!Number.isFinite(configured) || configured < 0) return 0;
|
|
38
38
|
return Math.min(30_000, Math.floor(configured));
|
|
39
39
|
})();
|
|
40
40
|
|
|
@@ -709,9 +709,8 @@ export async function tryServeFuzzySearch(args, execOptions = {}) {
|
|
|
709
709
|
limit: Math.max(1, Math.min(1_000, Math.floor(Number(args?.limit) || 25))),
|
|
710
710
|
hidden: args?.hidden !== false,
|
|
711
711
|
includeNoise: args?.includeNoise === true,
|
|
712
|
-
//
|
|
713
|
-
//
|
|
714
|
-
// the native worker stop an idle broad walk instead of crawling forever.
|
|
712
|
+
// Background inventory continuation is explicit opt-in, not a
|
|
713
|
+
// prerequisite for a normal find response.
|
|
715
714
|
...(FUZZY_INVENTORY_LEASE_MS > 0
|
|
716
715
|
? { keepInventoryMs: FUZZY_INVENTORY_LEASE_MS }
|
|
717
716
|
: {}),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { performance } from 'node:perf_hooks';
|
|
2
2
|
import { tryServeSearch } from './native-search-client.mjs';
|
|
3
|
-
import { recordLocalSearchBackend, recordNativeSearchTiming } from './local-search-telemetry.mjs';
|
|
3
|
+
import { recordLocalSearchBackend, recordNativeSearchTiming, recordNativeSearchFailure } from './local-search-telemetry.mjs';
|
|
4
4
|
|
|
5
5
|
function unavailable(argsList) {
|
|
6
6
|
const error = new Error(`native search unavailable or unsupported for args: ${JSON.stringify(argsList)}`);
|
|
@@ -17,6 +17,7 @@ async function serve(argsList, execOptions, opts) {
|
|
|
17
17
|
recordLocalSearchBackend('native', performance.now() - startedAt, 'hit');
|
|
18
18
|
return result;
|
|
19
19
|
} catch (error) {
|
|
20
|
+
recordNativeSearchFailure(error);
|
|
20
21
|
// Unsupported request shapes are classified separately in telemetry,
|
|
21
22
|
// but still throw: there is no alternate local-search backend.
|
|
22
23
|
const code = String(error?.code || '');
|