mixdog 0.9.116 → 0.9.118
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/tool-stress.mjs +145 -0
- package/src/cli.mjs +1 -0
- package/src/rules/shared/01-tool.md +24 -20
- package/src/runtime/agent/orchestrator/session/loop/pre-dispatch-deny.mjs +0 -2
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +3 -3
- package/src/runtime/agent/orchestrator/tools/builtin/cache-layers.mjs +6 -10
- package/src/runtime/agent/orchestrator/tools/builtin/fs-reachability.mjs +24 -1
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +131 -26
- package/src/runtime/agent/orchestrator/tools/builtin/native-search-client.mjs +90 -5
- package/src/runtime/agent/orchestrator/tools/builtin/rg-runner.mjs +8 -9
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +33 -71
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-spawn.mjs +13 -0
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +4 -1
- package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +50 -1
- package/src/runtime/agent/orchestrator/tools/code-graph/graph-binary.mjs +7 -0
- package/src/runtime/agent/orchestrator/tools/graph-manifest.json +11 -11
- package/src/runtime/agent/orchestrator/tools/lib/pwsh-standby-pool.mjs +32 -4
- package/src/runtime/agent/orchestrator/tools/patch-tool-defs.mjs +1 -0
- package/src/runtime/agent/orchestrator/tools/progress-message.mjs +0 -6
- package/src/runtime/agent/orchestrator/tools/shell-command.mjs +19 -9
- package/src/runtime/channels/lib/tool-dispatch.mjs +0 -26
- package/src/runtime/channels/lib/worker-main.mjs +0 -18
- package/src/runtime/channels/tool-defs.mjs +3 -45
- package/src/runtime/shared/child-spawn-gate.mjs +16 -5
- package/src/runtime/shared/tool-card-model.mjs +25 -0
- package/src/runtime/shared/tool-result-summary.mjs +0 -1
- package/src/runtime/shared/tool-surface.mjs +0 -7
- package/src/runtime/shared/uv-threadpool-boot.mjs +13 -0
- package/src/session-runtime/runtime-core.mjs +1 -0
- package/src/session-runtime/tool-catalog-data.mjs +1 -3
- package/src/session-runtime/warmup-schedulers.mjs +8 -0
- package/src/session-runtime/workflow.mjs +13 -1
- package/src/standalone/channel-worker.mjs +0 -2
- package/src/standalone/daemon.mjs +3 -0
- package/src/tui/components/ToolExecution.jsx +9 -3
- package/src/tui/components/TranscriptItem.jsx +1 -1
- package/src/tui/components/tool-execution/surface-detail.mjs +8 -25
- package/src/tui/dist/index.mjs +30 -22
- package/src/tui/session/session-api.mjs +74 -10
- package/src/runtime/channels/lib/provider-dispatch.mjs +0 -61
package/package.json
CHANGED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// Tool-layer burst stress: hammers the in-process tool execution surface with
|
|
2
|
+
// concurrent multi-session waves and checks that results stay correct and
|
|
3
|
+
// bounded under load (no unhandled rejections, no resource-pressure failures,
|
|
4
|
+
// patch/read integrity preserved, cancellation clean). One-shot script:
|
|
5
|
+
// `node scripts/tool-stress.mjs`. Exit 0 = stable, 1 = instability found.
|
|
6
|
+
import '../src/runtime/shared/uv-threadpool-boot.mjs';
|
|
7
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
8
|
+
import { tmpdir } from 'node:os';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
import { executeBuiltinTool } from '../src/runtime/agent/orchestrator/tools/builtin.mjs';
|
|
12
|
+
import { executeCodeGraphTool } from '../src/runtime/agent/orchestrator/tools/code-graph.mjs';
|
|
13
|
+
import { executePatchTool } from '../src/runtime/agent/orchestrator/tools/patch.mjs';
|
|
14
|
+
import { normalizeToolEnvelope } from '../src/runtime/agent/orchestrator/session/tool-envelope.mjs';
|
|
15
|
+
|
|
16
|
+
const root = join(fileURLToPath(new URL('.', import.meta.url)), '..');
|
|
17
|
+
const SESSIONS = 8;
|
|
18
|
+
const WAVES = 5;
|
|
19
|
+
const stats = new Map(); // tool -> {n, errs:[], lat:[]}
|
|
20
|
+
const failures = [];
|
|
21
|
+
|
|
22
|
+
function record(tool, ms, result, expectRe) {
|
|
23
|
+
let s = stats.get(tool);
|
|
24
|
+
if (!s) { s = { n: 0, errs: [], lat: [] }; stats.set(tool, s); }
|
|
25
|
+
s.n += 1; s.lat.push(ms);
|
|
26
|
+
const text = String(result ?? '');
|
|
27
|
+
if (/^Error:|resource pressure|ERESOURCEPRESSURE|EAGAIN/i.test(text)) s.errs.push(text.slice(0, 160));
|
|
28
|
+
else if (expectRe && !expectRe.test(text)) s.errs.push(`unexpected output: ${text.slice(0, 120)}`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function timed(tool, expectRe, fn) {
|
|
32
|
+
const t0 = Date.now();
|
|
33
|
+
try {
|
|
34
|
+
const raw = await fn();
|
|
35
|
+
const out = normalizeToolEnvelope(raw).result;
|
|
36
|
+
record(tool, Date.now() - t0, out, expectRe);
|
|
37
|
+
return out;
|
|
38
|
+
} catch (err) {
|
|
39
|
+
record(tool, Date.now() - t0, `Error: thrown ${err?.message || err}`);
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function pct(list, p) {
|
|
45
|
+
if (!list.length) return 0;
|
|
46
|
+
const sorted = [...list].sort((a, b) => a - b);
|
|
47
|
+
return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const tmp = mkdtempSync(join(tmpdir(), 'mixdog-tool-stress-'));
|
|
51
|
+
const t0 = Date.now();
|
|
52
|
+
try {
|
|
53
|
+
// ── Phase A+C: concurrent multi-session waves (search/read/graph/shell +
|
|
54
|
+
// per-session patch integrity riding the same load) ──────────────────────
|
|
55
|
+
for (let wave = 0; wave < WAVES; wave++) {
|
|
56
|
+
const calls = [];
|
|
57
|
+
for (let s = 0; s < SESSIONS; s++) {
|
|
58
|
+
const opts = { sessionId: `stress-s${s}` };
|
|
59
|
+
const marker = `stress_w${wave}_s${s}`;
|
|
60
|
+
calls.push(
|
|
61
|
+
timed('grep', /path-string|paths only|grep|\(no matches\)|Fuzzy/i, () => executeBuiltinTool('grep', {
|
|
62
|
+
pattern: ['Fuzzy filename', 'paths only'], path: 'src/runtime/agent/orchestrator/tools/builtin', glob: '*.mjs', limit: 20, context: 0,
|
|
63
|
+
}, root, opts)),
|
|
64
|
+
timed('glob', /tool-defs\.mjs|\.mjs/, () => executeBuiltinTool('glob', {
|
|
65
|
+
pattern: '**/*.mjs', path: 'src/session-runtime', limit: 40,
|
|
66
|
+
}, root, opts)),
|
|
67
|
+
timed('find', /tool-defs|no fuzzy match/, () => executeBuiltinTool('find', {
|
|
68
|
+
query: 'tool-defs', limit: 8,
|
|
69
|
+
}, root, opts)),
|
|
70
|
+
timed('list', /01-tool\.md|file/, () => executeBuiltinTool('list', {
|
|
71
|
+
path: 'src/rules/shared',
|
|
72
|
+
}, root, opts)),
|
|
73
|
+
timed('read', /Tool Use|read/, () => executeBuiltinTool('read', {
|
|
74
|
+
path: [['src/rules/shared/01-tool.md', 0, 10], ['package.json', 0, 5]],
|
|
75
|
+
}, root, opts)),
|
|
76
|
+
timed('code_graph', /symbol|binding|files|edges/i, () => executeCodeGraphTool('code_graph', {
|
|
77
|
+
mode: 'symbols', files: 'scripts/smoke.mjs',
|
|
78
|
+
}, root)),
|
|
79
|
+
timed('shell', /55350/, () => executeBuiltinTool('shell', {
|
|
80
|
+
command: 'node -e "console.log(123*450)"', timeout: 60_000,
|
|
81
|
+
}, root, opts)),
|
|
82
|
+
(async () => {
|
|
83
|
+
const patch = [
|
|
84
|
+
'*** Begin Patch',
|
|
85
|
+
`*** Add File: ${marker}.txt`,
|
|
86
|
+
`+payload ${marker}`,
|
|
87
|
+
'*** End Patch',
|
|
88
|
+
].join('\n');
|
|
89
|
+
await timed('apply_patch', /applied|OK/i, () => executePatchTool('apply_patch', { patch, base_path: tmp }, tmp, opts));
|
|
90
|
+
const back = await timed('read-verify', new RegExp(`payload ${marker}`), () => executeBuiltinTool('read', { path: `${marker}.txt` }, tmp, opts));
|
|
91
|
+
if (!String(back || '').includes(`payload ${marker}`)) failures.push(`patch integrity lost for ${marker}`);
|
|
92
|
+
})(),
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
await Promise.all(calls);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ── Phase B: oversized inputs stay budget-bounded ─────────────────────────
|
|
99
|
+
await Promise.all([
|
|
100
|
+
timed('grep-broad', /Showing|import|export/, () => executeBuiltinTool('grep', {
|
|
101
|
+
pattern: 'import', path: 'src', limit: 300, mode: 'files',
|
|
102
|
+
}, root, { sessionId: 'stress-big' })),
|
|
103
|
+
timed('glob-broad', /\.mjs|entries/, () => executeBuiltinTool('glob', {
|
|
104
|
+
pattern: '**/*', path: 'src/runtime/agent/orchestrator/tools', limit: 0,
|
|
105
|
+
}, root, { sessionId: 'stress-big' })),
|
|
106
|
+
timed('read-big', /./, () => executeBuiltinTool('read', {
|
|
107
|
+
path: 'src/tui/dist/index.mjs', limit: 2000,
|
|
108
|
+
}, root, { sessionId: 'stress-big' })),
|
|
109
|
+
]);
|
|
110
|
+
for (const [tool, s] of stats) {
|
|
111
|
+
if (tool.endsWith('-broad') || tool === 'read-big') {
|
|
112
|
+
const worst = Math.max(...s.lat);
|
|
113
|
+
if (worst > 30_000) failures.push(`${tool} exceeded 30s budget: ${worst}ms`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ── Phase D: cancellation under load ─────────────────────────────────────
|
|
118
|
+
const bg = await timed('shell-async', /task_id/, () => executeBuiltinTool('shell', {
|
|
119
|
+
command: 'node -e "setTimeout(()=>{}, 30000)"', mode: 'async', timeout: 60_000,
|
|
120
|
+
}, root, { sessionId: 'stress-cancel' }));
|
|
121
|
+
const bgId = (/task_id:\s*(\S+)/.exec(String(bg)) || [])[1];
|
|
122
|
+
if (!bgId) failures.push('async shell did not return task_id');
|
|
123
|
+
else {
|
|
124
|
+
await timed('task-cancel', /cancelled/, () => executeBuiltinTool('task', { action: 'cancel', task_id: bgId }, root, { sessionId: 'stress-cancel' }));
|
|
125
|
+
const st = await timed('task-status', /cancelled|failed/, () => executeBuiltinTool('task', { action: 'status', task_id: bgId }, root, { sessionId: 'stress-cancel' }));
|
|
126
|
+
if (!/cancelled/.test(String(st))) failures.push(`cancelled task not reported cancelled: ${String(st).slice(0, 120)}`);
|
|
127
|
+
}
|
|
128
|
+
} finally {
|
|
129
|
+
try { rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// ── Report ──────────────────────────────────────────────────────────────────
|
|
133
|
+
let errTotal = 0;
|
|
134
|
+
for (const [tool, s] of [...stats.entries()].sort()) {
|
|
135
|
+
errTotal += s.errs.length;
|
|
136
|
+
console.log(
|
|
137
|
+
`${tool.padEnd(14)} n=${String(s.n).padStart(3)} errs=${s.errs.length}`
|
|
138
|
+
+ ` p50=${pct(s.lat, 50)}ms p95=${pct(s.lat, 95)}ms max=${Math.max(...s.lat)}ms`,
|
|
139
|
+
);
|
|
140
|
+
for (const e of s.errs.slice(0, 3)) console.log(` ! ${e}`);
|
|
141
|
+
}
|
|
142
|
+
for (const f of failures) console.log(`FAIL ${f}`);
|
|
143
|
+
const calls = [...stats.values()].reduce((a, s) => a + s.n, 0);
|
|
144
|
+
console.log(`tool stress ${failures.length || errTotal ? 'FAILED' : 'passed'} calls=${calls} errors=${errTotal} failures=${failures.length} elapsed=${Math.round((Date.now() - t0) / 1000)}s`);
|
|
145
|
+
process.exit(failures.length || errTotal ? 1 : 0);
|
package/src/cli.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Tool Use
|
|
2
2
|
|
|
3
3
|
- Baseline routing assigns each facet directly by the evidence needed to
|
|
4
|
-
determine the complete edit:
|
|
4
|
+
determine the complete answer or edit:
|
|
5
5
|
path/name only→`find`; wildcard/recursive paths→`glob` (including known-root
|
|
6
6
|
unknown descendants); exact directory entries→`list`;
|
|
7
7
|
source content/value/`path:line`→`grep`; exact symbol/relation→`code_graph`;
|
|
@@ -18,41 +18,45 @@
|
|
|
18
18
|
relies on it. Within the current project, pass
|
|
19
19
|
project-relative paths and omit optional scopes equal to its root; explicit
|
|
20
20
|
paths may be outside cwd only for targets outside the project.
|
|
21
|
-
- Plan the fewest dependent rounds, then the fewest calls.
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
re-
|
|
21
|
+
- Plan the fewest dependent rounds, then the fewest calls. Known state —
|
|
22
|
+
anything the task supplied, a tool returned, or a check already proved —
|
|
23
|
+
is never re-found, re-derived, or re-verified; a change to its subject
|
|
24
|
+
re-opens it. Batch calls iff none needs
|
|
25
25
|
another's output or can change another's inputs/state; otherwise
|
|
26
26
|
serialize. Before each batch, deduplicate the facets still required by the request,
|
|
27
27
|
route each once to the cheapest sufficient tool with all required
|
|
28
28
|
variants/scopes — every distinct sample/format in the same batch —
|
|
29
29
|
and launch every independent call together — never
|
|
30
30
|
split or duplicate a facet across tools, mutate merely to widen
|
|
31
|
-
retrieval, reserve known work, or cap fanout.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
31
|
+
retrieval, reserve known work, or cap fanout. Each round asks everything
|
|
32
|
+
it can and keeps everything it gets: guesses go wide in one batch, scopes
|
|
33
|
+
narrow only on verified cues — returned siblings/conventions or known
|
|
34
|
+
literals — and returned output is fully mined before the next round.
|
|
35
|
+
Symbol relations end at
|
|
35
36
|
`code_graph`; values/locations end at the context grep returns; `read`
|
|
36
37
|
covers only what returned spans cannot, as an anchored offset/limit
|
|
37
|
-
window.
|
|
38
|
-
|
|
38
|
+
window. A conclusive result ends its facet; evidence that determines the
|
|
39
|
+
answer, edit, or deliverable ends retrieval — patch if needed.
|
|
39
40
|
- Once the edit or deliverable is determined, finish in one assistant turn:
|
|
40
41
|
before `apply_patch`, obtain every target hunk's exact current content and
|
|
41
42
|
anchor from `grep`, `code_graph`, or `read`; never infer patch context from
|
|
42
43
|
another file, a sample, or expected text. Then issue `apply_patch` calls
|
|
43
44
|
serially, never in parallel; use one cohesive call with one file section per
|
|
44
|
-
target, all patches first,
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
after a
|
|
45
|
+
target, all patches first, then their one batched verification `shell` in
|
|
46
|
+
the same turn — the runtime runs it after every patch and only if all
|
|
47
|
+
succeeded, so verification never needs its own turn. It runs the real
|
|
48
|
+
required postconditions on every changed file and produced artifact,
|
|
49
|
+
never echoing a claim; a postcondition that did not actually run is
|
|
50
|
+
unresolved, not passed. Retry only failed envelopes; rerun a failed check only
|
|
51
|
+
after a change that can alter its outcome — commands alike; else switch
|
|
52
|
+
route or report it unresolved.
|
|
51
53
|
Hand-authored text is edited only with `apply_patch`; computed artifacts
|
|
52
54
|
(data/reports/derived values) come from `shell` computation, never
|
|
53
55
|
hand-transcribed numbers. Earlier `shell` is only for runtime/state
|
|
54
56
|
evidence unavailable to file tools—an independent facet, batched with
|
|
55
57
|
the rest; independent probes are parallel shell calls, never one
|
|
56
58
|
serial script per round.
|
|
57
|
-
- A
|
|
58
|
-
|
|
59
|
+
- A long or uncertain command runs async — never nohup — and its
|
|
60
|
+
`task_id` ends the turn; completion resumes work. Never poll in any
|
|
61
|
+
form — sleep/status probes included; task control is for recovery or a
|
|
62
|
+
required blocking result only.
|
|
@@ -19,8 +19,6 @@ const WORKER_DENIED_TOOLS = new Set([
|
|
|
19
19
|
// (type=spawn|send|close|list). Denying the one name blocks all worker
|
|
20
20
|
// session control.
|
|
21
21
|
'agent',
|
|
22
|
-
// channels module (owner/Discord-facing)
|
|
23
|
-
'reply', 'fetch',
|
|
24
22
|
// host input injection
|
|
25
23
|
'inject_input',
|
|
26
24
|
]);
|
|
@@ -32,7 +32,7 @@ export const BUILTIN_TOOLS = [
|
|
|
32
32
|
name: 'read',
|
|
33
33
|
title: 'Mixdog Read',
|
|
34
34
|
annotations: { title: 'Mixdog Read', readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false, compressible: false },
|
|
35
|
-
description: 'Known-file contents or line ranges; not directories. Replaces cat/head/tail.',
|
|
35
|
+
description: 'Known-file contents or line ranges; images render for viewing; not directories. Replaces cat/head/tail.',
|
|
36
36
|
inputSchema: {
|
|
37
37
|
type: 'object',
|
|
38
38
|
properties: {
|
|
@@ -75,7 +75,7 @@ export const BUILTIN_TOOLS = [
|
|
|
75
75
|
name: 'shell',
|
|
76
76
|
title: 'Mixdog Shell',
|
|
77
77
|
annotations: { title: 'Mixdog Shell', readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true, compressible: true },
|
|
78
|
-
description: 'Run a shell command
|
|
78
|
+
description: 'Run a shell command; async returns task_id and sends a completion notification. Executable/runtime/state evidence only — never file exploration in any command segment: NOT ls/find/cat/head/tail/grep/rg/sed; dedicated file tools cover those.',
|
|
79
79
|
inputSchema: {
|
|
80
80
|
type: 'object',
|
|
81
81
|
properties: {
|
|
@@ -194,7 +194,7 @@ export const BUILTIN_TOOLS = [
|
|
|
194
194
|
description: 'Filename or directory path fragments matched against path strings; query[] batches.',
|
|
195
195
|
},
|
|
196
196
|
path: { type: 'string', description: 'Project-relative base; omit for project root; absolute only outside.' },
|
|
197
|
-
limit: { type: 'number', description: 'Max paths
|
|
197
|
+
limit: { type: 'number', description: 'Max paths; default 25; 0 unlimited.' },
|
|
198
198
|
include_noise: { type: 'boolean', description: 'Also search gitignored/dependency trees.' },
|
|
199
199
|
},
|
|
200
200
|
required: ['query'],
|
|
@@ -357,19 +357,17 @@ export async function lstatPathsForMtime(paths, workDir, concurrency = Infinity,
|
|
|
357
357
|
return out;
|
|
358
358
|
}
|
|
359
359
|
|
|
360
|
-
// Extra invalidation listeners: sibling modules with
|
|
361
|
-
//
|
|
362
|
-
//
|
|
363
|
-
// raw caches also drops theirs. Full clear is intentional — those entries are
|
|
364
|
-
// cheap to rebuild and a path-scoped diff is not worth the coupling.
|
|
360
|
+
// Extra invalidation listeners: sibling modules with derived caches receive
|
|
361
|
+
// the normalized affected paths, or null for a full clear, so unrelated
|
|
362
|
+
// project inventories survive writes in another temporary/project root.
|
|
365
363
|
const EXTRA_INVALIDATION_LISTENERS = new Set();
|
|
366
364
|
export function registerCacheInvalidationListener(fn) {
|
|
367
365
|
if (typeof fn === 'function') EXTRA_INVALIDATION_LISTENERS.add(fn);
|
|
368
366
|
return () => EXTRA_INVALIDATION_LISTENERS.delete(fn);
|
|
369
367
|
}
|
|
370
|
-
function runExtraInvalidationListeners() {
|
|
368
|
+
function runExtraInvalidationListeners(affectedPaths = null) {
|
|
371
369
|
for (const fn of EXTRA_INVALIDATION_LISTENERS) {
|
|
372
|
-
try { fn(); } catch { /* best-effort: one listener must not block others */ }
|
|
370
|
+
try { fn(affectedPaths); } catch { /* best-effort: one listener must not block others */ }
|
|
373
371
|
}
|
|
374
372
|
}
|
|
375
373
|
|
|
@@ -411,9 +409,7 @@ function cacheInvalidatePaths(paths) {
|
|
|
411
409
|
deleteReadRangeIndexForPath(affected);
|
|
412
410
|
bumpPathMutationGeneration(affected);
|
|
413
411
|
}
|
|
414
|
-
|
|
415
|
-
// invalidation still fully drops them (cheap to rebuild).
|
|
416
|
-
runExtraInvalidationListeners();
|
|
412
|
+
runExtraInvalidationListeners(affectedPaths);
|
|
417
413
|
}
|
|
418
414
|
|
|
419
415
|
export function invalidateBuiltinResultCache(paths = null) {
|
|
@@ -13,6 +13,12 @@
|
|
|
13
13
|
import { stat } from 'node:fs/promises';
|
|
14
14
|
|
|
15
15
|
const FS_REACHABILITY_DEADLINE_MS = 5000;
|
|
16
|
+
// Under burst load the libuv threadpool can queue a healthy local stat past
|
|
17
|
+
// the base deadline (slow ≠ dead). Before declaring the path unreachable,
|
|
18
|
+
// keep waiting for the SAME pending probe up to this extended ceiling: a
|
|
19
|
+
// loaded-but-alive filesystem answers within it, a dead mount stays silent.
|
|
20
|
+
const FS_REACHABILITY_EXTENDED_MS = 15_000;
|
|
21
|
+
let _lastSlowStatWarnAt = 0;
|
|
16
22
|
|
|
17
23
|
// Resolve true when the path is reachable (exists OR cleanly absent — ENOENT,
|
|
18
24
|
// EACCES, etc. are "the FS answered", let the real sync logic produce its own
|
|
@@ -34,8 +40,25 @@ export async function assertPathReachable(path, deadlineMs = FS_REACHABILITY_DEA
|
|
|
34
40
|
deadline,
|
|
35
41
|
]);
|
|
36
42
|
if (result === 'TIMEOUT') {
|
|
43
|
+
// Grace window: distinguish threadpool queueing from a dead mount by
|
|
44
|
+
// waiting longer for the probe that is still in flight.
|
|
45
|
+
let extTimer = null;
|
|
46
|
+
const extended = await Promise.race([
|
|
47
|
+
probe.finally(() => { if (extTimer) clearTimeout(extTimer); }),
|
|
48
|
+
new Promise((resolve) => {
|
|
49
|
+
extTimer = setTimeout(() => resolve('TIMEOUT'), Math.max(ms, FS_REACHABILITY_EXTENDED_MS - ms));
|
|
50
|
+
}),
|
|
51
|
+
]);
|
|
52
|
+
if (extended !== 'TIMEOUT') {
|
|
53
|
+
const now = Date.now();
|
|
54
|
+
if (now - _lastSlowStatWarnAt > 30_000) {
|
|
55
|
+
_lastSlowStatWarnAt = now;
|
|
56
|
+
process.stderr.write(`[fs-reachability] slow stat >${ms}ms under load (resolved in grace window): ${path}\n`);
|
|
57
|
+
}
|
|
58
|
+
return extended;
|
|
59
|
+
}
|
|
37
60
|
const err = new Error(
|
|
38
|
-
`path unreachable: stat exceeded ${
|
|
61
|
+
`path unreachable: stat exceeded ${FS_REACHABILITY_EXTENDED_MS}ms (possible dead mount / hung filesystem): ${path}`,
|
|
39
62
|
);
|
|
40
63
|
err.code = 'EFSUNREACHABLE';
|
|
41
64
|
throw err;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readdirSync } from 'fs';
|
|
2
|
-
import { basename, relative } from 'path';
|
|
2
|
+
import { basename, isAbsolute, relative } from 'path';
|
|
3
3
|
import {
|
|
4
4
|
coerceReadFamilyPathArg,
|
|
5
5
|
extractGlobBaseDirectory,
|
|
@@ -415,17 +415,52 @@ export async function executeTreeTool(args, workDir, options = {}) {
|
|
|
415
415
|
// known-incomplete and are NEVER cached.
|
|
416
416
|
const FIND_ENUM_CACHE = new Map(); // key -> { files, expiresAt, gen }
|
|
417
417
|
const FIND_ENUM_INFLIGHT = new Map(); // key -> { promise, controller, subscribers }
|
|
418
|
+
const FIND_TARGETED_BATCHES_BY_RUNNER = new WeakMap();
|
|
419
|
+
const FIND_ENUM_ROOT_GEN = new Map();
|
|
418
420
|
let FIND_ENUM_GEN = 0;
|
|
419
421
|
|
|
420
422
|
// The broad enumeration is a DERIVED cache the scope/path invalidation layer
|
|
421
|
-
// does not otherwise know about
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
423
|
+
// does not otherwise know about. Invalidate only inventories whose roots
|
|
424
|
+
// overlap the written paths; a patch in an isolated temp root must not force
|
|
425
|
+
// every active Project to rescan.
|
|
426
|
+
function findEnumerationPathsOverlap(left, right) {
|
|
427
|
+
const contains = (base, target) => {
|
|
428
|
+
const rel = relative(base, target);
|
|
429
|
+
return rel === '' || (!isAbsolute(rel) && !/^\.\.(?:[\\/]|$)/.test(rel));
|
|
430
|
+
};
|
|
431
|
+
return contains(left, right) || contains(right, left);
|
|
432
|
+
}
|
|
433
|
+
function findEnumerationRootFromKey(key) {
|
|
434
|
+
return String(key).split('\u0000', 1)[0];
|
|
435
|
+
}
|
|
436
|
+
function findEnumerationRootGeneration(root) {
|
|
437
|
+
return FIND_ENUM_ROOT_GEN.get(root) || 0;
|
|
438
|
+
}
|
|
439
|
+
registerCacheInvalidationListener((affectedPaths) => {
|
|
440
|
+
if (!Array.isArray(affectedPaths) || affectedPaths.length === 0) {
|
|
441
|
+
FIND_ENUM_GEN += 1;
|
|
442
|
+
FIND_ENUM_ROOT_GEN.clear();
|
|
443
|
+
FIND_ENUM_CACHE.clear();
|
|
444
|
+
FIND_ENUM_INFLIGHT.clear();
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
const keys = new Set([...FIND_ENUM_CACHE.keys(), ...FIND_ENUM_INFLIGHT.keys()]);
|
|
448
|
+
const affectedRoots = new Set();
|
|
449
|
+
for (const key of keys) {
|
|
450
|
+
const root = findEnumerationRootFromKey(key);
|
|
451
|
+
if (affectedPaths.some((affected) => findEnumerationPathsOverlap(root, affected))) {
|
|
452
|
+
affectedRoots.add(root);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
for (const root of affectedRoots) {
|
|
456
|
+
FIND_ENUM_ROOT_GEN.set(root, findEnumerationRootGeneration(root) + 1);
|
|
457
|
+
for (const key of [...FIND_ENUM_CACHE.keys()]) {
|
|
458
|
+
if (findEnumerationRootFromKey(key) === root) FIND_ENUM_CACHE.delete(key);
|
|
459
|
+
}
|
|
460
|
+
for (const key of [...FIND_ENUM_INFLIGHT.keys()]) {
|
|
461
|
+
if (findEnumerationRootFromKey(key) === root) FIND_ENUM_INFLIGHT.delete(key);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
429
464
|
});
|
|
430
465
|
|
|
431
466
|
function findEnumTtlMs() {
|
|
@@ -503,9 +538,10 @@ function subscribeToFindEnumeration(key, entry, signal = null) {
|
|
|
503
538
|
async function getBroadEnumeration({ root, hidden, depth, includeNoise, ignoreMode, rgArgs, cwd, runRgImpl = runRg, bestEffort = false, signal = null }) {
|
|
504
539
|
const ttl = findEnumTtlMs();
|
|
505
540
|
const key = findEnumKey({ root, hidden, depth, includeNoise, ignoreMode });
|
|
541
|
+
const rootGen = findEnumerationRootGeneration(root);
|
|
506
542
|
if (ttl > 0) {
|
|
507
543
|
const hit = FIND_ENUM_CACHE.get(key);
|
|
508
|
-
if (hit && hit.gen === FIND_ENUM_GEN && hit.expiresAt > Date.now()) {
|
|
544
|
+
if (hit && hit.gen === FIND_ENUM_GEN && hit.rootGen === rootGen && hit.expiresAt > Date.now()) {
|
|
509
545
|
return { files: hit.files, truncated: false, partial: false };
|
|
510
546
|
}
|
|
511
547
|
if (hit) FIND_ENUM_CACHE.delete(key); // expired
|
|
@@ -525,6 +561,7 @@ async function getBroadEnumeration({ root, hidden, depth, includeNoise, ignoreMo
|
|
|
525
561
|
return { files: [], truncated: false, partial: true };
|
|
526
562
|
}
|
|
527
563
|
const genAtStart = FIND_ENUM_GEN;
|
|
564
|
+
const rootGenAtStart = rootGen;
|
|
528
565
|
const controller = new AbortController();
|
|
529
566
|
const entry = { promise: null, controller, subscribers: new Set() };
|
|
530
567
|
entry.promise = (async () => {
|
|
@@ -536,8 +573,15 @@ async function getBroadEnumeration({ root, hidden, depth, includeNoise, ignoreMo
|
|
|
536
573
|
// later query with a larger head_limit must re-run the enumeration.
|
|
537
574
|
// Also never let an in-flight prewarm/real sweep repopulate after a
|
|
538
575
|
// write invalidation cleared the cache during the sweep.
|
|
539
|
-
if (ttl > 0 && !truncated && !partial
|
|
540
|
-
|
|
576
|
+
if (ttl > 0 && !truncated && !partial
|
|
577
|
+
&& FIND_ENUM_GEN === genAtStart
|
|
578
|
+
&& findEnumerationRootGeneration(root) === rootGenAtStart) {
|
|
579
|
+
FIND_ENUM_CACHE.set(key, {
|
|
580
|
+
files,
|
|
581
|
+
expiresAt: Date.now() + ttl,
|
|
582
|
+
gen: genAtStart,
|
|
583
|
+
rootGen: rootGenAtStart,
|
|
584
|
+
});
|
|
541
585
|
}
|
|
542
586
|
return { files, truncated, partial };
|
|
543
587
|
})();
|
|
@@ -585,21 +629,82 @@ async function getTargetedFindEnumeration({
|
|
|
585
629
|
const key = JSON.stringify([root, hidden, depth ?? '', includeNoise, terms]);
|
|
586
630
|
const runs = context?.targetedRuns;
|
|
587
631
|
if (runs?.has(key)) return runs.get(key);
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
partial: Boolean(stdout && typeof stdout === 'object' && stdout.partial),
|
|
632
|
+
let batches = FIND_TARGETED_BATCHES_BY_RUNNER.get(runRgImpl);
|
|
633
|
+
if (!batches) {
|
|
634
|
+
batches = new Map();
|
|
635
|
+
FIND_TARGETED_BATCHES_BY_RUNNER.set(runRgImpl, batches);
|
|
636
|
+
}
|
|
637
|
+
const batchKey = JSON.stringify([root, hidden, depth ?? '', includeNoise]);
|
|
638
|
+
let batch = batches.get(batchKey);
|
|
639
|
+
if (!batch) {
|
|
640
|
+
batch = {
|
|
641
|
+
terms: new Set(),
|
|
642
|
+
waiters: new Set(),
|
|
643
|
+
controller: new AbortController(),
|
|
601
644
|
};
|
|
602
|
-
|
|
645
|
+
batches.set(batchKey, batch);
|
|
646
|
+
setImmediate(async () => {
|
|
647
|
+
if (batches.get(batchKey) === batch) batches.delete(batchKey);
|
|
648
|
+
if (batch.waiters.size === 0) return;
|
|
649
|
+
const rgArgs = ['--files', '--no-ignore'];
|
|
650
|
+
if (hidden) rgArgs.push('--hidden');
|
|
651
|
+
if (depth != null) rgArgs.push('--max-depth', String(depth));
|
|
652
|
+
for (const query of batch.terms) {
|
|
653
|
+
rgArgs.push('--iglob', `*${escapeFindGlobLiteral(query)}*`);
|
|
654
|
+
}
|
|
655
|
+
if (!includeNoise) {
|
|
656
|
+
for (const ex of DEFAULT_IGNORE_GLOBS) rgArgs.push('--glob', ex);
|
|
657
|
+
}
|
|
658
|
+
rgArgs.push('.');
|
|
659
|
+
try {
|
|
660
|
+
const stdout = await runRgImpl(rgArgs, {
|
|
661
|
+
cwd: root,
|
|
662
|
+
signal: batch.controller.signal,
|
|
663
|
+
});
|
|
664
|
+
const paths = parseRgFileList(stdout).filter((path) =>
|
|
665
|
+
includeNoise || !path.split('/').some((segment) => NOISE_DIR_NAMES.has(segment)));
|
|
666
|
+
const result = {
|
|
667
|
+
files: paths,
|
|
668
|
+
truncated: Boolean(stdout && typeof stdout === 'object' && stdout.truncated),
|
|
669
|
+
partial: Boolean(stdout && typeof stdout === 'object' && stdout.partial),
|
|
670
|
+
};
|
|
671
|
+
for (const waiter of [...batch.waiters]) waiter.resolve(result);
|
|
672
|
+
} catch (error) {
|
|
673
|
+
for (const waiter of [...batch.waiters]) waiter.reject(error);
|
|
674
|
+
}
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
for (const term of terms) batch.terms.add(term);
|
|
678
|
+
const run = new Promise((resolve, reject) => {
|
|
679
|
+
let settled = false;
|
|
680
|
+
const waiter = {
|
|
681
|
+
resolve(value) {
|
|
682
|
+
if (settled) return;
|
|
683
|
+
settled = true;
|
|
684
|
+
batch.waiters.delete(waiter);
|
|
685
|
+
if (signal instanceof AbortSignal) signal.removeEventListener('abort', onAbort);
|
|
686
|
+
resolve(value);
|
|
687
|
+
},
|
|
688
|
+
reject(error) {
|
|
689
|
+
if (settled) return;
|
|
690
|
+
settled = true;
|
|
691
|
+
batch.waiters.delete(waiter);
|
|
692
|
+
if (signal instanceof AbortSignal) signal.removeEventListener('abort', onAbort);
|
|
693
|
+
reject(error);
|
|
694
|
+
},
|
|
695
|
+
};
|
|
696
|
+
const onAbort = () => {
|
|
697
|
+
waiter.reject(findEnumerationAbortError(signal));
|
|
698
|
+
if (batch.waiters.size === 0) {
|
|
699
|
+
try { batch.controller.abort(findEnumerationAbortError(signal)); } catch {}
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
batch.waiters.add(waiter);
|
|
703
|
+
if (signal instanceof AbortSignal) {
|
|
704
|
+
if (signal.aborted) onAbort();
|
|
705
|
+
else signal.addEventListener('abort', onAbort, { once: true });
|
|
706
|
+
}
|
|
707
|
+
});
|
|
603
708
|
if (runs) runs.set(key, run);
|
|
604
709
|
return run;
|
|
605
710
|
}
|