@qmediat.io/gemini-code-context-mcp 1.23.1 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +161 -1
- package/README.md +1 -1
- package/dist/cache/ttl-watcher.d.ts +29 -0
- package/dist/cache/ttl-watcher.d.ts.map +1 -1
- package/dist/cache/ttl-watcher.js +104 -43
- package/dist/cache/ttl-watcher.js.map +1 -1
- package/dist/gemini/client.d.ts +7 -0
- package/dist/gemini/client.d.ts.map +1 -1
- package/dist/gemini/client.js +8 -1
- package/dist/gemini/client.js.map +1 -1
- package/dist/gemini/long-decode.d.ts +153 -0
- package/dist/gemini/long-decode.d.ts.map +1 -0
- package/dist/gemini/long-decode.js +178 -0
- package/dist/gemini/long-decode.js.map +1 -0
- package/dist/gemini/model-registry.d.ts +2 -0
- package/dist/gemini/model-registry.d.ts.map +1 -1
- package/dist/gemini/model-registry.js +5 -2
- package/dist/gemini/model-registry.js.map +1 -1
- package/dist/gemini/models.d.ts.map +1 -1
- package/dist/gemini/models.js +1 -0
- package/dist/gemini/models.js.map +1 -1
- package/dist/gemini/retry.d.ts +8 -0
- package/dist/gemini/retry.d.ts.map +1 -1
- package/dist/gemini/retry.js +33 -1
- package/dist/gemini/retry.js.map +1 -1
- package/dist/tools/agentic/dispatch.d.ts +68 -0
- package/dist/tools/agentic/dispatch.d.ts.map +1 -0
- package/dist/tools/agentic/dispatch.js +167 -0
- package/dist/tools/agentic/dispatch.js.map +1 -0
- package/dist/tools/agentic/input.d.ts +28 -0
- package/dist/tools/agentic/input.d.ts.map +1 -0
- package/dist/tools/agentic/input.js +66 -0
- package/dist/tools/agentic/input.js.map +1 -0
- package/dist/tools/agentic/instructions.d.ts +46 -0
- package/dist/tools/agentic/instructions.d.ts.map +1 -0
- package/dist/tools/agentic/instructions.js +170 -0
- package/dist/tools/agentic/instructions.js.map +1 -0
- package/dist/tools/agentic/iteration.d.ts +81 -0
- package/dist/tools/agentic/iteration.d.ts.map +1 -0
- package/dist/tools/agentic/iteration.js +317 -0
- package/dist/tools/agentic/iteration.js.map +1 -0
- package/dist/tools/agentic/ledger.d.ts +64 -0
- package/dist/tools/agentic/ledger.d.ts.map +1 -0
- package/dist/tools/agentic/ledger.js +110 -0
- package/dist/tools/agentic/ledger.js.map +1 -0
- package/dist/tools/agentic/loop.d.ts +10 -0
- package/dist/tools/agentic/loop.d.ts.map +1 -0
- package/dist/tools/agentic/loop.js +179 -0
- package/dist/tools/agentic/loop.js.map +1 -0
- package/dist/tools/agentic/no-progress.d.ts +90 -0
- package/dist/tools/agentic/no-progress.d.ts.map +1 -0
- package/dist/tools/agentic/no-progress.js +107 -0
- package/dist/tools/agentic/no-progress.js.map +1 -0
- package/dist/tools/agentic/pass-clock.d.ts +85 -0
- package/dist/tools/agentic/pass-clock.d.ts.map +1 -0
- package/dist/tools/agentic/pass-clock.js +162 -0
- package/dist/tools/agentic/pass-clock.js.map +1 -0
- package/dist/tools/agentic/phase-defaults.d.ts +29 -0
- package/dist/tools/agentic/phase-defaults.d.ts.map +1 -0
- package/dist/tools/agentic/phase-defaults.js +23 -0
- package/dist/tools/agentic/phase-defaults.js.map +1 -0
- package/dist/tools/agentic/rescue.d.ts +69 -0
- package/dist/tools/agentic/rescue.d.ts.map +1 -0
- package/dist/tools/agentic/rescue.js +203 -0
- package/dist/tools/agentic/rescue.js.map +1 -0
- package/dist/tools/agentic/response.d.ts +16 -0
- package/dist/tools/agentic/response.d.ts.map +1 -0
- package/dist/tools/agentic/response.js +25 -0
- package/dist/tools/agentic/response.js.map +1 -0
- package/dist/tools/agentic/results.d.ts +76 -0
- package/dist/tools/agentic/results.d.ts.map +1 -0
- package/dist/tools/agentic/results.js +228 -0
- package/dist/tools/agentic/results.js.map +1 -0
- package/dist/tools/agentic/run.d.ts +130 -0
- package/dist/tools/agentic/run.d.ts.map +1 -0
- package/dist/tools/agentic/run.js +131 -0
- package/dist/tools/agentic/run.js.map +1 -0
- package/dist/tools/agentic/sandbox.d.ts +13 -2
- package/dist/tools/agentic/sandbox.d.ts.map +1 -1
- package/dist/tools/agentic/sandbox.js +127 -111
- package/dist/tools/agentic/sandbox.js.map +1 -1
- package/dist/tools/agentic/workspace-tools.d.ts +30 -27
- package/dist/tools/agentic/workspace-tools.d.ts.map +1 -1
- package/dist/tools/agentic/workspace-tools.js +324 -385
- package/dist/tools/agentic/workspace-tools.js.map +1 -1
- package/dist/tools/ask-agentic-fallback.d.ts +33 -0
- package/dist/tools/ask-agentic-fallback.d.ts.map +1 -0
- package/dist/tools/ask-agentic-fallback.js +113 -0
- package/dist/tools/ask-agentic-fallback.js.map +1 -0
- package/dist/tools/ask-agentic.tool.d.ts +11 -44
- package/dist/tools/ask-agentic.tool.d.ts.map +1 -1
- package/dist/tools/ask-agentic.tool.js +143 -1618
- package/dist/tools/ask-agentic.tool.js.map +1 -1
- package/dist/tools/ask.tool.d.ts.map +1 -1
- package/dist/tools/ask.tool.js +195 -555
- package/dist/tools/ask.tool.js.map +1 -1
- package/dist/tools/code.tool.d.ts +6 -0
- package/dist/tools/code.tool.d.ts.map +1 -1
- package/dist/tools/code.tool.js +155 -306
- package/dist/tools/code.tool.js.map +1 -1
- package/dist/tools/shared/abort-timeout.d.ts +48 -4
- package/dist/tools/shared/abort-timeout.d.ts.map +1 -1
- package/dist/tools/shared/abort-timeout.js +116 -95
- package/dist/tools/shared/abort-timeout.js.map +1 -1
- package/dist/tools/shared/call-ledger.d.ts +88 -0
- package/dist/tools/shared/call-ledger.d.ts.map +1 -0
- package/dist/tools/shared/call-ledger.js +133 -0
- package/dist/tools/shared/call-ledger.js.map +1 -0
- package/dist/tools/shared/context-decode.d.ts +77 -0
- package/dist/tools/shared/context-decode.d.ts.map +1 -0
- package/dist/tools/shared/context-decode.js +125 -0
- package/dist/tools/shared/context-decode.js.map +1 -0
- package/dist/tools/shared/continuation.d.ts +124 -0
- package/dist/tools/shared/continuation.d.ts.map +1 -0
- package/dist/tools/shared/continuation.js +302 -0
- package/dist/tools/shared/continuation.js.map +1 -0
- package/dist/tools/shared/dispatch-throttle.d.ts +43 -0
- package/dist/tools/shared/dispatch-throttle.d.ts.map +1 -0
- package/dist/tools/shared/dispatch-throttle.js +65 -0
- package/dist/tools/shared/dispatch-throttle.js.map +1 -0
- package/dist/tools/shared/question-fields.d.ts +10 -0
- package/dist/tools/shared/question-fields.d.ts.map +1 -0
- package/dist/tools/shared/question-fields.js +13 -0
- package/dist/tools/shared/question-fields.js.map +1 -0
- package/dist/tools/shared/settle-call.d.ts +68 -0
- package/dist/tools/shared/settle-call.d.ts.map +1 -0
- package/dist/tools/shared/settle-call.js +59 -0
- package/dist/tools/shared/settle-call.js.map +1 -0
- package/dist/tools/shared/stream-collector.d.ts +7 -0
- package/dist/tools/shared/stream-collector.d.ts.map +1 -1
- package/dist/tools/shared/stream-collector.js +5 -1
- package/dist/tools/shared/stream-collector.js.map +1 -1
- package/dist/tools/shared/thinking.d.ts +41 -11
- package/dist/tools/shared/thinking.d.ts.map +1 -1
- package/dist/tools/shared/thinking.js +62 -9
- package/dist/tools/shared/thinking.js.map +1 -1
- package/dist/tools/shared/throttle.d.ts +7 -0
- package/dist/tools/shared/throttle.d.ts.map +1 -1
- package/dist/tools/shared/throttle.js +18 -0
- package/dist/tools/shared/throttle.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/cost-estimator.d.ts +17 -0
- package/dist/utils/cost-estimator.d.ts.map +1 -1
- package/dist/utils/cost-estimator.js +15 -2
- package/dist/utils/cost-estimator.js.map +1 -1
- package/package.json +4 -3
|
@@ -20,9 +20,11 @@
|
|
|
20
20
|
*/
|
|
21
21
|
import { readFile, readdir, realpath, stat } from 'node:fs/promises';
|
|
22
22
|
import { join } from 'node:path';
|
|
23
|
+
import { throwIfAborted } from '../../gemini/retry.js';
|
|
23
24
|
import { defaultMatchConfig, isPathExcluded } from '../../indexer/globs.js';
|
|
24
25
|
import { isSourceFile } from '../../indexer/source-files.js';
|
|
25
|
-
import {
|
|
26
|
+
import { logger } from '../../utils/logger.js';
|
|
27
|
+
import { SandboxError, resolveInsideWorkspace, secretListOf } from './sandbox.js';
|
|
26
28
|
/**
|
|
27
29
|
* Build a `MatchConfig` honouring the caller's `includeGlobs`/`excludeGlobs`
|
|
28
30
|
* (v1.9.0+). When `matchConfig` is undefined, returns the defaults — the same
|
|
@@ -38,12 +40,21 @@ import { SandboxError, resolveInsideWorkspace } from './sandbox.js';
|
|
|
38
40
|
function resolveMatchConfig(matchConfig) {
|
|
39
41
|
return matchConfig ?? defaultMatchConfig({});
|
|
40
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* A checkpoint of an executor's work — every place it reads the signal: rejects with the signal's reason once it
|
|
45
|
+
* fired, else reports progress. Reading the signal and reporting progress are one call, so they cannot drift apart.
|
|
46
|
+
*/
|
|
47
|
+
export function checkpoint(watch) {
|
|
48
|
+
throwIfAborted(watch?.signal);
|
|
49
|
+
watch?.onProgress?.();
|
|
50
|
+
}
|
|
41
51
|
/**
|
|
42
52
|
* Exclude-side check for `list_directory` (v1.9.0+). Returns `true` when the
|
|
43
|
-
* file should be hidden from a navigation listing — covers default filename
|
|
44
|
-
*
|
|
53
|
+
* file should be hidden from a navigation listing — covers the default filename
|
|
54
|
+
* excludes (lockfiles, `.netrc`, …), the default extension excludes
|
|
45
55
|
* (`.tsbuildinfo`), user `excludeGlobs` filename / extension entries, AND
|
|
46
|
-
* dir-prefix excludes that match the file's parent path.
|
|
56
|
+
* dir-prefix excludes that match the file's parent path. The secret denylist is
|
|
57
|
+
* the sandbox's (`secretListOf`), checked beside this. Deliberately does
|
|
47
58
|
* NOT enforce include-extension membership — `list_directory` is a
|
|
48
59
|
* navigation aid; the read-side gate lives in `readFileExecutor`.
|
|
49
60
|
*
|
|
@@ -143,36 +154,15 @@ function globToRegExp(glob) {
|
|
|
143
154
|
return new RegExp(`^${pattern}$`, 'i');
|
|
144
155
|
}
|
|
145
156
|
/**
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
* paths it is allowed to ask for via `read_file` / `grep`.
|
|
157
|
+
* The children of a directory the model asked to list. A path that exists but is a file is `NOT_A_DIRECTORY` (the model
|
|
158
|
+
* then calls `read_file`), anything else `NOT_FOUND`.
|
|
149
159
|
*/
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
const config = resolveMatchConfig(matchConfig);
|
|
153
|
-
// Top-level gate (v1.9.0 Phase 1.1, /6step Finding #1): the requested
|
|
154
|
-
// directory itself must be checked against user `excludeGlobs`. Without
|
|
155
|
-
// this, `list_directory("internal-secrets")` (where `internal-secrets`
|
|
156
|
-
// is in user excludes but NOT in `DEFAULT_EXCLUDE_DIRS`) succeeds and
|
|
157
|
-
// returns a filtered child list — the model can probe path existence
|
|
158
|
-
// by comparing success vs `NOT_FOUND` from `resolveInsideWorkspace`.
|
|
159
|
-
// The sandbox layer (`resolveInsideWorkspace`) checks default-excluded
|
|
160
|
-
// dirs only; user-supplied excludes were unenforced at this level.
|
|
161
|
-
// Generic message — no path interpolation — to avoid the same
|
|
162
|
-
// existence-leak via error string (see Finding #2 for the file-level
|
|
163
|
-
// analogue).
|
|
164
|
-
if (target.relpath && isPathExcluded(target.relpath, config, target.kind)) {
|
|
165
|
-
throw new SandboxError('EXCLUDED_DIR', 'directory is excluded by configured policy', relPath);
|
|
166
|
-
}
|
|
167
|
-
let rawEntries;
|
|
160
|
+
async function directoryEntries(target, relPath, watch) {
|
|
161
|
+
checkpoint(watch);
|
|
168
162
|
try {
|
|
169
|
-
|
|
163
|
+
return await readdir(target.absolutePath, { withFileTypes: true, encoding: 'utf8' });
|
|
170
164
|
}
|
|
171
165
|
catch (err) {
|
|
172
|
-
// Distinguish "path exists but is a file" from "path missing" — same
|
|
173
|
-
// taxonomy grep's `pathPrefix` already uses. Blanket NOT_FOUND here
|
|
174
|
-
// misled the model into retrying a non-existent alias when the real
|
|
175
|
-
// fix was calling `read_file`. PR #24 round-4 review (Copilot P1).
|
|
176
166
|
const code = typeof err === 'object' && err !== null && 'code' in err
|
|
177
167
|
? String(err.code)
|
|
178
168
|
: undefined;
|
|
@@ -181,425 +171,374 @@ export async function listDirectoryExecutor(workspaceRoot, relPath, matchConfig)
|
|
|
181
171
|
}
|
|
182
172
|
throw new SandboxError('NOT_FOUND', `cannot list: ${String(err)}`, relPath);
|
|
183
173
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
try {
|
|
217
|
-
const s = await stat(join(target.absolutePath, entry.name));
|
|
218
|
-
child.size = s.size;
|
|
219
|
-
}
|
|
220
|
-
catch {
|
|
221
|
-
/* stat failure — omit size, still include entry */
|
|
222
|
-
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* A child as a listing shows it, or `null` when it is hidden: symlinks, sockets and other non-file, non-directory
|
|
177
|
+
* entries are skipped; an excluded directory (defaults and the caller's `excludeGlobs`, matched on the full nested
|
|
178
|
+
* path), a file on the secret denylist and an excluded file are omitted. A file is NOT required to match an include
|
|
179
|
+
* extension: `list_directory` is a navigation aid, the read-side gate is `read_file`'s.
|
|
180
|
+
*/
|
|
181
|
+
function visibleChild(parentRel, entry, config) {
|
|
182
|
+
const type = entry.isDirectory() ? 'dir' : entry.isFile() ? 'file' : null;
|
|
183
|
+
if (type === null)
|
|
184
|
+
return null;
|
|
185
|
+
const relpath = parentRel ? `${parentRel}/${entry.name}` : entry.name;
|
|
186
|
+
const hidden = type === 'dir'
|
|
187
|
+
? isPathExcluded(relpath, config, 'dir')
|
|
188
|
+
: secretListOf(entry.name) !== null || isFileExcludedByConfig(relpath, entry.name, config);
|
|
189
|
+
return hidden ? null : { relpath, type };
|
|
190
|
+
}
|
|
191
|
+
/** Add one child to a listing: past `MAX_LIST_ENTRIES` it is counted, not returned; a file's size is omitted when its
|
|
192
|
+
* `stat` fails. */
|
|
193
|
+
async function listEntry(target, entry, config, listing) {
|
|
194
|
+
const child = visibleChild(target.relpath, entry, config);
|
|
195
|
+
if (child === null)
|
|
196
|
+
return;
|
|
197
|
+
listing.total += 1;
|
|
198
|
+
if (listing.entries.length >= MAX_LIST_ENTRIES)
|
|
199
|
+
return;
|
|
200
|
+
if (child.type === 'file') {
|
|
201
|
+
try {
|
|
202
|
+
child.size = (await stat(join(target.absolutePath, entry.name))).size;
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
/* stat failure — omit size, still include entry */
|
|
223
206
|
}
|
|
224
|
-
|
|
207
|
+
}
|
|
208
|
+
listing.entries.push(child);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* List immediate children of `relPath` (non-recursive). Default-excluded dirs and denylisted filenames are omitted; the
|
|
212
|
+
* model sees only paths it is allowed to ask for via `read_file` / `grep`. The requested directory itself is checked
|
|
213
|
+
* against the caller's `excludeGlobs` first — with a message that names no path, so success vs `NOT_FOUND` is no oracle
|
|
214
|
+
* for an excluded directory's existence. A checkpoint (`watch`) precedes the read and each entry.
|
|
215
|
+
*/
|
|
216
|
+
export async function listDirectoryExecutor(workspaceRoot, relPath, matchConfig, watch) {
|
|
217
|
+
const target = await resolveInsideWorkspace(workspaceRoot, relPath);
|
|
218
|
+
const config = resolveMatchConfig(matchConfig);
|
|
219
|
+
if (target.relpath && isPathExcluded(target.relpath, config, target.kind)) {
|
|
220
|
+
throw new SandboxError('EXCLUDED_DIR', 'directory is excluded by configured policy', relPath);
|
|
221
|
+
}
|
|
222
|
+
const listing = { entries: [], total: 0 };
|
|
223
|
+
for (const entry of await directoryEntries(target, relPath, watch)) {
|
|
224
|
+
checkpoint(watch);
|
|
225
|
+
await listEntry(target, entry, config, listing);
|
|
225
226
|
}
|
|
226
227
|
return {
|
|
227
228
|
path: target.relpath || '.',
|
|
228
|
-
entries,
|
|
229
|
-
truncated:
|
|
230
|
-
totalEntries,
|
|
229
|
+
entries: listing.entries,
|
|
230
|
+
truncated: listing.total > listing.entries.length,
|
|
231
|
+
totalEntries: listing.total,
|
|
231
232
|
};
|
|
232
233
|
}
|
|
234
|
+
function treeWalk(config, watch, visitFile, full = () => false) {
|
|
235
|
+
return { config, watch, seenReal: new Set(), visitFile, full, depthExceeded: false };
|
|
236
|
+
}
|
|
237
|
+
/** A directory's entries, or `null` when it was walked already (a symlink loop) or cannot be read. */
|
|
238
|
+
async function enterDirectory(walk, currentAbs) {
|
|
239
|
+
try {
|
|
240
|
+
const real = await realpath(currentAbs);
|
|
241
|
+
if (walk.seenReal.has(real))
|
|
242
|
+
return null;
|
|
243
|
+
walk.seenReal.add(real);
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
try {
|
|
249
|
+
return await readdir(currentAbs, { withFileTypes: true, encoding: 'utf8' });
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
233
255
|
/**
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
* sees paths it couldn't `read_file` on anyway — avoids a foot-gun where
|
|
237
|
-
* the model tries to open a secret it saw in a `find_files` response).
|
|
256
|
+
* One entry of a walked directory: an included directory is walked, a visible file visited, the rest skipped — a file
|
|
257
|
+
* on the secret denylist is never visited, so `find_files` does not list it and `grep` never opens it.
|
|
238
258
|
*/
|
|
239
|
-
|
|
259
|
+
async function walkEntry(walk, entry, parent) {
|
|
260
|
+
const childRel = parent.rel ? `${parent.rel}/${entry.name}` : entry.name;
|
|
261
|
+
const childAbs = join(parent.abs, entry.name);
|
|
262
|
+
if (entry.isDirectory()) {
|
|
263
|
+
if (isPathExcluded(childRel, walk.config, 'dir'))
|
|
264
|
+
return;
|
|
265
|
+
await walkTree(walk, childAbs, childRel, parent.depth + 1);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
if (!entry.isFile() || secretListOf(entry.name) !== null)
|
|
269
|
+
return;
|
|
270
|
+
if (!(await isSourceFile(childAbs, childRel, walk.config)))
|
|
271
|
+
return;
|
|
272
|
+
await walk.visitFile(childAbs, childRel);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Walk a tree depth-first, at most `MAX_WALK_DEPTH` levels below its start (deeper directories set `depthExceeded`).
|
|
276
|
+
* A checkpoint (`walk.watch`) at every directory and before every entry: once the signal fired the walk rejects with
|
|
277
|
+
* its reason.
|
|
278
|
+
*/
|
|
279
|
+
async function walkTree(walk, currentAbs, currentRel, depth) {
|
|
280
|
+
if (walk.full())
|
|
281
|
+
return;
|
|
282
|
+
if (depth > MAX_WALK_DEPTH) {
|
|
283
|
+
walk.depthExceeded = true;
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
checkpoint(walk.watch);
|
|
287
|
+
const entries = await enterDirectory(walk, currentAbs);
|
|
288
|
+
for (const entry of entries ?? []) {
|
|
289
|
+
if (walk.full())
|
|
290
|
+
return;
|
|
291
|
+
checkpoint(walk.watch);
|
|
292
|
+
await walkEntry(walk, entry, { abs: currentAbs, rel: currentRel, depth });
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Recursive glob search inside the workspace. Respects default excludes, the secret denylist and extension gating, so
|
|
297
|
+
* the model never sees paths it could not `read_file` on anyway.
|
|
298
|
+
*/
|
|
299
|
+
export async function findFilesExecutor(workspaceRoot, pattern, matchConfig, watch) {
|
|
240
300
|
if (typeof pattern !== 'string' || pattern.trim().length === 0) {
|
|
241
301
|
throw new SandboxError('INVALID_INPUT', 'pattern is empty', pattern);
|
|
242
302
|
}
|
|
243
303
|
const regex = globToRegExp(pattern.trim());
|
|
244
|
-
const config = resolveMatchConfig(matchConfig);
|
|
245
304
|
const matches = [];
|
|
246
305
|
let totalMatches = 0;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
const seenReal = new Set();
|
|
250
|
-
let depthExceeded = false;
|
|
251
|
-
async function walk(currentAbs, currentRel, depth) {
|
|
252
|
-
if (depth > MAX_WALK_DEPTH) {
|
|
253
|
-
depthExceeded = true;
|
|
306
|
+
const walk = treeWalk(resolveMatchConfig(matchConfig), watch, async (_absFile, relFile) => {
|
|
307
|
+
if (!regex.test(relFile))
|
|
254
308
|
return;
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
return;
|
|
261
|
-
seenReal.add(real);
|
|
262
|
-
}
|
|
263
|
-
catch {
|
|
264
|
-
// Unreadable parent — nothing to walk into.
|
|
265
|
-
return;
|
|
266
|
-
}
|
|
267
|
-
let entries;
|
|
268
|
-
try {
|
|
269
|
-
entries = await readdir(currentAbs, { withFileTypes: true, encoding: 'utf8' });
|
|
270
|
-
}
|
|
271
|
-
catch {
|
|
272
|
-
return;
|
|
273
|
-
}
|
|
274
|
-
for (const entry of entries) {
|
|
275
|
-
const childRel = currentRel ? `${currentRel}/${entry.name}` : entry.name;
|
|
276
|
-
if (entry.isDirectory()) {
|
|
277
|
-
// Honour user `excludeGlobs` dir entries in addition to defaults
|
|
278
|
-
// — `isPathExcluded` checks the full nested path, so multi-segment
|
|
279
|
-
// globs like `src/vendor` work even when we hit them mid-walk.
|
|
280
|
-
if (isPathExcluded(childRel, config, 'dir'))
|
|
281
|
-
continue;
|
|
282
|
-
await walk(join(currentAbs, entry.name), childRel, depth + 1);
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
if (!entry.isFile())
|
|
286
|
-
continue;
|
|
287
|
-
// Unified filter: covers default filename / extension excludes,
|
|
288
|
-
// user `excludeGlobs` filename / extension entries, AND requires
|
|
289
|
-
// an include-extension match (default + user) or a `#!` script.
|
|
290
|
-
// Same predicate the eager scanner uses — no agentic / eager divergence.
|
|
291
|
-
if (!(await isSourceFile(join(currentAbs, entry.name), childRel, config)))
|
|
292
|
-
continue;
|
|
293
|
-
if (!regex.test(childRel))
|
|
294
|
-
continue;
|
|
295
|
-
totalMatches += 1;
|
|
296
|
-
if (matches.length < MAX_FIND_MATCHES)
|
|
297
|
-
matches.push(childRel);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
await walk(workspaceRoot, '', 0);
|
|
309
|
+
totalMatches += 1;
|
|
310
|
+
if (matches.length < MAX_FIND_MATCHES)
|
|
311
|
+
matches.push(relFile);
|
|
312
|
+
});
|
|
313
|
+
await walkTree(walk, workspaceRoot, '', 0);
|
|
301
314
|
return {
|
|
302
315
|
pattern,
|
|
303
316
|
matches,
|
|
304
|
-
truncated: totalMatches > matches.length || depthExceeded,
|
|
317
|
+
truncated: totalMatches > matches.length || walk.depthExceeded,
|
|
305
318
|
totalMatches,
|
|
306
319
|
};
|
|
307
320
|
}
|
|
321
|
+
/** A file larger than this is never loaded: its response carries metadata only. */
|
|
322
|
+
const HARD_FILE_SIZE_LIMIT = 5 * MAX_READ_BYTES; // 1MB
|
|
308
323
|
/**
|
|
309
|
-
*
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
* - Omit both → return the first `DEFAULT_READ_LINE_LIMIT` lines, up to
|
|
316
|
-
* `MAX_READ_BYTES`, whichever hits first.
|
|
324
|
+
* Refuse a file the source rule does not admit. A file a rule excludes (a caller's `excludeGlobs`, a default exclude:
|
|
325
|
+
* directory, filename or extension) gets a generic message with NO path, whatever its extension — the error string is
|
|
326
|
+
* no existence oracle for paths the caller excluded. Any other refused file is structurally non-source (binary, image,
|
|
327
|
+
* a file without `#!`): the path stays in the message, the model needs to know which file is "wrong-tool" territory.
|
|
328
|
+
* `relPath` is kept either way for internal logging via `requestedPath`. The same gate holds on `ask`'s fallback, so a
|
|
329
|
+
* filter given to `ask` is honoured by `ask_agentic`.
|
|
317
330
|
*/
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
// "default-rejected source-set membership".
|
|
335
|
-
if (!(await isSourceFile(target.absolutePath, target.relpath, config))) {
|
|
336
|
-
// Two failure modes. A file a rule excludes (a caller's `excludeGlobs`, a default exclude: directory, filename
|
|
337
|
-
// or extension) gets a generic message with NO path (v1.9.0 Finding #2) — whatever its extension — so the error
|
|
338
|
-
// string can't be used as an existence oracle for paths the user explicitly excluded. Any other refused file is
|
|
339
|
-
// structurally non-source (binary, image, a file without `#!`): the path stays in the message, the model needs
|
|
340
|
-
// to know which file it asked for is "wrong-tool" territory. The third `SandboxError` argument (`relPath`) is
|
|
341
|
-
// kept either way for internal logging via `requestedPath`.
|
|
342
|
-
const fileName = target.relpath.slice(target.relpath.lastIndexOf('/') + 1);
|
|
343
|
-
if (isFileExcludedByConfig(target.relpath, fileName, config)) {
|
|
344
|
-
throw new SandboxError('EXCLUDED_FILE', 'file is excluded by configured policy', relPath);
|
|
345
|
-
}
|
|
346
|
-
throw new SandboxError('NON_SOURCE_FILE', `file extension not in allowed source set, and not a readable #! script: ${target.relpath}`, relPath);
|
|
347
|
-
}
|
|
348
|
-
// Stat first so we never allocate a >200MB Buffer for a minified bundle.
|
|
349
|
-
// Files larger than 5× MAX_READ_BYTES (1MB) get rejected with a
|
|
350
|
-
// metadata-only response so the model can skip them instead of DOSing
|
|
351
|
-
// the process. Reported in PR #24 review by GPT.
|
|
352
|
-
let totalBytes;
|
|
353
|
-
let regularFile;
|
|
331
|
+
async function assertReadable(target, relPath, config) {
|
|
332
|
+
if (await isSourceFile(target.absolutePath, target.relpath, config))
|
|
333
|
+
return;
|
|
334
|
+
const fileName = target.relpath.slice(target.relpath.lastIndexOf('/') + 1);
|
|
335
|
+
if (isFileExcludedByConfig(target.relpath, fileName, config)) {
|
|
336
|
+
throw new SandboxError('EXCLUDED_FILE', 'file is excluded by configured policy', relPath);
|
|
337
|
+
}
|
|
338
|
+
throw new SandboxError('NON_SOURCE_FILE', `file extension not in allowed source set, and not a readable #! script: ${target.relpath}`, relPath);
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* The size of a regular file, stated before it is read so a minified bundle is never loaded whole. A named pipe,
|
|
342
|
+
* socket or device with a source extension (`pipe.ts`) passes the source rule, and reading it could block forever:
|
|
343
|
+
* it is refused.
|
|
344
|
+
*/
|
|
345
|
+
async function regularFileSize(target, relPath) {
|
|
346
|
+
let st;
|
|
354
347
|
try {
|
|
355
|
-
|
|
356
|
-
totalBytes = st.size;
|
|
357
|
-
regularFile = st.isFile();
|
|
348
|
+
st = await stat(target.absolutePath);
|
|
358
349
|
}
|
|
359
350
|
catch (err) {
|
|
360
351
|
throw new SandboxError('NOT_FOUND', `stat failed: ${String(err)}`, relPath);
|
|
361
352
|
}
|
|
362
|
-
|
|
363
|
-
// block forever. The scan and find/grep skip such entries by their dirent type; read_file has only the path.
|
|
364
|
-
if (!regularFile) {
|
|
353
|
+
if (!st.isFile()) {
|
|
365
354
|
throw new SandboxError('NON_SOURCE_FILE', `not a regular file: ${target.relpath}`, relPath);
|
|
366
355
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
388
|
-
let buffer;
|
|
356
|
+
return st.size;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* The response for a file over `HARD_FILE_SIZE_LIMIT`: its buffer is never allocated, so slicing is not offered — the
|
|
360
|
+
* message steers the model to `grep` or to skipping the file.
|
|
361
|
+
*/
|
|
362
|
+
function tooLargeToRead(relpath, totalBytes) {
|
|
363
|
+
return {
|
|
364
|
+
relpath,
|
|
365
|
+
content: `[file too large to inline: ${totalBytes} bytes > ${HARD_FILE_SIZE_LIMIT} byte cap. Use \`grep\` with a narrow pattern, or skip this file. Slicing via startLine/endLine is not supported at this size.]`,
|
|
366
|
+
startLine: 1,
|
|
367
|
+
endLine: 0,
|
|
368
|
+
totalLines: 0,
|
|
369
|
+
truncated: true,
|
|
370
|
+
truncationReason: 'max_bytes',
|
|
371
|
+
totalBytes,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
/** The file's bytes. A read the signal aborted rejects with the signal's reason, any other failure is `NOT_FOUND`. */
|
|
375
|
+
async function readWhole(absolutePath, relPath, watch) {
|
|
389
376
|
try {
|
|
390
|
-
|
|
377
|
+
return await readFile(absolutePath, { signal: watch?.signal });
|
|
391
378
|
}
|
|
392
379
|
catch (err) {
|
|
380
|
+
checkpoint(watch);
|
|
393
381
|
throw new SandboxError('NOT_FOUND', `read failed: ${String(err)}`, relPath);
|
|
394
382
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
const sliceStart = typeof startLine === 'number' && Number.isFinite(startLine) && startLine >= 1
|
|
383
|
+
}
|
|
384
|
+
/** The 1-indexed inclusive line range to return: the given bounds when they are finite and ordered, else the defaults. */
|
|
385
|
+
function lineRange(startLine, endLine, totalLines) {
|
|
386
|
+
const start = typeof startLine === 'number' && Number.isFinite(startLine) && startLine >= 1
|
|
400
387
|
? Math.floor(startLine)
|
|
401
388
|
: 1;
|
|
402
|
-
const rawEnd = typeof endLine === 'number' && Number.isFinite(endLine) && endLine >=
|
|
389
|
+
const rawEnd = typeof endLine === 'number' && Number.isFinite(endLine) && endLine >= start
|
|
403
390
|
? Math.floor(endLine)
|
|
404
|
-
:
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
391
|
+
: start + DEFAULT_READ_LINE_LIMIT - 1;
|
|
392
|
+
return { start, end: Math.min(totalLines, rawEnd) };
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* `content` cut to `MAX_READ_BYTES`: bytes, not characters, re-decoded with `fatal: false` so a mid-rune cut is one
|
|
396
|
+
* replacement character, then backed off to the last newline (which drops that character too). Content with no
|
|
397
|
+
* newline in the capped region (single-line minified code) has its trailing replacement characters stripped.
|
|
398
|
+
*/
|
|
399
|
+
function capToReadBytes(content) {
|
|
400
|
+
const cappedBuf = Buffer.from(content, 'utf8').subarray(0, MAX_READ_BYTES);
|
|
401
|
+
const decoder = new TextDecoder('utf-8', { fatal: false, ignoreBOM: true });
|
|
402
|
+
const cappedText = decoder.decode(cappedBuf);
|
|
403
|
+
const lastNewline = cappedText.lastIndexOf('\n');
|
|
404
|
+
return lastNewline > 0 ? cappedText.slice(0, lastNewline) : cappedText.replace(/\uFFFD+$/, '');
|
|
405
|
+
}
|
|
406
|
+
/** The slice of a file's text a `read_file` returns, with what the model needs to ask for the next slice. */
|
|
407
|
+
function readSlice(relpath, buffer, range, totalBytes) {
|
|
408
|
+
const allLines = buffer.toString('utf8').split('\n');
|
|
409
|
+
const totalLines = allLines.length;
|
|
410
|
+
const { start, end } = lineRange(range.startLine, range.endLine, totalLines);
|
|
411
|
+
let content = allLines.slice(start - 1, end).join('\n');
|
|
412
|
+
let truncationReason = end < totalLines ? 'max_lines' : undefined;
|
|
412
413
|
if (Buffer.byteLength(content, 'utf8') > MAX_READ_BYTES) {
|
|
413
|
-
|
|
414
|
-
// match the limit exactly; then re-decode with `TextDecoder` using
|
|
415
|
-
// `ignoreBOM` + `fatal: false` so a mid-rune cut produces a single
|
|
416
|
-
// replacement character instead of a malformed string. Back off to
|
|
417
|
-
// the last newline to avoid emitting a partial line — that naturally
|
|
418
|
-
// chops any trailing replacement char as well. Reported in PR #24
|
|
419
|
-
// review by Gemini and Grok.
|
|
420
|
-
const cappedBuf = Buffer.from(content, 'utf8').subarray(0, MAX_READ_BYTES);
|
|
421
|
-
const decoder = new TextDecoder('utf-8', { fatal: false, ignoreBOM: true });
|
|
422
|
-
const cappedText = decoder.decode(cappedBuf);
|
|
423
|
-
const lastNewline = cappedText.lastIndexOf('\n');
|
|
424
|
-
// Two truncation paths:
|
|
425
|
-
// - If there's a newline in the capped region, drop everything after
|
|
426
|
-
// the last newline — that naturally trims any mid-rune replacement
|
|
427
|
-
// character that landed on the tail.
|
|
428
|
-
// - If there's NO newline (realistic for single-line minified
|
|
429
|
-
// content at 200 KB), the fallback used to keep the raw decoded
|
|
430
|
-
// text including a trailing U+FFFD. Strip trailing replacements
|
|
431
|
-
// explicitly. Reported in PR #24 round-3 review by Gemini +
|
|
432
|
-
// self-review.
|
|
433
|
-
content =
|
|
434
|
-
lastNewline > 0 ? cappedText.slice(0, lastNewline) : cappedText.replace(/\uFFFD+$/, '');
|
|
435
|
-
truncated = true;
|
|
414
|
+
content = capToReadBytes(content);
|
|
436
415
|
truncationReason = 'max_bytes';
|
|
437
416
|
}
|
|
438
417
|
return {
|
|
439
|
-
relpath
|
|
418
|
+
relpath,
|
|
440
419
|
content,
|
|
441
|
-
startLine:
|
|
442
|
-
endLine:
|
|
420
|
+
startLine: start,
|
|
421
|
+
endLine: start + content.split('\n').length - 1,
|
|
443
422
|
totalLines,
|
|
444
|
-
truncated,
|
|
423
|
+
truncated: truncationReason !== undefined,
|
|
445
424
|
...(truncationReason ? { truncationReason } : {}),
|
|
446
425
|
totalBytes,
|
|
447
426
|
};
|
|
448
427
|
}
|
|
449
428
|
/**
|
|
450
|
-
*
|
|
451
|
-
*
|
|
452
|
-
*
|
|
453
|
-
*
|
|
454
|
-
* Hard caps:
|
|
455
|
-
* - `MAX_GREP_MATCHES` matches returned
|
|
456
|
-
* - each match line truncated to 500 chars (guard against minified
|
|
457
|
-
* lines flooding the payload)
|
|
429
|
+
* Read a file with bounded bytes + lines. Every response includes enough metadata (`totalLines`, `totalBytes`,
|
|
430
|
+
* `truncated`, `truncationReason`) for the model to request the next slice intelligently. `startLine` / `endLine` are
|
|
431
|
+
* 1-indexed inclusive; omit both → the first `DEFAULT_READ_LINE_LIMIT` lines, up to `MAX_READ_BYTES`, whichever hits
|
|
432
|
+
* first. A checkpoint (`watch`) precedes the read and follows it; the signal aborts the read itself.
|
|
458
433
|
*/
|
|
459
|
-
export async function
|
|
434
|
+
export async function readFileExecutor(workspaceRoot, relPath, startLine, endLine, matchConfig, watch) {
|
|
435
|
+
const target = await resolveInsideWorkspace(workspaceRoot, relPath);
|
|
436
|
+
await assertReadable(target, relPath, resolveMatchConfig(matchConfig));
|
|
437
|
+
checkpoint(watch);
|
|
438
|
+
const totalBytes = await regularFileSize(target, relPath);
|
|
439
|
+
if (totalBytes > HARD_FILE_SIZE_LIMIT)
|
|
440
|
+
return tooLargeToRead(target.relpath, totalBytes);
|
|
441
|
+
const buffer = await readWhole(target.absolutePath, relPath, watch);
|
|
442
|
+
checkpoint(watch);
|
|
443
|
+
return readSlice(target.relpath, buffer, { startLine, endLine }, totalBytes);
|
|
444
|
+
}
|
|
445
|
+
function grepRegex(pattern) {
|
|
460
446
|
if (typeof pattern !== 'string' || pattern.length === 0) {
|
|
461
447
|
throw new SandboxError('INVALID_INPUT', 'grep pattern is empty', pattern);
|
|
462
448
|
}
|
|
463
|
-
let regex;
|
|
464
449
|
try {
|
|
465
|
-
|
|
450
|
+
return new RegExp(pattern);
|
|
466
451
|
}
|
|
467
452
|
catch (err) {
|
|
468
453
|
throw new SandboxError('INVALID_INPUT', `invalid regex: ${String(err)}`, pattern);
|
|
469
454
|
}
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
let startAbs = workspaceRoot;
|
|
480
|
-
let startRel = '';
|
|
481
|
-
if (typeof pathPrefix === 'string' && pathPrefix.trim().length > 0) {
|
|
482
|
-
const resolved = await resolveInsideWorkspace(workspaceRoot, pathPrefix);
|
|
483
|
-
// Top-level gate (v1.9.0 Phase 1.1, /6step Finding #1 ext): same threat
|
|
484
|
-
// as `listDirectoryExecutor` — the requested pathPrefix must be checked
|
|
485
|
-
// against user `excludeGlobs` BEFORE walking it. Without this,
|
|
486
|
-
// `grep("regex", "internal-secrets")` (where the dir is in user
|
|
487
|
-
// excludes but NOT in DEFAULT_EXCLUDE_DIRS) succeeds-with-zero-results
|
|
488
|
-
// when the dir exists vs throws NOT_FOUND when it doesn't — same
|
|
489
|
-
// existence-probe oracle. Generic message, no path leak.
|
|
490
|
-
if (resolved.relpath && isPathExcluded(resolved.relpath, config, resolved.kind)) {
|
|
491
|
-
throw new SandboxError('EXCLUDED_DIR', 'pathPrefix is excluded by configured policy', pathPrefix);
|
|
492
|
-
}
|
|
493
|
-
try {
|
|
494
|
-
const s = await stat(resolved.absolutePath);
|
|
495
|
-
if (!s.isDirectory()) {
|
|
496
|
-
throw new SandboxError('NOT_A_DIRECTORY', `pathPrefix must be a directory, got a file: ${resolved.relpath}. To search a single file, use read_file instead.`, pathPrefix);
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
catch (err) {
|
|
500
|
-
if (err instanceof SandboxError)
|
|
501
|
-
throw err;
|
|
502
|
-
throw new SandboxError('NOT_FOUND', `pathPrefix cannot be stat'd: ${String(err)}`, pathPrefix);
|
|
503
|
-
}
|
|
504
|
-
startAbs = resolved.absolutePath;
|
|
505
|
-
startRel = resolved.relpath;
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Where a grep starts: the workspace root, or `pathPrefix` resolved inside it. The prefix itself is checked against the
|
|
458
|
+
* caller's `excludeGlobs` before it is walked (a generic message: no existence oracle), and must be a directory —
|
|
459
|
+
* grep'ing a single file would silently return zero matches.
|
|
460
|
+
*/
|
|
461
|
+
async function grepStart(workspaceRoot, pathPrefix, config) {
|
|
462
|
+
if (typeof pathPrefix !== 'string' || pathPrefix.trim().length === 0) {
|
|
463
|
+
return { abs: workspaceRoot, rel: '' };
|
|
506
464
|
}
|
|
507
|
-
const
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
465
|
+
const resolved = await resolveInsideWorkspace(workspaceRoot, pathPrefix);
|
|
466
|
+
if (resolved.relpath && isPathExcluded(resolved.relpath, config, resolved.kind)) {
|
|
467
|
+
throw new SandboxError('EXCLUDED_DIR', 'pathPrefix is excluded by configured policy', pathPrefix);
|
|
468
|
+
}
|
|
469
|
+
let isDirectory;
|
|
470
|
+
try {
|
|
471
|
+
isDirectory = (await stat(resolved.absolutePath)).isDirectory();
|
|
472
|
+
}
|
|
473
|
+
catch (err) {
|
|
474
|
+
throw new SandboxError('NOT_FOUND', `pathPrefix cannot be stat'd: ${String(err)}`, pathPrefix);
|
|
475
|
+
}
|
|
476
|
+
if (!isDirectory) {
|
|
477
|
+
throw new SandboxError('NOT_A_DIRECTORY', `pathPrefix must be a directory, got a file: ${resolved.relpath}. To search a single file, use read_file instead.`, pathPrefix);
|
|
478
|
+
}
|
|
479
|
+
return { abs: resolved.absolutePath, rel: resolved.relpath };
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Add a file's matching lines. Every match is counted; at most `MAX_GREP_MATCHES` are returned, each line cut to 500
|
|
483
|
+
* characters (minified files have multi-megabyte lines). The response size is counted in UTF-8 bytes (+32 for the
|
|
484
|
+
* envelope: JSON keys, quotes, the line number), so the 500 000-byte cap holds on CJK / emoji-heavy matches.
|
|
485
|
+
*/
|
|
486
|
+
function grepLines(text, relpath, regex, found) {
|
|
487
|
+
const lines = text.split('\n');
|
|
488
|
+
for (let i = 0; i < lines.length; i++) {
|
|
489
|
+
const line = lines[i];
|
|
490
|
+
if (line === undefined || !regex.test(line))
|
|
491
|
+
continue;
|
|
492
|
+
found.total += 1;
|
|
493
|
+
if (found.matches.length >= MAX_GREP_MATCHES)
|
|
494
|
+
continue;
|
|
495
|
+
const trimmed = line.length > 500 ? `${line.slice(0, 500)}…` : line;
|
|
496
|
+
const match = { relpath, line: i + 1, text: trimmed };
|
|
497
|
+
found.matches.push(match);
|
|
498
|
+
found.respBytes +=
|
|
499
|
+
Buffer.byteLength(match.text, 'utf8') + Buffer.byteLength(match.relpath, 'utf8') + 32;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Grep one visible file: a file over the read size limit, or one that cannot be stated or read, is skipped. A read the
|
|
504
|
+
* signal aborted rejects with the signal's reason (the checkpoint after it), never as a skipped file.
|
|
505
|
+
*/
|
|
506
|
+
async function grepFile(absFile, relFile, regex, found, watch) {
|
|
507
|
+
let text;
|
|
508
|
+
try {
|
|
509
|
+
if ((await stat(absFile)).size > HARD_FILE_SIZE_LIMIT)
|
|
530
510
|
return;
|
|
531
|
-
}
|
|
532
|
-
for (const entry of entries) {
|
|
533
|
-
if (respBytes >= MAX_RESPONSE_BYTES)
|
|
534
|
-
return;
|
|
535
|
-
const childRel = currentRel ? `${currentRel}/${entry.name}` : entry.name;
|
|
536
|
-
if (entry.isDirectory()) {
|
|
537
|
-
// Honour user `excludeGlobs` dir entries in addition to defaults
|
|
538
|
-
// (mirror of `findFilesExecutor`). Critical for grep specifically:
|
|
539
|
-
// dropping the user's filter here means the regex would scan files
|
|
540
|
-
// they explicitly excluded — and grep returns matched LINES, so
|
|
541
|
-
// a regex like `password|secret|api_key` could leak content from
|
|
542
|
-
// an excluded `internal-config/` dir straight into the model.
|
|
543
|
-
if (isPathExcluded(childRel, config, 'dir'))
|
|
544
|
-
continue;
|
|
545
|
-
await walk(join(currentAbs, entry.name), childRel, depth + 1);
|
|
546
|
-
continue;
|
|
547
|
-
}
|
|
548
|
-
if (!entry.isFile())
|
|
549
|
-
continue;
|
|
550
|
-
// Unified filter — same predicate as `findFilesExecutor` and the
|
|
551
|
-
// eager scanner. Honours default + user excludes; requires an include
|
|
552
|
-
// extension or a `#!` script. v1.9.0 closes the agentic / eager divergence.
|
|
553
|
-
if (!(await isSourceFile(join(currentAbs, entry.name), childRel, config)))
|
|
554
|
-
continue;
|
|
555
|
-
// Read full file (bounded by MAX_READ_BYTES via soft stat check).
|
|
556
|
-
const absFile = join(currentAbs, entry.name);
|
|
557
|
-
let fileSize = 0;
|
|
558
|
-
try {
|
|
559
|
-
fileSize = (await stat(absFile)).size;
|
|
560
|
-
}
|
|
561
|
-
catch {
|
|
562
|
-
continue;
|
|
563
|
-
}
|
|
564
|
-
if (fileSize > 5 * MAX_READ_BYTES)
|
|
565
|
-
continue; // skip large files in grep
|
|
566
|
-
let text;
|
|
567
|
-
try {
|
|
568
|
-
text = await readFile(absFile, 'utf8');
|
|
569
|
-
}
|
|
570
|
-
catch {
|
|
571
|
-
continue;
|
|
572
|
-
}
|
|
573
|
-
const lines = text.split('\n');
|
|
574
|
-
for (let i = 0; i < lines.length; i++) {
|
|
575
|
-
const line = lines[i];
|
|
576
|
-
if (line === undefined)
|
|
577
|
-
continue;
|
|
578
|
-
if (!regex.test(line))
|
|
579
|
-
continue;
|
|
580
|
-
totalMatches += 1;
|
|
581
|
-
if (matches.length >= MAX_GREP_MATCHES)
|
|
582
|
-
continue;
|
|
583
|
-
// Cap line length in the payload to 500 chars — minified files
|
|
584
|
-
// can have multi-megabyte single lines.
|
|
585
|
-
const trimmed = line.length > 500 ? `${line.slice(0, 500)}…` : line;
|
|
586
|
-
const match = { relpath: childRel, line: i + 1, text: trimmed };
|
|
587
|
-
matches.push(match);
|
|
588
|
-
// Use UTF-8 byte length (not .length which is UTF-16 code units)
|
|
589
|
-
// so the 500k cap holds on CJK / emoji-heavy matches. `+32` is
|
|
590
|
-
// the envelope overhead (JSON keys, quotes, line-number int).
|
|
591
|
-
respBytes +=
|
|
592
|
-
Buffer.byteLength(match.text, 'utf8') + Buffer.byteLength(match.relpath, 'utf8') + 32;
|
|
593
|
-
}
|
|
594
|
-
}
|
|
511
|
+
text = await readFile(absFile, { encoding: 'utf8', signal: watch?.signal });
|
|
595
512
|
}
|
|
596
|
-
|
|
513
|
+
catch (err) {
|
|
514
|
+
checkpoint(watch);
|
|
515
|
+
logger.debug(`ask_agentic grep: skipped a file that could not be read: ${String(err)}`);
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
checkpoint(watch);
|
|
519
|
+
grepLines(text, relFile, regex, found);
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Recursive regex search. Supports a basic anchored `pathPrefix` so the model can scope to a subtree. Pattern is
|
|
523
|
+
* compiled as JS RegExp — the model can use standard metacharacters. Hard caps: `MAX_GREP_MATCHES` matches returned,
|
|
524
|
+
* each match line truncated to 500 chars, the walk ends once the response reaches `MAX_RESPONSE_BYTES`. The walk
|
|
525
|
+
* honours the caller's directory excludes, so a regex like `password|secret` reads no file in an excluded directory.
|
|
526
|
+
*/
|
|
527
|
+
export async function grepExecutor(workspaceRoot, pattern, pathPrefix, matchConfig, watch) {
|
|
528
|
+
const regex = grepRegex(pattern);
|
|
529
|
+
const config = resolveMatchConfig(matchConfig);
|
|
530
|
+
const start = await grepStart(workspaceRoot, pathPrefix, config);
|
|
531
|
+
const found = { matches: [], total: 0, respBytes: 0 };
|
|
532
|
+
const walk = treeWalk(config, watch, (absFile, relFile) => grepFile(absFile, relFile, regex, found, watch), () => found.respBytes >= MAX_RESPONSE_BYTES);
|
|
533
|
+
await walkTree(walk, start.abs, start.rel, 0);
|
|
597
534
|
return {
|
|
598
535
|
pattern,
|
|
599
536
|
pathPrefix: pathPrefix ?? null,
|
|
600
|
-
matches,
|
|
601
|
-
truncated:
|
|
602
|
-
|
|
537
|
+
matches: found.matches,
|
|
538
|
+
truncated: found.total > found.matches.length ||
|
|
539
|
+
found.respBytes >= MAX_RESPONSE_BYTES ||
|
|
540
|
+
walk.depthExceeded,
|
|
541
|
+
totalMatches: found.total,
|
|
603
542
|
};
|
|
604
543
|
}
|
|
605
544
|
/** Export knobs for tests to reference without re-declaring numeric values. */
|