@descent-vtt/spec-guard 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +162 -0
- package/README.md +201 -14
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +10 -0
- package/dist/cli.js.map +1 -1
- package/dist/comments.d.ts.map +1 -1
- package/dist/comments.js +41 -19
- package/dist/comments.js.map +1 -1
- package/dist/engine.d.ts +117 -6
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +279 -150
- package/dist/engine.js.map +1 -1
- package/dist/glob.d.ts +11 -1
- package/dist/glob.d.ts.map +1 -1
- package/dist/glob.js +20 -5
- package/dist/glob.js.map +1 -1
- package/dist/graph.d.ts +124 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/graph.js +241 -0
- package/dist/graph.js.map +1 -0
- package/dist/imports.d.ts.map +1 -1
- package/dist/imports.js +10 -4
- package/dist/imports.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/layers.d.ts +72 -0
- package/dist/layers.d.ts.map +1 -0
- package/dist/layers.js +102 -0
- package/dist/layers.js.map +1 -0
- package/dist/parser.d.ts +36 -1
- package/dist/parser.d.ts.map +1 -1
- package/dist/parser.js +189 -4
- package/dist/parser.js.map +1 -1
- package/dist/polyglot.d.ts +11 -1
- package/dist/polyglot.d.ts.map +1 -1
- package/dist/polyglot.js +21 -17
- package/dist/polyglot.js.map +1 -1
- package/dist/reporter.d.ts.map +1 -1
- package/dist/reporter.js +62 -5
- package/dist/reporter.js.map +1 -1
- package/dist/runner.d.ts +25 -0
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +318 -93
- package/dist/runner.js.map +1 -1
- package/dist/text.d.ts +9 -1
- package/dist/text.d.ts.map +1 -1
- package/dist/text.js +29 -18
- package/dist/text.js.map +1 -1
- package/dist/types.d.ts +60 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/engine.js
CHANGED
|
@@ -22,19 +22,72 @@ import { spawn } from 'node:child_process';
|
|
|
22
22
|
import { promises as fs } from 'node:fs';
|
|
23
23
|
import path from 'node:path';
|
|
24
24
|
import { createCommentMask } from './comments.js';
|
|
25
|
-
import { createExcludeMatcher, createGlobMatcher, toPosix, walkFiles } from './glob.js';
|
|
26
|
-
import { isBinary, LedgerBuilder,
|
|
25
|
+
import { createExcludeMatcher, createGlobMatcher, statOrNull, toPosix, walkFiles, } from './glob.js';
|
|
26
|
+
import { isBinary, LedgerBuilder, UNCERTAIN_REASONS } from './scope.js';
|
|
27
|
+
import { lineStarts, locate } from './text.js';
|
|
27
28
|
/** Files larger than this are skipped by both engines, keeping them in sync. */
|
|
28
29
|
export const MAX_FILE_SIZE = 20 * 1024 * 1024;
|
|
30
|
+
/**
|
|
31
|
+
* One rule about file size, in one place.
|
|
32
|
+
*
|
|
33
|
+
* It used to be written twice and spelled differently each time - `size <=
|
|
34
|
+
* MAX_FILE_SIZE` where a file was admitted and `size > MAX_FILE_SIZE` where one
|
|
35
|
+
* was skipped - so the boundary was defined by two expressions that had to be
|
|
36
|
+
* kept complementary by hand. ripgrep is given the same number as
|
|
37
|
+
* `--max-filesize`, which is also inclusive, so all three now agree by
|
|
38
|
+
* construction rather than by inspection.
|
|
39
|
+
*/
|
|
40
|
+
export function withinSizeLimit(size) {
|
|
41
|
+
return size <= MAX_FILE_SIZE;
|
|
42
|
+
}
|
|
29
43
|
/** Hard cap on collected locations; the reporter only ever shows a handful. */
|
|
30
|
-
const MAX_COLLECTED_MATCHES = 500;
|
|
44
|
+
export const MAX_COLLECTED_MATCHES = 500;
|
|
31
45
|
/** Longest line snippet echoed back to the terminal. */
|
|
32
46
|
const MAX_SNIPPET_LENGTH = 200;
|
|
47
|
+
/** Most files read at once. Enough to keep a disk busy, few enough to be polite. */
|
|
48
|
+
export const MAX_CONCURRENT_READS = 16;
|
|
49
|
+
/** How many readers to start for a file list: never more than there are files. */
|
|
50
|
+
export function readConcurrency(fileCount) {
|
|
51
|
+
return Math.min(MAX_CONCURRENT_READS, fileCount);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The target list for a request that names none: the root itself.
|
|
55
|
+
*
|
|
56
|
+
* Shared by the walker and the ripgrep argv rather than written out at each,
|
|
57
|
+
* because the two must agree on what "no target" means - ripgrep given an empty
|
|
58
|
+
* path argument searches nothing at all, and a walk given one searches
|
|
59
|
+
* everything.
|
|
60
|
+
*/
|
|
61
|
+
export const ROOT_TARGETS = ['.'];
|
|
62
|
+
/** The targets a request asks for, or the root when it asks for none. */
|
|
63
|
+
function targetsOf(request) {
|
|
64
|
+
return request.targets.length > 0 ? request.targets : ROOT_TARGETS;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Total ordering on POSIX paths, ties included.
|
|
68
|
+
*
|
|
69
|
+
* Three call sites used to inline `a < b ? -1 : 1`, which is correct only
|
|
70
|
+
* because no caller can produce two equal paths - and therefore had no
|
|
71
|
+
* behaviour to test at the tie. Stating the tie makes the function total, the
|
|
72
|
+
* ordering identical, and the comparison something a test can pin down.
|
|
73
|
+
*/
|
|
74
|
+
export function comparePaths(a, b) {
|
|
75
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
76
|
+
}
|
|
33
77
|
export function escapeRegExp(value) {
|
|
34
78
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
35
79
|
}
|
|
36
|
-
|
|
37
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Trims a line to snippet length, and takes the line terminator off it.
|
|
82
|
+
*
|
|
83
|
+
* The terminator is a lone CR, not a newline: the caller slices up to the
|
|
84
|
+
* `\n`, so on a CRLF file the carriage return is the one byte that survives.
|
|
85
|
+
* This used to strip `\r?\n$`, which cannot match anything the caller passes -
|
|
86
|
+
* so every snippet from a Windows-authored file carried a CR into the report,
|
|
87
|
+
* where it returns the terminal cursor to column 0 and overwrites the line.
|
|
88
|
+
*/
|
|
89
|
+
export function truncate(value) {
|
|
90
|
+
const trimmed = value.replace(/\r$/, '');
|
|
38
91
|
return trimmed.length > MAX_SNIPPET_LENGTH ? `${trimmed.slice(0, MAX_SNIPPET_LENGTH)}…` : trimmed;
|
|
39
92
|
}
|
|
40
93
|
/**
|
|
@@ -42,34 +95,88 @@ function truncate(value) {
|
|
|
42
95
|
* files in no fixed order, so this is what makes snippet output stable.
|
|
43
96
|
*/
|
|
44
97
|
export function sortLocations(locations) {
|
|
45
|
-
return locations.sort((a, b) => (a.file === b.file ? a.line - b.line : a.file
|
|
98
|
+
return locations.sort((a, b) => (a.file === b.file ? a.line - b.line : comparePaths(a.file, b.file)));
|
|
46
99
|
}
|
|
47
100
|
function emptyTally() {
|
|
48
101
|
return { count: 0, locations: [], commentCount: 0 };
|
|
49
102
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
103
|
+
/* ------------------------------------------------------------ request keys */
|
|
104
|
+
/**
|
|
105
|
+
* When two requests are the same question, in three nested scopes.
|
|
106
|
+
*
|
|
107
|
+
* Three places need this and each used to answer it separately: whether two
|
|
108
|
+
* requests can share one walk, whether they can share one scan of that walk,
|
|
109
|
+
* and whether one result can be served from the cache instead of searched
|
|
110
|
+
* again. The lists drifted, as hand-maintained parallel lists do, and two of
|
|
111
|
+
* the gaps were live defects rather than missed cache hits:
|
|
112
|
+
*
|
|
113
|
+
* - the result cache ignored `ignoreComments`, so an assertion written
|
|
114
|
+
* `comments="include"` was answered with the comment-stripped count from
|
|
115
|
+
* the assertion above it, or the other way round depending on which ran
|
|
116
|
+
* first;
|
|
117
|
+
* - the grouping test ignored it too, so the same two assertions were merged
|
|
118
|
+
* into one pass and the second was scanned with the first's mask;
|
|
119
|
+
* - neither considered `scope`, which is one object per run today and would
|
|
120
|
+
* have done the same thing the day it stopped being one.
|
|
121
|
+
*
|
|
122
|
+
* Now each key is the one inside it plus exactly what that layer adds, so a new
|
|
123
|
+
* search option is either in `walkKey` or in `passKey` and cannot be in neither.
|
|
124
|
+
*
|
|
125
|
+
* The keys are conservative by construction: equal keys mean the same answer,
|
|
126
|
+
* while two spellings of one question merely cost a repeated search. That is
|
|
127
|
+
* the safe direction, and it is why there is no canonicalisation here - sorting
|
|
128
|
+
* the exclude set or memoising the result would buy cache hits and could not
|
|
129
|
+
* buy correctness.
|
|
130
|
+
*/
|
|
131
|
+
function walkKey(request) {
|
|
132
|
+
const { options } = request;
|
|
133
|
+
return JSON.stringify([
|
|
134
|
+
request.root,
|
|
135
|
+
request.targets,
|
|
136
|
+
options.globs,
|
|
137
|
+
options.excludeGlobs,
|
|
138
|
+
[...options.excludeFiles],
|
|
139
|
+
[...options.scope.skippedDirectories],
|
|
140
|
+
]);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* One walk, plus everything that decides what counts as a match within it.
|
|
144
|
+
*
|
|
145
|
+
* Exported because the runner groups assertions before handing them here, and
|
|
146
|
+
* it used to do so against a list of its own that was missing four of these
|
|
147
|
+
* fields. Being too coarse there costs no correctness - `sharesOnePass` checks
|
|
148
|
+
* again and declines - but it did cost the batching: a group of five where two
|
|
149
|
+
* requests disagreed was refused as a whole and run as five separate searches,
|
|
150
|
+
* rather than as the three-and-two it actually was.
|
|
151
|
+
*/
|
|
152
|
+
export function passKey(request) {
|
|
153
|
+
const { options } = request;
|
|
154
|
+
return JSON.stringify([walkKey(request), options.regex, options.word, options.ignoreCase, options.ignoreComments]);
|
|
155
|
+
}
|
|
156
|
+
/** One pass, plus the pattern it is looking for. */
|
|
157
|
+
function searchKey(request) {
|
|
158
|
+
return JSON.stringify([passKey(request), request.symbol]);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Requests that may share one pass: same walk, same matching semantics.
|
|
162
|
+
*
|
|
163
|
+
* Takes the first request rather than finding it, so there is no empty list to
|
|
164
|
+
* defend against here. `runSearches` has already answered that question, and
|
|
165
|
+
* answering it twice left a branch nothing could reach.
|
|
166
|
+
*/
|
|
167
|
+
function sharesOnePass(first, requests) {
|
|
168
|
+
const key = passKey(first);
|
|
169
|
+
return requests.every((request) => passKey(request) === key);
|
|
67
170
|
}
|
|
68
171
|
/** Runs requests through the batch API when the engine has one. */
|
|
69
172
|
export async function runSearches(engine, requests) {
|
|
70
|
-
|
|
173
|
+
const [first] = requests;
|
|
174
|
+
if (!first)
|
|
71
175
|
return [];
|
|
72
|
-
|
|
176
|
+
// No special case for a single request. Handing one request to `searchBatch`
|
|
177
|
+
// is what `search` does anyway in all three engines here, so the shortcut
|
|
178
|
+
// saved one key comparison and cost a branch that no result could depend on.
|
|
179
|
+
if (!engine.searchBatch || !sharesOnePass(first, requests)) {
|
|
73
180
|
return Promise.all(requests.map((request) => engine.search(request)));
|
|
74
181
|
}
|
|
75
182
|
return engine.searchBatch(requests);
|
|
@@ -84,15 +191,13 @@ export function resetRipgrepProbe() {
|
|
|
84
191
|
export function findRipgrep() {
|
|
85
192
|
ripgrepProbe ??= new Promise((resolve) => {
|
|
86
193
|
const binary = process.env.SPEC_GUARD_RG || 'rg';
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
catch
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
194
|
+
// No try/catch. `spawn` throws synchronously only for invalid arguments,
|
|
195
|
+
// and every argument here is a literal except the binary name, which comes
|
|
196
|
+
// from the environment and therefore cannot contain the NUL byte that is
|
|
197
|
+
// the only thing that would make a string argument invalid. The catch was
|
|
198
|
+
// unreachable, and an unreachable catch that resolves to "not installed"
|
|
199
|
+
// would have turned a real failure into a silent fallback if it ever ran.
|
|
200
|
+
const child = spawn(binary, ['--version'], { stdio: 'ignore', windowsHide: true });
|
|
96
201
|
child.once('error', () => resolve(null));
|
|
97
202
|
child.once('close', (code) => resolve(code === 0 ? binary : null));
|
|
98
203
|
});
|
|
@@ -152,7 +257,7 @@ export function buildRipgrepArgs(request, patterns = [request.symbol]) {
|
|
|
152
257
|
for (const pattern of patterns)
|
|
153
258
|
args.push('--regexp', pattern);
|
|
154
259
|
args.push('--');
|
|
155
|
-
args.push(...(request
|
|
260
|
+
args.push(...targetsOf(request));
|
|
156
261
|
return args;
|
|
157
262
|
}
|
|
158
263
|
/**
|
|
@@ -206,37 +311,68 @@ class RipgrepEngine {
|
|
|
206
311
|
windowsHide: true,
|
|
207
312
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
208
313
|
});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
314
|
+
// Bytes, decoded once at the end, rather than a string built chunk by
|
|
315
|
+
// chunk. `setEncoding('utf8')` would also have been correct, but the
|
|
316
|
+
// failure it prevents is invisible when it is missing: a path containing
|
|
317
|
+
// a multi-byte character that straddles a chunk boundary decodes to two
|
|
318
|
+
// replacement characters, and only for files large enough and named
|
|
319
|
+
// awkwardly enough to land on the seam. Holding bytes until there are no
|
|
320
|
+
// more of them makes that unrepresentable rather than merely handled.
|
|
321
|
+
const stdout = [];
|
|
322
|
+
const stderr = [];
|
|
323
|
+
child.stdout.on('data', (chunk) => stdout.push(chunk));
|
|
324
|
+
child.stderr.on('data', (chunk) => stderr.push(chunk));
|
|
215
325
|
child.once('error', reject);
|
|
216
326
|
child.once('close', (code) => {
|
|
327
|
+
const errors = Buffer.concat(stderr).toString('utf8');
|
|
217
328
|
// 0 = matches, 1 = none, anything else is a real failure.
|
|
218
329
|
if (code !== 0 && code !== 1) {
|
|
219
|
-
reject(new Error(
|
|
330
|
+
reject(new Error(ripgrepFailureMessage(code, errors)));
|
|
220
331
|
return;
|
|
221
332
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
for (const entry of stdout.split('\0')) {
|
|
227
|
-
if (entry.length === 0)
|
|
228
|
-
continue;
|
|
229
|
-
const absolutePath = path.resolve(request.root, entry);
|
|
230
|
-
if (request.options.excludeFiles.has(absolutePath))
|
|
231
|
-
continue;
|
|
232
|
-
files.push({ absolutePath, relativePath: toPosix(path.relative(request.root, absolutePath)) });
|
|
233
|
-
}
|
|
234
|
-
files.sort((a, b) => (a.relativePath < b.relativePath ? -1 : 1));
|
|
235
|
-
resolve({ files, unreadable: parseRipgrepErrors(stderr) });
|
|
333
|
+
resolve({
|
|
334
|
+
files: parseRipgrepFiles(Buffer.concat(stdout).toString('utf8'), request.root, request.options.excludeFiles),
|
|
335
|
+
unreadable: parseRipgrepErrors(errors),
|
|
336
|
+
});
|
|
236
337
|
});
|
|
237
338
|
});
|
|
238
339
|
}
|
|
239
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* What a run says when ripgrep exits with a code that is neither 0 nor 1.
|
|
343
|
+
*
|
|
344
|
+
* Separate from the spawn because it is the only part a reader ever sees, and
|
|
345
|
+
* inside the promise the only way to produce one was to break a real
|
|
346
|
+
* subprocess - so the exact text, including whether the trailing newline every
|
|
347
|
+
* command writes ends up in the middle of the sentence, was never asserted.
|
|
348
|
+
*/
|
|
349
|
+
export function ripgrepFailureMessage(code, errors) {
|
|
350
|
+
return `ripgrep exited with code ${code}${errors ? `: ${errors.trim()}` : ''}`;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Turns ripgrep's `--files-with-matches --null` output into candidate files.
|
|
354
|
+
*
|
|
355
|
+
* Split out from the spawn so that the parsing has somewhere to be tested from:
|
|
356
|
+
* inside the promise it could only be reached by running a real subprocess,
|
|
357
|
+
* which is why the separator handling, the exclusion filter and the ordering
|
|
358
|
+
* were all covered only incidentally, by whatever a real tree happened to
|
|
359
|
+
* contain.
|
|
360
|
+
*/
|
|
361
|
+
export function parseRipgrepFiles(stdout, root, excludeFiles) {
|
|
362
|
+
const files = [];
|
|
363
|
+
// --null separates paths with NUL, which is the only separator a file name
|
|
364
|
+
// cannot contain. Splitting on newlines loses files whose names contain one.
|
|
365
|
+
for (const entry of stdout.split('\0')) {
|
|
366
|
+
// The list is NUL-*terminated*, so the last split is always empty.
|
|
367
|
+
if (entry.length === 0)
|
|
368
|
+
continue;
|
|
369
|
+
const absolutePath = path.resolve(root, entry);
|
|
370
|
+
if (excludeFiles.has(absolutePath))
|
|
371
|
+
continue;
|
|
372
|
+
files.push({ absolutePath, relativePath: toPosix(path.relative(root, absolutePath)) });
|
|
373
|
+
}
|
|
374
|
+
return files.sort((a, b) => comparePaths(a.relativePath, b.relativePath));
|
|
375
|
+
}
|
|
240
376
|
/**
|
|
241
377
|
* Pulls the paths out of ripgrep's stderr.
|
|
242
378
|
*
|
|
@@ -310,7 +446,10 @@ class JavaScriptEngine {
|
|
|
310
446
|
const patterns = [...new Set(requests.map((request) => request.symbol))];
|
|
311
447
|
const regexps = new Map(patterns.map((pattern) => [pattern, buildJsRegExp(pattern, first.options)]));
|
|
312
448
|
const tallies = new Map(patterns.map((pattern) => [pattern, emptyTally()]));
|
|
313
|
-
|
|
449
|
+
// No floor of 1: an empty file list starts no readers, and a loop that
|
|
450
|
+
// never runs produces the same empty tallies as one that runs once and
|
|
451
|
+
// finds nothing. The guard was defending against an outcome it shared.
|
|
452
|
+
const concurrency = readConcurrency(files.length);
|
|
314
453
|
let cursor = 0;
|
|
315
454
|
const perFile = new Map();
|
|
316
455
|
// Per pattern, how many matches each file holds. Only files that matched
|
|
@@ -413,7 +552,15 @@ export const ANY_FILE_PROBE = { maxFiles: 0, maxBytes: 0 };
|
|
|
413
552
|
* work, so finding out a tree is "too big" costs a bounded number of stats
|
|
414
553
|
* rather than a full traversal.
|
|
415
554
|
*/
|
|
416
|
-
export async function enumerateCandidates(request, budget
|
|
555
|
+
export async function enumerateCandidates(request, budget,
|
|
556
|
+
/**
|
|
557
|
+
* Directory reader, forwarded to the walk. Exists for the same reason
|
|
558
|
+
* `walkFiles` takes one: the interesting behaviour here is what the
|
|
559
|
+
* enumeration does with a directory it *cannot* read, and there is no
|
|
560
|
+
* portable way to create one - Windows has no chmod, and a permission bit set
|
|
561
|
+
* by a test is a permission bit a failed test leaves behind.
|
|
562
|
+
*/
|
|
563
|
+
readDirectory) {
|
|
417
564
|
const matcher = createGlobMatcher(request.options.globs);
|
|
418
565
|
const excluded = createExcludeMatcher(request.options.excludeGlobs);
|
|
419
566
|
// excludeFiles is applied here rather than after the walk so that a budgeted
|
|
@@ -422,7 +569,11 @@ export async function enumerateCandidates(request, budget) {
|
|
|
422
569
|
// directory look empty.
|
|
423
570
|
const admits = (absolutePath, relativePath) => matcher(relativePath) && !excluded(relativePath) && !request.options.excludeFiles.has(absolutePath);
|
|
424
571
|
const found = new Map();
|
|
425
|
-
|
|
572
|
+
// A LedgerBuilder rather than an array and a cap of its own: the sample cap
|
|
573
|
+
// is one rule, it is applied again downstream, and a second implementation of
|
|
574
|
+
// it here was reachable only by a tree with a hundred unreadable paths in it -
|
|
575
|
+
// which is to say, by nothing the suite could build.
|
|
576
|
+
const ledger = new LedgerBuilder();
|
|
426
577
|
const note = (relativePath, reason) => {
|
|
427
578
|
// Only gaps go in the ledger. `.git` and `node_modules` are configuration,
|
|
428
579
|
// not news: they are the same on every run, they are documented, and
|
|
@@ -431,8 +582,7 @@ export async function enumerateCandidates(request, budget) {
|
|
|
431
582
|
// ever asked about files it did not skip.
|
|
432
583
|
if (!UNCERTAIN_REASONS.has(reason))
|
|
433
584
|
return;
|
|
434
|
-
|
|
435
|
-
skipped.push({ path: relativePath, reason });
|
|
585
|
+
ledger.add(relativePath, reason);
|
|
436
586
|
};
|
|
437
587
|
let bytes = 0;
|
|
438
588
|
let exceeded = false;
|
|
@@ -445,14 +595,14 @@ export async function enumerateCandidates(request, budget) {
|
|
|
445
595
|
}
|
|
446
596
|
return true;
|
|
447
597
|
};
|
|
448
|
-
outer: for (const target of request
|
|
598
|
+
outer: for (const target of targetsOf(request)) {
|
|
449
599
|
const absoluteTarget = path.resolve(request.root, target);
|
|
450
|
-
const stats = await
|
|
600
|
+
const stats = await statOrNull(absoluteTarget);
|
|
451
601
|
if (!stats)
|
|
452
602
|
continue;
|
|
453
603
|
if (stats.isFile()) {
|
|
454
604
|
const relativePath = toPosix(path.relative(request.root, absoluteTarget));
|
|
455
|
-
if (stats.size
|
|
605
|
+
if (withinSizeLimit(stats.size) && admits(absoluteTarget, relativePath)) {
|
|
456
606
|
if (!admit(absoluteTarget, relativePath, stats.size))
|
|
457
607
|
break outer;
|
|
458
608
|
}
|
|
@@ -462,9 +612,10 @@ export async function enumerateCandidates(request, budget) {
|
|
|
462
612
|
const walkOptions = {
|
|
463
613
|
scope: request.options.scope,
|
|
464
614
|
onSkip: (relativePath, reason) => note(prefix ? `${prefix}/${relativePath}` : relativePath, reason),
|
|
615
|
+
...(readDirectory ? { readDirectory } : {}),
|
|
465
616
|
};
|
|
466
617
|
for await (const file of walkFiles(absoluteTarget, walkOptions)) {
|
|
467
|
-
if (file.size
|
|
618
|
+
if (!withinSizeLimit(file.size))
|
|
468
619
|
continue;
|
|
469
620
|
const relativePath = toPosix(path.relative(request.root, file.absolutePath));
|
|
470
621
|
if (!admits(file.absolutePath, relativePath))
|
|
@@ -473,8 +624,8 @@ export async function enumerateCandidates(request, budget) {
|
|
|
473
624
|
break outer;
|
|
474
625
|
}
|
|
475
626
|
}
|
|
476
|
-
const files = [...found.values()].sort((a, b) => (a.relativePath
|
|
477
|
-
return { files, exceeded, bytes, skipped };
|
|
627
|
+
const files = [...found.values()].sort((a, b) => comparePaths(a.relativePath, b.relativePath));
|
|
628
|
+
return { files, exceeded, bytes, skipped: ledger.build().skipped };
|
|
478
629
|
}
|
|
479
630
|
/** Counts matches in one file and records per-line snippets. */
|
|
480
631
|
export function scanContent(content, relativePath, regexp,
|
|
@@ -490,26 +641,12 @@ getMask) {
|
|
|
490
641
|
let commentCount = 0;
|
|
491
642
|
const byLine = new Map();
|
|
492
643
|
// Lazily built line index: only paid for when the file actually matches.
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
lineStarts.push(i + 1);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
let low = 0;
|
|
503
|
-
let high = lineStarts.length - 1;
|
|
504
|
-
while (low < high) {
|
|
505
|
-
const mid = (low + high + 1) >> 1;
|
|
506
|
-
if (lineStarts[mid] <= index)
|
|
507
|
-
low = mid;
|
|
508
|
-
else
|
|
509
|
-
high = mid - 1;
|
|
510
|
-
}
|
|
511
|
-
return { line: low + 1, column: index - lineStarts[low] + 1 };
|
|
512
|
-
};
|
|
644
|
+
// The arithmetic itself belongs to text.ts, which every other reader of a
|
|
645
|
+
// source file already shares. This function used to carry its own copy of
|
|
646
|
+
// both halves, which is precisely the second chance to be off by one that
|
|
647
|
+
// module exists to remove.
|
|
648
|
+
let starts = null;
|
|
649
|
+
const lineOf = (index) => locate((starts ??= lineStarts(content)), index);
|
|
513
650
|
let match;
|
|
514
651
|
while ((match = regexp.exec(content)) !== null) {
|
|
515
652
|
if (getMask?.().isComment(match.index)) {
|
|
@@ -560,43 +697,35 @@ export const javascriptEngine = new JavaScriptEngine();
|
|
|
560
697
|
* The budget is set just below each crossover, so choosing JavaScript is never
|
|
561
698
|
* the slower option by more than a few milliseconds, while a small tree on
|
|
562
699
|
* Windows avoids a spawn that would cost ten times the whole search.
|
|
700
|
+
*
|
|
701
|
+
* Taking the platform as an argument rather than reading it: a decision that is
|
|
702
|
+
* explicitly about two platforms cannot be verified on one of them if the other
|
|
703
|
+
* branch is only reachable by being that other platform. Both are now asserted
|
|
704
|
+
* everywhere the suite runs.
|
|
563
705
|
*/
|
|
564
|
-
export
|
|
565
|
-
|
|
566
|
-
const MISSING_BINARY_CODES = new Set(['ENOENT', 'EACCES', 'EPERM', 'EINVAL', 'UNKNOWN']);
|
|
567
|
-
/** Exported so the code list can be asserted; EACCES and friends are not
|
|
568
|
-
* reproducible on demand from a real spawn. */
|
|
569
|
-
export function isMissingBinary(error) {
|
|
570
|
-
const code = error?.code;
|
|
571
|
-
return typeof code === 'string' && MISSING_BINARY_CODES.has(code);
|
|
706
|
+
export function smallTreeBudget(platform) {
|
|
707
|
+
return platform === 'win32' ? { maxFiles: 512, maxBytes: 1024 * 1024 } : { maxFiles: 32, maxBytes: 64 * 1024 };
|
|
572
708
|
}
|
|
709
|
+
export const SMALL_TREE_BUDGET = smallTreeBudget(process.platform);
|
|
573
710
|
/**
|
|
574
|
-
*
|
|
711
|
+
* Spawn failures that mean "this binary is not installed", not "search failed".
|
|
575
712
|
*
|
|
576
|
-
*
|
|
577
|
-
*
|
|
578
|
-
*
|
|
579
|
-
*
|
|
580
|
-
* run, so identity is both cheap and sufficient.
|
|
713
|
+
* Typed as a set of `unknown` so the lookup can be handed whatever an error
|
|
714
|
+
* object carried. The alternative - narrowing with `typeof code === 'string'`
|
|
715
|
+
* first - was a branch that could not change an answer, because a code that is
|
|
716
|
+
* not a string is not in the set either.
|
|
581
717
|
*/
|
|
582
|
-
const
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
return JSON.stringify([
|
|
594
|
-
request.root,
|
|
595
|
-
request.targets,
|
|
596
|
-
request.options.globs,
|
|
597
|
-
request.options.excludeGlobs,
|
|
598
|
-
excludeSetId(request.options.excludeFiles),
|
|
599
|
-
]);
|
|
718
|
+
const MISSING_BINARY_CODES = new Set([
|
|
719
|
+
'ENOENT',
|
|
720
|
+
'EACCES',
|
|
721
|
+
'EPERM',
|
|
722
|
+
'EINVAL',
|
|
723
|
+
'UNKNOWN',
|
|
724
|
+
]);
|
|
725
|
+
/** Exported so the code list can be asserted; EACCES and friends are not
|
|
726
|
+
* reproducible on demand from a real spawn. */
|
|
727
|
+
export function isMissingBinary(error) {
|
|
728
|
+
return MISSING_BINARY_CODES.has(error?.code);
|
|
600
729
|
}
|
|
601
730
|
/**
|
|
602
731
|
* Picks an engine per search group instead of once per run.
|
|
@@ -627,7 +756,7 @@ class AdaptiveEngine {
|
|
|
627
756
|
if (!first)
|
|
628
757
|
return [];
|
|
629
758
|
if (!this.ripgrepMissing) {
|
|
630
|
-
const key =
|
|
759
|
+
const key = walkKey(first);
|
|
631
760
|
let probe = this.enumerations.get(key);
|
|
632
761
|
if (!probe) {
|
|
633
762
|
probe = enumerateCandidates(first, SMALL_TREE_BUDGET);
|
|
@@ -637,7 +766,13 @@ class AdaptiveEngine {
|
|
|
637
766
|
if (!enumeration.exceeded) {
|
|
638
767
|
// Small tree: the walk already produced the file list, so scanning it
|
|
639
768
|
// here costs less than starting a process.
|
|
640
|
-
|
|
769
|
+
//
|
|
770
|
+
// `enumeration.skipped` is passed on, which it was not: the walk's own
|
|
771
|
+
// ledger was collected and then dropped, so a directory the walk could
|
|
772
|
+
// not read went unreported on any tree small enough to take this branch
|
|
773
|
+
// and was reported on any tree that was not. Two answers about the same
|
|
774
|
+
// repository, decided by its size - the drift ADR-0007 exists to stop.
|
|
775
|
+
return javascriptEngine.searchFiles(enumeration.files, requests, [], enumeration.skipped);
|
|
641
776
|
}
|
|
642
777
|
try {
|
|
643
778
|
const results = await this.ripgrep.searchBatch(requests);
|
|
@@ -655,16 +790,18 @@ class AdaptiveEngine {
|
|
|
655
790
|
}
|
|
656
791
|
/** Resolves the engine to use, honouring an explicit preference. */
|
|
657
792
|
export async function resolveEngine(preference = 'auto') {
|
|
793
|
+
// Every preference is compared, including the default. Leaving `auto` as the
|
|
794
|
+
// fall-through meant its name was never read, so the default could have said
|
|
795
|
+
// anything at all and every run would still have behaved identically.
|
|
658
796
|
if (preference === 'javascript')
|
|
659
797
|
return javascriptEngine;
|
|
660
|
-
if (preference === '
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
return new RipgrepEngine(binary);
|
|
798
|
+
if (preference === 'auto')
|
|
799
|
+
return new AdaptiveEngine();
|
|
800
|
+
const binary = await findRipgrep();
|
|
801
|
+
if (!binary) {
|
|
802
|
+
throw new Error('ripgrep (rg) was requested with --engine rg but is not available on PATH.');
|
|
666
803
|
}
|
|
667
|
-
return new
|
|
804
|
+
return new RipgrepEngine(binary);
|
|
668
805
|
}
|
|
669
806
|
/**
|
|
670
807
|
* Wraps an engine with a de-duplicating cache plus an automatic fallback to the
|
|
@@ -673,17 +810,7 @@ export async function resolveEngine(preference = 'auto') {
|
|
|
673
810
|
export function createCachedEngine(engine) {
|
|
674
811
|
const cache = new Map();
|
|
675
812
|
const fallbacks = [];
|
|
676
|
-
const keyOf =
|
|
677
|
-
request.root,
|
|
678
|
-
request.symbol,
|
|
679
|
-
request.targets,
|
|
680
|
-
request.options.regex,
|
|
681
|
-
request.options.word,
|
|
682
|
-
request.options.ignoreCase,
|
|
683
|
-
request.options.globs,
|
|
684
|
-
request.options.excludeGlobs,
|
|
685
|
-
[...request.options.excludeFiles].sort(),
|
|
686
|
-
]);
|
|
813
|
+
const keyOf = searchKey;
|
|
687
814
|
const recordFallback = (error) => {
|
|
688
815
|
fallbacks.push(error instanceof Error ? error.message : String(error));
|
|
689
816
|
};
|
|
@@ -712,17 +839,19 @@ export function createCachedEngine(engine) {
|
|
|
712
839
|
const keys = requests.map(keyOf);
|
|
713
840
|
const missing = requests.filter((_, index) => !cache.has(keys[index]));
|
|
714
841
|
const uncached = [...new Map(missing.map((request) => [keyOf(request), request])).values()];
|
|
715
|
-
if (uncached.length > 0)
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
842
|
+
// No `if (uncached.length > 0)` around this: `runSearches` returns an
|
|
843
|
+
// empty array without touching the engine, and a forEach over nothing
|
|
844
|
+
// does nothing. The guard could not change an outcome, which is why
|
|
845
|
+
// nothing could be written to hold it in place.
|
|
846
|
+
const pending = runSearches(engine, uncached).catch(async (error) => {
|
|
847
|
+
if (engine === javascriptEngine)
|
|
848
|
+
throw error;
|
|
849
|
+
recordFallback(error);
|
|
850
|
+
return runSearches(javascriptEngine, uncached);
|
|
851
|
+
});
|
|
852
|
+
uncached.forEach((request, index) => {
|
|
853
|
+
cache.set(keyOf(request), pending.then((results) => results[index]));
|
|
854
|
+
});
|
|
726
855
|
return Promise.all(keys.map((key) => cache.get(key)));
|
|
727
856
|
},
|
|
728
857
|
};
|