@lzehrung/codegraph-core 2.3.2 → 2.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/query-index/content.d.ts.map +1 -1
- package/dist/agent/query-index/content.js +4 -1
- package/dist/agent/query-index/content.js.map +1 -1
- package/dist/agent/query-index/update.js +2 -2
- package/dist/agent/query-index/update.js.map +1 -1
- package/dist/agent/query-index/workerPool.d.ts +22 -0
- package/dist/agent/query-index/workerPool.d.ts.map +1 -1
- package/dist/agent/query-index/workerPool.js +31 -1
- package/dist/agent/query-index/workerPool.js.map +1 -1
- package/dist/duplicate-identifier-ranges.d.ts +5 -0
- package/dist/duplicate-identifier-ranges.d.ts.map +1 -0
- package/dist/duplicate-identifier-ranges.js +1500 -0
- package/dist/duplicate-identifier-ranges.js.map +1 -0
- package/dist/duplicate-token-normalization.d.ts.map +1 -1
- package/dist/duplicate-token-normalization.js +15 -1
- package/dist/duplicate-token-normalization.js.map +1 -1
- package/dist/duplicates/unitCache.d.ts +1 -1
- package/dist/duplicates/unitCache.d.ts.map +1 -1
- package/dist/duplicates/unitCache.js +5 -1
- package/dist/duplicates/unitCache.js.map +1 -1
- package/dist/indexer/build-cache/manifest.d.ts.map +1 -1
- package/dist/indexer/build-cache/manifest.js +25 -7
- package/dist/indexer/build-cache/manifest.js.map +1 -1
- package/dist/indexer/build-cache/module-cache.d.ts.map +1 -1
- package/dist/indexer/build-cache/module-cache.js +29 -14
- package/dist/indexer/build-cache/module-cache.js.map +1 -1
- package/dist/indexer/build-cache/project-snapshot.d.ts.map +1 -1
- package/dist/indexer/build-cache/project-snapshot.js +4 -1
- package/dist/indexer/build-cache/project-snapshot.js.map +1 -1
- package/dist/indexer/build-index.d.ts.map +1 -1
- package/dist/indexer/build-index.js +61 -13
- package/dist/indexer/build-index.js.map +1 -1
- package/dist/indexer/build-workers.d.ts +5 -2
- package/dist/indexer/build-workers.d.ts.map +1 -1
- package/dist/indexer/build-workers.js +5 -3
- package/dist/indexer/build-workers.js.map +1 -1
- package/dist/indexer/finalize.d.ts +2 -1
- package/dist/indexer/finalize.d.ts.map +1 -1
- package/dist/indexer/finalize.js +3 -2
- package/dist/indexer/finalize.js.map +1 -1
- package/dist/indexer/types.d.ts +1 -1
- package/dist/indexer/types.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/util/bloomFilter.d.ts +1 -1
- package/dist/util/bloomFilter.d.ts.map +1 -1
- package/dist/util/bloomFilter.js +2 -1
- package/dist/util/bloomFilter.js.map +1 -1
- package/dist/util/git.d.ts +2 -18
- package/dist/util/git.d.ts.map +1 -1
- package/dist/util/git.js +23 -6
- package/dist/util/git.js.map +1 -1
- package/dist/util/identifiers.d.ts +0 -6
- package/dist/util/identifiers.d.ts.map +1 -1
- package/dist/util/identifiers.js +0 -6
- package/dist/util/identifiers.js.map +1 -1
- package/dist/util/projectFiles.d.ts +54 -6
- package/dist/util/projectFiles.d.ts.map +1 -1
- package/dist/util/projectFiles.js +277 -104
- package/dist/util/projectFiles.js.map +1 -1
- package/dist/worker/nativeExtractWorker.d.ts +8 -0
- package/dist/worker/nativeExtractWorker.d.ts.map +1 -1
- package/dist/worker/nativeExtractWorker.js +25 -2
- package/dist/worker/nativeExtractWorker.js.map +1 -1
- package/dist/worker/nativeWorkerPool.d.ts +2 -0
- package/dist/worker/nativeWorkerPool.d.ts.map +1 -1
- package/dist/worker/nativeWorkerPool.js +13 -1
- package/dist/worker/nativeWorkerPool.js.map +1 -1
- package/package.json +2 -2
|
@@ -11,7 +11,7 @@ export { isRelativePathInside, matchesDiscoveryGlob } from "./discoveryPath.js";
|
|
|
11
11
|
import { PROJECT_FILE_DEFINITIONS, } from "./projectFiles/definitions.js";
|
|
12
12
|
import { trimToNull } from "./projectFiles/parsers.js";
|
|
13
13
|
import { mapLimitSemaphore } from "./concurrency.js";
|
|
14
|
-
import { isGitProjectRootIgnored, isGitRepo, listGitExcludeFiles, listGitSubmoduleDirectories, listTrackedFiles, listUntrackedFiles, } from "./git.js";
|
|
14
|
+
import { getGitRepositoryRoot, isGitProjectRootIgnored, isGitRepo, listGitExcludeFiles, listGitSubmoduleDirectories, listTrackedFiles, listUntrackedFiles, } from "./git.js";
|
|
15
15
|
export const DEFAULT_PROJECT_FILE_IGNORES = [
|
|
16
16
|
"**/node_modules/**",
|
|
17
17
|
"**/.git/**",
|
|
@@ -99,6 +99,15 @@ export const DEFAULT_PROJECT_PATTERNS = [
|
|
|
99
99
|
...DEFAULT_PROJECT_MANIFESTS.map((name) => `**/${name}`),
|
|
100
100
|
];
|
|
101
101
|
const REALPATH_FILTER_CONCURRENCY = 64;
|
|
102
|
+
function createPathCheckProgressReporter(total, onProgress) {
|
|
103
|
+
onProgress?.(0, total);
|
|
104
|
+
let completed = 0;
|
|
105
|
+
return () => {
|
|
106
|
+
completed += 1;
|
|
107
|
+
if (completed === total || completed % 100 === 0)
|
|
108
|
+
onProgress?.(completed, total);
|
|
109
|
+
};
|
|
110
|
+
}
|
|
102
111
|
function isPresent(value) {
|
|
103
112
|
return value !== undefined;
|
|
104
113
|
}
|
|
@@ -188,7 +197,12 @@ function parseGitignoreRule(baseDir, rawLine) {
|
|
|
188
197
|
matches,
|
|
189
198
|
};
|
|
190
199
|
}
|
|
191
|
-
const EMPTY_GITIGNORE_INDEX = {
|
|
200
|
+
const EMPTY_GITIGNORE_INDEX = {
|
|
201
|
+
hasRules: false,
|
|
202
|
+
repositoryRoots: [],
|
|
203
|
+
repositoryAliases: [],
|
|
204
|
+
byBaseDir: new Map(),
|
|
205
|
+
};
|
|
192
206
|
/**
|
|
193
207
|
* Parse `.gitignore` rules from an already-discovered set of `.gitignore` files.
|
|
194
208
|
*
|
|
@@ -198,22 +212,36 @@ const EMPTY_GITIGNORE_INDEX = { hasRules: false, byBaseDir: new Map() };
|
|
|
198
212
|
* child negation could re-include files. Depth-first ordering also keeps the "last match
|
|
199
213
|
* wins" order deeper files depend on, and drops the locale sensitivity of `localeCompare`.
|
|
200
214
|
*/
|
|
201
|
-
async function buildGitignoreIndex(sources) {
|
|
215
|
+
async function buildGitignoreIndex(sources, repositoryRoots = [], repositoryAliases = []) {
|
|
202
216
|
const sorted = [...sources]
|
|
203
|
-
.map(({ file, baseDir }) => ({
|
|
217
|
+
.map(({ file, baseDir, repositoryRoot, priority }) => ({
|
|
218
|
+
file: normalizePath(file),
|
|
219
|
+
baseDir: normalizePath(baseDir),
|
|
220
|
+
priority: priority ?? 2,
|
|
221
|
+
...(repositoryRoot === undefined ? {} : { repositoryRoot: normalizePath(repositoryRoot) }),
|
|
222
|
+
}))
|
|
204
223
|
.sort((left, right) => {
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
224
|
+
const baseDepthDelta = left.baseDir.split("/").length - right.baseDir.split("/").length;
|
|
225
|
+
if (baseDepthDelta !== 0)
|
|
226
|
+
return baseDepthDelta;
|
|
227
|
+
const priorityDelta = left.priority - right.priority;
|
|
228
|
+
if (priorityDelta !== 0)
|
|
229
|
+
return priorityDelta;
|
|
230
|
+
if (left.file < right.file)
|
|
211
231
|
return -1;
|
|
212
|
-
return
|
|
232
|
+
return left.file > right.file ? 1 : 0;
|
|
213
233
|
});
|
|
214
|
-
const gitignoreIndex = {
|
|
215
|
-
|
|
216
|
-
|
|
234
|
+
const gitignoreIndex = {
|
|
235
|
+
hasRules: false,
|
|
236
|
+
repositoryRoots: Array.from(new Set(repositoryRoots.map(normalizePath))).sort((left, right) => right.length - left.length),
|
|
237
|
+
repositoryAliases: Array.from(new Map(repositoryAliases.map(({ path: aliasPath, repositoryRoot }) => [
|
|
238
|
+
fileIdentityKey(aliasPath),
|
|
239
|
+
{ path: normalizePath(aliasPath), repositoryRoot: normalizePath(repositoryRoot) },
|
|
240
|
+
])).values()).sort((left, right) => right.path.length - left.path.length),
|
|
241
|
+
byBaseDir: new Map(),
|
|
242
|
+
};
|
|
243
|
+
for (const { file, baseDir, repositoryRoot } of sorted) {
|
|
244
|
+
if (isIgnoredByGitignore(path.dirname(file), gitignoreIndex, true)) {
|
|
217
245
|
continue;
|
|
218
246
|
}
|
|
219
247
|
let raw;
|
|
@@ -232,7 +260,7 @@ async function buildGitignoreIndex(sources) {
|
|
|
232
260
|
if (existing)
|
|
233
261
|
existing.rules.push(rule);
|
|
234
262
|
else
|
|
235
|
-
gitignoreIndex.byBaseDir.set(baseKey, { baseDir, rules: [rule] });
|
|
263
|
+
gitignoreIndex.byBaseDir.set(baseKey, { baseDir, repositoryRoot, rules: [rule] });
|
|
236
264
|
gitignoreIndex.hasRules = true;
|
|
237
265
|
}
|
|
238
266
|
}
|
|
@@ -277,9 +305,12 @@ async function loadGitignoreIndexForRootAliases(projectRoot) {
|
|
|
277
305
|
* `path.relative` and normalization work; on the same project that repetition was the
|
|
278
306
|
* dominant remaining cost, at 2.6s of a 3.8s discovery.
|
|
279
307
|
*/
|
|
280
|
-
function isIgnoredByGitignore(absolutePath, gitignoreIndex) {
|
|
308
|
+
function isIgnoredByGitignore(absolutePath, gitignoreIndex, isDirectory = false) {
|
|
281
309
|
if (!gitignoreIndex.hasRules)
|
|
282
310
|
return false;
|
|
311
|
+
const owningRepositoryRoot = gitignoreIndex.repositoryRoots.find((root) => isFilePathWithinRoot(root, absolutePath)) ??
|
|
312
|
+
gitignoreIndex.repositoryAliases.find(({ path: aliasPath }) => isFilePathWithinRoot(aliasPath, absolutePath))
|
|
313
|
+
?.repositoryRoot;
|
|
283
314
|
const chain = [];
|
|
284
315
|
let current = normalizePath(path.dirname(absolutePath));
|
|
285
316
|
for (;;) {
|
|
@@ -294,6 +325,11 @@ function isIgnoredByGitignore(absolutePath, gitignoreIndex) {
|
|
|
294
325
|
let ignored = false;
|
|
295
326
|
for (let depth = chain.length - 1; depth >= 0; depth -= 1) {
|
|
296
327
|
const group = chain[depth];
|
|
328
|
+
if (group.repositoryRoot !== undefined &&
|
|
329
|
+
owningRepositoryRoot !== undefined &&
|
|
330
|
+
fileIdentityKey(group.repositoryRoot) !== fileIdentityKey(owningRepositoryRoot)) {
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
297
333
|
const relativePath = path.relative(group.baseDir, absolutePath);
|
|
298
334
|
if (!isRelativePathInside(relativePath)) {
|
|
299
335
|
continue;
|
|
@@ -301,10 +337,11 @@ function isIgnoredByGitignore(absolutePath, gitignoreIndex) {
|
|
|
301
337
|
const normalizedRelativePath = normalizePath(relativePath);
|
|
302
338
|
const hasSeparator = normalizedRelativePath.includes("/");
|
|
303
339
|
for (const rule of group.rules) {
|
|
304
|
-
if (rule.dirOnly && !hasSeparator) {
|
|
340
|
+
if (rule.dirOnly && !hasSeparator && !isDirectory) {
|
|
305
341
|
continue;
|
|
306
342
|
}
|
|
307
|
-
|
|
343
|
+
const matchesDirectory = isDirectory && rule.dirOnly && rule.matches(`${normalizedRelativePath}/.directory`);
|
|
344
|
+
if (rule.matches(normalizedRelativePath) || matchesDirectory) {
|
|
308
345
|
ignored = !rule.negated;
|
|
309
346
|
}
|
|
310
347
|
}
|
|
@@ -350,65 +387,91 @@ function collectCandidateAncestorDirectories(root, files) {
|
|
|
350
387
|
}
|
|
351
388
|
return directories;
|
|
352
389
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
390
|
+
async function findGitIgnoreSources(sourceRoots, files) {
|
|
391
|
+
const roots = Array.from(new Map(sourceRoots.map(({ path: sourceRoot, repositoryRoot }) => [
|
|
392
|
+
fileIdentityKey(sourceRoot),
|
|
393
|
+
{ path: normalizePath(sourceRoot), repositoryRoot: normalizePath(repositoryRoot) },
|
|
394
|
+
])).values());
|
|
395
|
+
const sources = [];
|
|
396
|
+
for (const { path: sourceRoot, repositoryRoot } of roots) {
|
|
397
|
+
const directories = collectCandidateAncestorDirectories(sourceRoot, files.filter((file) => isFilePathWithinRoot(sourceRoot, file)));
|
|
398
|
+
const present = await mapLimitSemaphore(directories, REALPATH_FILTER_CONCURRENCY, async (directory) => {
|
|
399
|
+
const candidate = normalizePath(path.join(directory, ".gitignore"));
|
|
400
|
+
try {
|
|
401
|
+
return (await fsp.stat(candidate)).isFile()
|
|
402
|
+
? { file: candidate, baseDir: directory, repositoryRoot, priority: 2 }
|
|
403
|
+
: null;
|
|
404
|
+
}
|
|
405
|
+
catch {
|
|
406
|
+
return null;
|
|
407
|
+
}
|
|
408
|
+
});
|
|
409
|
+
sources.push(...present.filter((entry) => entry !== null));
|
|
410
|
+
sources.push(...(await listGitExcludeFiles(sourceRoot)).map((source) => ({
|
|
411
|
+
...source,
|
|
412
|
+
repositoryRoot: source.baseDir,
|
|
413
|
+
})));
|
|
414
|
+
}
|
|
377
415
|
return sources;
|
|
378
416
|
}
|
|
379
417
|
/**
|
|
380
418
|
* Enumerate candidate project files through Git, or `null` when Git cannot answer.
|
|
381
419
|
*
|
|
382
|
-
* Tracked plus untracked-but-not-ignored files are
|
|
383
|
-
*
|
|
384
|
-
* to scanning the filesystem.
|
|
385
|
-
*
|
|
386
|
-
* Submodules need explicit handling or their contents vanish from the index: the
|
|
387
|
-
* superproject records only a gitlink. Tracked files come from `--recurse-submodules`,
|
|
388
|
-
* which cannot combine with `--others`, so untracked files are listed per submodule
|
|
389
|
-
* directory, including submodules nested inside submodules.
|
|
390
|
-
*
|
|
391
|
-
* A root that Git itself ignores falls back to scanning. Enumerating it through Git
|
|
392
|
-
* would return only tracked files and silently drop the rest, which would make a
|
|
393
|
-
* directory the caller named explicitly look nearly empty.
|
|
420
|
+
* Tracked plus untracked-but-not-ignored files are Git candidates. Submodule files retain
|
|
421
|
+
* their own repository boundary, and source roots cover logical and physical aliases.
|
|
394
422
|
*/
|
|
395
423
|
async function listGitCandidateFiles(root, logLevel) {
|
|
396
424
|
try {
|
|
397
|
-
|
|
425
|
+
const realRoot = normalizePath(await fsp.realpath(root));
|
|
426
|
+
const gitRoot = (await isGitRepo(root)) ? root : realRoot;
|
|
427
|
+
if (!(await isGitRepo(gitRoot)))
|
|
398
428
|
return null;
|
|
399
|
-
if (await isGitProjectRootIgnored(
|
|
429
|
+
if (await isGitProjectRootIgnored(realRoot))
|
|
430
|
+
return null;
|
|
431
|
+
const gitRepositoryRoot = await getGitRepositoryRoot(gitRoot);
|
|
432
|
+
if (!gitRepositoryRoot)
|
|
400
433
|
return null;
|
|
401
434
|
const [tracked, untracked, submoduleDirectories] = await Promise.all([
|
|
402
|
-
listTrackedFiles(
|
|
403
|
-
listUntrackedFiles(
|
|
404
|
-
listGitSubmoduleDirectories(
|
|
435
|
+
listTrackedFiles(gitRoot, { recurseSubmodules: true, ...(logLevel === undefined ? {} : { logLevel }) }),
|
|
436
|
+
listUntrackedFiles(gitRoot, { respectGitignore: true }),
|
|
437
|
+
listGitSubmoduleDirectories(gitRoot, { recurse: true }),
|
|
405
438
|
]);
|
|
406
|
-
const
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const
|
|
411
|
-
|
|
439
|
+
const submoduleRoots = await Promise.all(submoduleDirectories.map(async (logicalPath) => ({
|
|
440
|
+
logicalPath: normalizePath(logicalPath),
|
|
441
|
+
physicalPath: normalizePath(await fsp.realpath(logicalPath).catch(() => logicalPath)),
|
|
442
|
+
})));
|
|
443
|
+
const submoduleUntracked = await Promise.all(submoduleRoots.map(async ({ physicalPath }) => await listUntrackedFiles(physicalPath, { respectGitignore: true })));
|
|
444
|
+
const rawFiles = Array.from(new Set([...tracked, ...untracked, ...submoduleUntracked.flat()])).sort();
|
|
445
|
+
const needsLogicalRemap = normalizePath(gitRoot) !== normalizePath(root);
|
|
446
|
+
const files = Array.from(new Set(rawFiles.map((file) => needsLogicalRemap && isFilePathWithinRoot(realRoot, file)
|
|
447
|
+
? normalizePath(path.resolve(root, path.relative(realRoot, file)))
|
|
448
|
+
: normalizePath(file)))).sort();
|
|
449
|
+
const physicalFiles = normalizePath(root) === realRoot
|
|
450
|
+
? []
|
|
451
|
+
: (await mapLimitSemaphore(files, REALPATH_FILTER_CONCURRENCY, async (file) => {
|
|
452
|
+
try {
|
|
453
|
+
return normalizePath(await fsp.realpath(file));
|
|
454
|
+
}
|
|
455
|
+
catch {
|
|
456
|
+
return null;
|
|
457
|
+
}
|
|
458
|
+
})).filter((file) => file !== null);
|
|
459
|
+
const gitignoreRoots = [gitRepositoryRoot, ...submoduleRoots.map(({ physicalPath }) => physicalPath)].map(normalizePath);
|
|
460
|
+
const sourceRoots = [
|
|
461
|
+
{ path: root, repositoryRoot: gitRepositoryRoot },
|
|
462
|
+
{ path: realRoot, repositoryRoot: gitRepositoryRoot },
|
|
463
|
+
{ path: gitRepositoryRoot, repositoryRoot: gitRepositoryRoot },
|
|
464
|
+
...submoduleRoots.flatMap(({ logicalPath, physicalPath }) => [
|
|
465
|
+
{ path: logicalPath, repositoryRoot: physicalPath },
|
|
466
|
+
{ path: physicalPath, repositoryRoot: physicalPath },
|
|
467
|
+
]),
|
|
468
|
+
];
|
|
469
|
+
return {
|
|
470
|
+
files,
|
|
471
|
+
gitignoreFiles: await findGitIgnoreSources(sourceRoots, [...rawFiles, ...files, ...physicalFiles]),
|
|
472
|
+
gitignoreRoots,
|
|
473
|
+
gitignoreAliases: sourceRoots,
|
|
474
|
+
};
|
|
412
475
|
}
|
|
413
476
|
catch (error) {
|
|
414
477
|
logWithLevel(logLevel, "debug", `Git discovery unavailable for ${root}: ${stringifyUnknown(error)}`);
|
|
@@ -416,6 +479,12 @@ async function listGitCandidateFiles(root, logLevel) {
|
|
|
416
479
|
}
|
|
417
480
|
}
|
|
418
481
|
export async function listProjectFiles(projectRoot, patterns = DEFAULT_PROJECT_PATTERNS, options) {
|
|
482
|
+
return await listProjectFilesInternal(projectRoot, patterns, options);
|
|
483
|
+
}
|
|
484
|
+
export async function listProjectFilesWithGitCandidates(projectRoot, patterns = DEFAULT_PROJECT_PATTERNS, options) {
|
|
485
|
+
return await listProjectFilesInternal(projectRoot, patterns, options);
|
|
486
|
+
}
|
|
487
|
+
async function listProjectFilesInternal(projectRoot, patterns = DEFAULT_PROJECT_PATTERNS, options) {
|
|
419
488
|
const root = await ensureDirectoryReadable(projectRoot, "Project root");
|
|
420
489
|
const globRoot = options?.globRoot ? await ensureDirectoryReadable(options.globRoot, "Glob root") : root;
|
|
421
490
|
const includeGlobs = (options?.includeGlobs ?? []).map(normalizeGlobPattern).filter(Boolean);
|
|
@@ -448,12 +517,17 @@ export async function listProjectFiles(projectRoot, patterns = DEFAULT_PROJECT_P
|
|
|
448
517
|
// `Intermediate/` tree, and discovery took 26.1s. Git lists 4,010 in about 111ms.
|
|
449
518
|
// An aliased root or a gitignore-root override means Git's notion of the project is
|
|
450
519
|
// not this call's, so those fall back to the scan.
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
|
|
520
|
+
const attemptedGitCandidates = useGitignore && options?.gitignoreRoot === undefined && normalizePath(realRoot) === normalizePath(root);
|
|
521
|
+
const gitCandidates = attemptedGitCandidates ? await listGitCandidateFiles(root, options?.logLevel) : null;
|
|
522
|
+
// Report only when this call actually asked Git, including a null result. Callers that
|
|
523
|
+
// already enumerated candidates (build-index) can hand the same set to metadata discovery
|
|
524
|
+
// and skip a second Git spawn; omitting the callback keeps prior caller behavior.
|
|
525
|
+
if (attemptedGitCandidates) {
|
|
526
|
+
options?.onGitCandidatesDiscovered?.(gitCandidates);
|
|
527
|
+
}
|
|
454
528
|
let gitignoreIndex = EMPTY_GITIGNORE_INDEX;
|
|
455
529
|
if (useGitignore && gitCandidates) {
|
|
456
|
-
gitignoreIndex = await buildGitignoreIndex(gitCandidates.gitignoreFiles);
|
|
530
|
+
gitignoreIndex = await buildGitignoreIndex(gitCandidates.gitignoreFiles, gitCandidates.gitignoreRoots, gitCandidates.gitignoreAliases);
|
|
457
531
|
}
|
|
458
532
|
else if (useGitignore) {
|
|
459
533
|
const gitignoreRoot = options?.gitignoreRoot
|
|
@@ -489,6 +563,12 @@ export async function listProjectFiles(projectRoot, patterns = DEFAULT_PROJECT_P
|
|
|
489
563
|
ignore: translatedUserIgnoreGlobs,
|
|
490
564
|
})
|
|
491
565
|
: [];
|
|
566
|
+
const reportSourceSymlinkChecks = options?.onDiscoveryProgress
|
|
567
|
+
? (current, total) => options.onDiscoveryProgress?.({ activity: "Checking source symlinks", current, total })
|
|
568
|
+
: undefined;
|
|
569
|
+
const reportSourcePathChecks = options?.onDiscoveryProgress
|
|
570
|
+
? (current, total) => options.onDiscoveryProgress?.({ activity: "Checking source file paths", current, total })
|
|
571
|
+
: undefined;
|
|
492
572
|
const symlinkOptions = {
|
|
493
573
|
globRoot,
|
|
494
574
|
...(gitCandidates ? { candidatePaths: gitCandidates.files } : {}),
|
|
@@ -498,6 +578,7 @@ export async function listProjectFiles(projectRoot, patterns = DEFAULT_PROJECT_P
|
|
|
498
578
|
...(options?.onSymlinkDirectoriesDiscovered
|
|
499
579
|
? { onSymlinkDirectoriesDiscovered: options.onSymlinkDirectoriesDiscovered }
|
|
500
580
|
: {}),
|
|
581
|
+
...(reportSourceSymlinkChecks ? { onPathCheckProgress: reportSourceSymlinkChecks } : {}),
|
|
501
582
|
};
|
|
502
583
|
const safeSymlinkDirectories = await resolveSafeSymlinkDirectories(root, realRoot, symlinkProbeIgnoreGlobs, symlinkOptions);
|
|
503
584
|
const linkedFiles = await listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, fastGlobIgnoreGlobs, {
|
|
@@ -512,7 +593,7 @@ export async function listProjectFiles(projectRoot, patterns = DEFAULT_PROJECT_P
|
|
|
512
593
|
filterIgnoreGlobs: userIgnoreGlobs,
|
|
513
594
|
resolvedSafeSymlinkDirectories: safeSymlinkDirectories,
|
|
514
595
|
});
|
|
515
|
-
const rootSafeFiles = await filterRealPathsWithinRootEntries([...files, ...includedOverrideFiles, ...linkedFiles, ...linkedOverrideFiles], realRoot);
|
|
596
|
+
const rootSafeFiles = await filterRealPathsWithinRootEntries([...files, ...includedOverrideFiles, ...linkedFiles, ...linkedOverrideFiles], realRoot, reportSourcePathChecks);
|
|
516
597
|
const seen = new Set();
|
|
517
598
|
return rootSafeFiles
|
|
518
599
|
.map(({ path: filePath, realPath }) => ({ filePath: normalizePath(filePath), realPath }))
|
|
@@ -612,6 +693,18 @@ async function isSafeSymlinkDirectory(root, linkPath, realRoot) {
|
|
|
612
693
|
return false;
|
|
613
694
|
}
|
|
614
695
|
}
|
|
696
|
+
async function verifySafeSymlinkDirectories(root, realRoot, paths, onPathCheckProgress) {
|
|
697
|
+
const reportPathCheck = createPathCheckProgressReporter(paths.length, onPathCheckProgress);
|
|
698
|
+
const verified = await mapLimitSemaphore(Array.from(paths), REALPATH_FILTER_CONCURRENCY, async (linkPath) => {
|
|
699
|
+
try {
|
|
700
|
+
return (await isSafeSymlinkDirectory(root, linkPath, realRoot)) ? linkPath : null;
|
|
701
|
+
}
|
|
702
|
+
finally {
|
|
703
|
+
reportPathCheck();
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
return verified.filter((entry) => entry !== null);
|
|
707
|
+
}
|
|
615
708
|
/**
|
|
616
709
|
* Resolve the symlinked directories under `root` that are safe to crawl.
|
|
617
710
|
*
|
|
@@ -624,8 +717,7 @@ async function isSafeSymlinkDirectory(root, linkPath, realRoot) {
|
|
|
624
717
|
*/
|
|
625
718
|
async function resolveSafeSymlinkDirectories(root, realRoot, ignore, options) {
|
|
626
719
|
if (options.knownSymlinkDirectories !== undefined) {
|
|
627
|
-
const
|
|
628
|
-
const resolved = verified.filter((entry) => entry !== null);
|
|
720
|
+
const resolved = await verifySafeSymlinkDirectories(root, realRoot, Array.from(new Set(options.knownSymlinkDirectories)), options.onPathCheckProgress);
|
|
629
721
|
options.onSymlinkDirectoriesDiscovered?.(resolved, "known");
|
|
630
722
|
return resolved;
|
|
631
723
|
}
|
|
@@ -640,8 +732,7 @@ async function resolveSafeSymlinkDirectories(root, realRoot, ignore, options) {
|
|
|
640
732
|
return false;
|
|
641
733
|
return !ignoreMatchers.some((matcher) => matcher(relativePath));
|
|
642
734
|
});
|
|
643
|
-
const
|
|
644
|
-
const discovered = verified.filter((entry) => entry !== null);
|
|
735
|
+
const discovered = await verifySafeSymlinkDirectories(root, realRoot, candidatePaths, options.onPathCheckProgress);
|
|
645
736
|
options.onSymlinkDirectoriesDiscovered?.(discovered, "git-candidates");
|
|
646
737
|
return discovered;
|
|
647
738
|
}
|
|
@@ -654,8 +745,7 @@ async function resolveSafeSymlinkDirectories(root, realRoot, ignore, options) {
|
|
|
654
745
|
objectMode: true,
|
|
655
746
|
ignore,
|
|
656
747
|
}));
|
|
657
|
-
const
|
|
658
|
-
const discovered = candidates.filter((entry) => entry !== null);
|
|
748
|
+
const discovered = await verifySafeSymlinkDirectories(root, realRoot, entries.filter((entry) => entry.dirent.isSymbolicLink()).map((entry) => entry.path), options.onPathCheckProgress);
|
|
659
749
|
options.onSymlinkDirectoriesDiscovered?.(discovered, "filesystem");
|
|
660
750
|
return discovered;
|
|
661
751
|
}
|
|
@@ -690,7 +780,8 @@ async function listEntriesFromSafeSymlinkDirectories(root, realRoot, patterns, i
|
|
|
690
780
|
}
|
|
691
781
|
return [...filesByPath.values()];
|
|
692
782
|
}
|
|
693
|
-
async function filterRealPathsWithinRootEntries(paths, realRoot) {
|
|
783
|
+
async function filterRealPathsWithinRootEntries(paths, realRoot, onPathCheckProgress) {
|
|
784
|
+
const reportPathCheck = createPathCheckProgressReporter(paths.length, onPathCheckProgress);
|
|
694
785
|
const filtered = await mapLimitSemaphore(paths, REALPATH_FILTER_CONCURRENCY, async (filePath) => {
|
|
695
786
|
try {
|
|
696
787
|
const realPath = await fsp.realpath(filePath);
|
|
@@ -699,11 +790,14 @@ async function filterRealPathsWithinRootEntries(paths, realRoot) {
|
|
|
699
790
|
catch {
|
|
700
791
|
return null;
|
|
701
792
|
}
|
|
793
|
+
finally {
|
|
794
|
+
reportPathCheck();
|
|
795
|
+
}
|
|
702
796
|
});
|
|
703
797
|
return filtered.filter((entry) => entry !== null);
|
|
704
798
|
}
|
|
705
|
-
export async function filterRealPathsWithinRoot(paths, realRoot) {
|
|
706
|
-
const entries = await filterRealPathsWithinRootEntries(paths, realRoot);
|
|
799
|
+
export async function filterRealPathsWithinRoot(paths, realRoot, onPathCheckProgress) {
|
|
800
|
+
const entries = await filterRealPathsWithinRootEntries(paths, realRoot, onPathCheckProgress);
|
|
707
801
|
return entries.map((entry) => entry.path);
|
|
708
802
|
}
|
|
709
803
|
function toProjectGlob(pattern) {
|
|
@@ -740,33 +834,115 @@ async function buildProjectFileInfo(def, filePath) {
|
|
|
740
834
|
};
|
|
741
835
|
}
|
|
742
836
|
export async function discoverProjectFiles(projectRoot, options) {
|
|
837
|
+
return await discoverProjectFilesInternal(projectRoot, options);
|
|
838
|
+
}
|
|
839
|
+
export async function discoverProjectFilesWithGitCandidates(projectRoot, options) {
|
|
840
|
+
return await discoverProjectFilesInternal(projectRoot, options);
|
|
841
|
+
}
|
|
842
|
+
async function discoverProjectFilesInternal(projectRoot, options) {
|
|
743
843
|
const root = await ensureDirectoryReadable(projectRoot, "Project root");
|
|
744
844
|
try {
|
|
745
845
|
const realRoot = await fsp.realpath(root);
|
|
746
|
-
const allPatterns = PROJECT_FILE_DEFINITIONS.flatMap((def) => def.patterns.map(toProjectGlob));
|
|
747
|
-
const matches = await fg(allPatterns, {
|
|
748
|
-
cwd: root,
|
|
749
|
-
absolute: true,
|
|
750
|
-
dot: true,
|
|
751
|
-
followSymbolicLinks: false,
|
|
752
|
-
ignore: DEFAULT_PROJECT_FILE_IGNORES,
|
|
753
|
-
markDirectories: true,
|
|
754
|
-
onlyFiles: false,
|
|
755
|
-
});
|
|
756
|
-
const linkedMatches = await listEntriesFromSafeSymlinkDirectories(root, realRoot, allPatterns, DEFAULT_PROJECT_FILE_IGNORES, {
|
|
757
|
-
markDirectories: true,
|
|
758
|
-
onlyFiles: false,
|
|
759
|
-
...(options?.knownSymlinkDirectories !== undefined
|
|
760
|
-
? { knownSymlinkDirectories: options.knownSymlinkDirectories }
|
|
761
|
-
: {}),
|
|
762
|
-
...(options?.onSymlinkDirectoriesDiscovered
|
|
763
|
-
? { onSymlinkDirectoriesDiscovered: options.onSymlinkDirectoriesDiscovered }
|
|
764
|
-
: {}),
|
|
765
|
-
});
|
|
766
|
-
const rootSafeMatches = await filterRealPathsWithinRoot([...matches, ...linkedMatches].map((match) => (match.endsWith("/") ? match.slice(0, -1) : match)), realRoot);
|
|
767
846
|
const projectFileDefinitionMatchers = PROJECT_FILE_DEFINITIONS.map((definition) => definition.patterns.map((pattern) => pattern.includes("*") || pattern.includes("?")
|
|
768
847
|
? new RegExp("^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$")
|
|
769
848
|
: undefined));
|
|
849
|
+
const matchesDefinition = (fileName, definitionIndex) => {
|
|
850
|
+
const definition = PROJECT_FILE_DEFINITIONS[definitionIndex];
|
|
851
|
+
return definition.patterns.some((pattern, patternIndex) => {
|
|
852
|
+
const matcher = projectFileDefinitionMatchers[definitionIndex][patternIndex];
|
|
853
|
+
return matcher ? matcher.test(fileName) : pattern === fileName;
|
|
854
|
+
});
|
|
855
|
+
};
|
|
856
|
+
let gitCandidates;
|
|
857
|
+
if (options?.knownGitCandidates !== undefined) {
|
|
858
|
+
gitCandidates = options.knownGitCandidates;
|
|
859
|
+
}
|
|
860
|
+
else {
|
|
861
|
+
gitCandidates = await listGitCandidateFiles(root, options?.logLevel);
|
|
862
|
+
}
|
|
863
|
+
options?.onGitCandidatesDiscovered?.(gitCandidates);
|
|
864
|
+
const reportMetadataSymlinkChecks = options?.onDiscoveryProgress
|
|
865
|
+
? (current, total) => options.onDiscoveryProgress?.({ activity: "Checking project metadata symlinks", current, total })
|
|
866
|
+
: undefined;
|
|
867
|
+
const reportMetadataFileChecks = options?.onDiscoveryProgress
|
|
868
|
+
? (current, total) => options.onDiscoveryProgress?.({ activity: "Checking project metadata files", current, total })
|
|
869
|
+
: undefined;
|
|
870
|
+
const reportMetadataDirectoryChecks = options?.onDiscoveryProgress
|
|
871
|
+
? (current, total) => options.onDiscoveryProgress?.({ activity: "Checking project metadata directories", current, total })
|
|
872
|
+
: undefined;
|
|
873
|
+
let rootSafeMatches;
|
|
874
|
+
if (gitCandidates) {
|
|
875
|
+
const gitignoreIndex = await buildGitignoreIndex(gitCandidates.gitignoreFiles, gitCandidates.gitignoreRoots, gitCandidates.gitignoreAliases);
|
|
876
|
+
const defaultIgnoreMatchers = DEFAULT_PROJECT_FILE_IGNORES.map((globPattern) => picomatch(globPattern, { dot: true }));
|
|
877
|
+
const filterGitIgnoredEntries = (entries, directoryKeys = new Set()) => entries
|
|
878
|
+
.map(({ path: filePath, realPath }) => ({ filePath: normalizePath(filePath), realPath }))
|
|
879
|
+
.filter(({ filePath, realPath }) => {
|
|
880
|
+
const isDirectory = directoryKeys.has(fileIdentityKey(filePath));
|
|
881
|
+
const rootRelative = normalizePath(path.relative(root, filePath));
|
|
882
|
+
if (!isRelativePathInside(rootRelative))
|
|
883
|
+
return false;
|
|
884
|
+
if (defaultIgnoreMatchers.some((matcher) => matcher(rootRelative)))
|
|
885
|
+
return false;
|
|
886
|
+
if (isIgnoredByGitignore(filePath, gitignoreIndex, isDirectory))
|
|
887
|
+
return false;
|
|
888
|
+
return normalizePath(realPath) === filePath || !isIgnoredByGitignore(realPath, gitignoreIndex, isDirectory);
|
|
889
|
+
})
|
|
890
|
+
.map(({ filePath }) => filePath);
|
|
891
|
+
const symlinkOptions = {
|
|
892
|
+
candidatePaths: gitCandidates.files,
|
|
893
|
+
...(options?.knownSymlinkDirectories !== undefined
|
|
894
|
+
? { knownSymlinkDirectories: options.knownSymlinkDirectories }
|
|
895
|
+
: {}),
|
|
896
|
+
...(options?.onSymlinkDirectoriesDiscovered
|
|
897
|
+
? { onSymlinkDirectoriesDiscovered: options.onSymlinkDirectoriesDiscovered }
|
|
898
|
+
: {}),
|
|
899
|
+
...(reportMetadataSymlinkChecks ? { onPathCheckProgress: reportMetadataSymlinkChecks } : {}),
|
|
900
|
+
};
|
|
901
|
+
const safeSymlinkDirectories = await resolveSafeSymlinkDirectories(root, realRoot, DEFAULT_PROJECT_FILE_IGNORES, symlinkOptions);
|
|
902
|
+
const allPatterns = PROJECT_FILE_DEFINITIONS.flatMap((definition) => definition.patterns.map(toProjectGlob));
|
|
903
|
+
const linkedMatches = await listEntriesFromSafeSymlinkDirectories(root, realRoot, allPatterns, DEFAULT_PROJECT_FILE_IGNORES, {
|
|
904
|
+
markDirectories: true,
|
|
905
|
+
onlyFiles: false,
|
|
906
|
+
resolvedSafeSymlinkDirectories: safeSymlinkDirectories,
|
|
907
|
+
});
|
|
908
|
+
const rootSafeCandidateEntries = await filterRealPathsWithinRootEntries(gitCandidates.files, realRoot, reportMetadataFileChecks);
|
|
909
|
+
const filteredFiles = filterGitIgnoredEntries(rootSafeCandidateEntries);
|
|
910
|
+
const fileMatches = filteredFiles.filter((file) => PROJECT_FILE_DEFINITIONS.some((definition, definitionIndex) => definition.kind === "file" && matchesDefinition(path.basename(file), definitionIndex)));
|
|
911
|
+
const rawCandidateDirectories = collectCandidateAncestorDirectories(root, rootSafeCandidateEntries.map(({ path: filePath }) => filePath));
|
|
912
|
+
const candidateDirectoryKeys = new Set(rawCandidateDirectories.map(fileIdentityKey));
|
|
913
|
+
const safeSymlinkDirectoryKeys = new Set(safeSymlinkDirectories.map(fileIdentityKey));
|
|
914
|
+
const directoryMatches = [
|
|
915
|
+
...filterGitIgnoredEntries(await filterRealPathsWithinRootEntries(rawCandidateDirectories, realRoot, reportMetadataDirectoryChecks), candidateDirectoryKeys),
|
|
916
|
+
...filterGitIgnoredEntries(await filterRealPathsWithinRootEntries(safeSymlinkDirectories, realRoot, reportMetadataDirectoryChecks), safeSymlinkDirectoryKeys),
|
|
917
|
+
].filter((directory) => PROJECT_FILE_DEFINITIONS.some((definition, definitionIndex) => definition.kind === "dir" && matchesDefinition(path.basename(directory), definitionIndex)));
|
|
918
|
+
const linkedDirectoryKeys = new Set(linkedMatches.filter((match) => match.endsWith("/")).map((match) => fileIdentityKey(match.slice(0, -1))));
|
|
919
|
+
const rootSafeLinked = filterGitIgnoredEntries(await filterRealPathsWithinRootEntries(linkedMatches.map((match) => (match.endsWith("/") ? match.slice(0, -1) : match)), realRoot, reportMetadataFileChecks), linkedDirectoryKeys);
|
|
920
|
+
rootSafeMatches = [...fileMatches, ...directoryMatches, ...rootSafeLinked];
|
|
921
|
+
}
|
|
922
|
+
else {
|
|
923
|
+
const allPatterns = PROJECT_FILE_DEFINITIONS.flatMap((definition) => definition.patterns.map(toProjectGlob));
|
|
924
|
+
const matches = await fg(allPatterns, {
|
|
925
|
+
cwd: root,
|
|
926
|
+
absolute: true,
|
|
927
|
+
dot: true,
|
|
928
|
+
followSymbolicLinks: false,
|
|
929
|
+
ignore: DEFAULT_PROJECT_FILE_IGNORES,
|
|
930
|
+
markDirectories: true,
|
|
931
|
+
onlyFiles: false,
|
|
932
|
+
});
|
|
933
|
+
const linkedMatches = await listEntriesFromSafeSymlinkDirectories(root, realRoot, allPatterns, DEFAULT_PROJECT_FILE_IGNORES, {
|
|
934
|
+
markDirectories: true,
|
|
935
|
+
onlyFiles: false,
|
|
936
|
+
...(options?.knownSymlinkDirectories !== undefined
|
|
937
|
+
? { knownSymlinkDirectories: options.knownSymlinkDirectories }
|
|
938
|
+
: {}),
|
|
939
|
+
...(options?.onSymlinkDirectoriesDiscovered
|
|
940
|
+
? { onSymlinkDirectoriesDiscovered: options.onSymlinkDirectoriesDiscovered }
|
|
941
|
+
: {}),
|
|
942
|
+
...(reportMetadataSymlinkChecks ? { onPathCheckProgress: reportMetadataSymlinkChecks } : {}),
|
|
943
|
+
});
|
|
944
|
+
rootSafeMatches = await filterRealPathsWithinRoot([...matches, ...linkedMatches].map((match) => (match.endsWith("/") ? match.slice(0, -1) : match)), realRoot, reportMetadataFileChecks);
|
|
945
|
+
}
|
|
770
946
|
const entries = [];
|
|
771
947
|
const matchTasks = rootSafeMatches.map(async (cleanMatch) => {
|
|
772
948
|
const stats = await fsp.stat(cleanMatch);
|
|
@@ -778,10 +954,7 @@ export async function discoverProjectFiles(projectRoot, options) {
|
|
|
778
954
|
continue;
|
|
779
955
|
if (!isDir && def.kind !== "file")
|
|
780
956
|
continue;
|
|
781
|
-
const matchesPattern =
|
|
782
|
-
const matcher = projectFileDefinitionMatchers[definitionIndex][patternIndex];
|
|
783
|
-
return matcher ? matcher.test(fileName) : pattern === fileName;
|
|
784
|
-
});
|
|
957
|
+
const matchesPattern = matchesDefinition(fileName, definitionIndex);
|
|
785
958
|
if (matchesPattern) {
|
|
786
959
|
entries.push(await buildProjectFileInfo(def, cleanMatch));
|
|
787
960
|
}
|