@j0hanz/filesystem-mcp 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +514 -188
- package/dist/cli.js +29 -12
- package/dist/completions.js +50 -24
- package/dist/config.d.ts +3 -2
- package/dist/config.js +1 -1
- package/dist/index.js +14 -12
- package/dist/instructions.md +109 -97
- package/dist/lib/constants.js +25 -14
- package/dist/lib/errors.js +11 -6
- package/dist/lib/file-operations/common.d.ts +4 -0
- package/dist/lib/file-operations/common.js +9 -0
- package/dist/lib/file-operations/file-info.js +22 -10
- package/dist/lib/file-operations/gitignore.js +14 -11
- package/dist/lib/file-operations/glob-engine.d.ts +1 -0
- package/dist/lib/file-operations/glob-engine.js +46 -33
- package/dist/lib/file-operations/list-directory.js +31 -35
- package/dist/lib/file-operations/read-multiple-files.js +70 -62
- package/dist/lib/file-operations/search-content.js +83 -64
- package/dist/lib/file-operations/search-files.js +32 -30
- package/dist/lib/file-operations/search-worker.js +22 -12
- package/dist/lib/file-operations/tree.js +43 -34
- package/dist/lib/fs-helpers.js +61 -124
- package/dist/lib/observability.js +29 -28
- package/dist/lib/path-format.d.ts +1 -0
- package/dist/lib/path-format.js +7 -0
- package/dist/lib/path-policy.js +22 -20
- package/dist/lib/path-validation.js +13 -7
- package/dist/lib/resource-store.d.ts +2 -0
- package/dist/lib/resource-store.js +26 -5
- package/dist/lib/type-guards.d.ts +1 -0
- package/dist/lib/type-guards.js +3 -0
- package/dist/prompts.d.ts +1 -5
- package/dist/prompts.js +9 -16
- package/dist/resources.d.ts +1 -5
- package/dist/resources.js +12 -26
- package/dist/schemas.d.ts +213 -30
- package/dist/schemas.js +52 -90
- package/dist/server.js +85 -44
- package/dist/tools/apply-patch.js +24 -24
- package/dist/tools/calculate-hash.js +42 -45
- package/dist/tools/create-directory.js +18 -21
- package/dist/tools/delete-file.js +36 -39
- package/dist/tools/diff-files.js +16 -21
- package/dist/tools/edit-file.js +16 -20
- package/dist/tools/list-directory.js +25 -25
- package/dist/tools/move-file.js +18 -21
- package/dist/tools/read-multiple.js +56 -68
- package/dist/tools/read.js +27 -32
- package/dist/tools/replace-in-files.js +28 -35
- package/dist/tools/roots.js +9 -10
- package/dist/tools/search-content.js +74 -74
- package/dist/tools/search-files.js +45 -52
- package/dist/tools/shared.d.ts +44 -6
- package/dist/tools/shared.js +86 -64
- package/dist/tools/stat-many.js +45 -68
- package/dist/tools/stat.js +11 -39
- package/dist/tools/task-support.d.ts +9 -1
- package/dist/tools/task-support.js +86 -81
- package/dist/tools/tree.js +13 -30
- package/dist/tools/write-file.js +18 -21
- package/dist/tools.js +23 -18
- package/package.json +6 -7
package/dist/lib/constants.js
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import { availableParallelism } from 'node:os';
|
|
2
|
+
const TRUE_ENV_VALUES = new Set(['1', 'true', 'yes', 'y', 'on']);
|
|
3
|
+
const FALSE_ENV_VALUES = new Set(['0', 'false', 'no', 'n', 'off']);
|
|
4
|
+
const KIB = 1024;
|
|
5
|
+
const MIB = 1024 * KIB;
|
|
6
|
+
function logInvalidEnvValue(envVar, value, expected, defaultValue) {
|
|
7
|
+
console.error(`[WARNING] Invalid ${envVar} value: ${value} (must be ${expected}). Using default: ${String(defaultValue)}`);
|
|
8
|
+
}
|
|
2
9
|
// Helper for parsing environment variables (only used for configurable values)
|
|
3
10
|
function parseEnvInt(envVar, defaultValue, min, max) {
|
|
4
11
|
const value = process.env[envVar];
|
|
@@ -6,7 +13,7 @@ function parseEnvInt(envVar, defaultValue, min, max) {
|
|
|
6
13
|
return defaultValue;
|
|
7
14
|
const parsed = parseInt(value, 10);
|
|
8
15
|
if (Number.isNaN(parsed) || parsed < min || parsed > max) {
|
|
9
|
-
|
|
16
|
+
logInvalidEnvValue(envVar, value, `${String(min)}-${String(max)}`, defaultValue);
|
|
10
17
|
return defaultValue;
|
|
11
18
|
}
|
|
12
19
|
return parsed;
|
|
@@ -16,25 +23,29 @@ function parseEnvBool(envVar, defaultValue) {
|
|
|
16
23
|
if (value === undefined)
|
|
17
24
|
return defaultValue;
|
|
18
25
|
const normalized = value.trim().toLowerCase();
|
|
19
|
-
if (
|
|
26
|
+
if (TRUE_ENV_VALUES.has(normalized))
|
|
20
27
|
return true;
|
|
21
|
-
if (
|
|
28
|
+
if (FALSE_ENV_VALUES.has(normalized))
|
|
22
29
|
return false;
|
|
23
|
-
|
|
30
|
+
logInvalidEnvValue(envVar, value, 'true/false', defaultValue);
|
|
24
31
|
return defaultValue;
|
|
25
32
|
}
|
|
26
33
|
function parseEnvList(envVar) {
|
|
27
34
|
const value = process.env[envVar];
|
|
28
35
|
if (!value)
|
|
29
36
|
return [];
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
37
|
+
const entries = [];
|
|
38
|
+
for (const token of value.split(/[,\n]/u)) {
|
|
39
|
+
const trimmed = token.trim();
|
|
40
|
+
if (trimmed.length > 0) {
|
|
41
|
+
entries.push(trimmed);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return entries;
|
|
34
45
|
}
|
|
35
46
|
// Auto-tuned parallelism based on CPU cores (no env override)
|
|
36
|
-
const BYTES_PER_PARALLEL_TASK = 64 *
|
|
37
|
-
const BYTES_PER_SEARCH_WORKER = 128 *
|
|
47
|
+
const BYTES_PER_PARALLEL_TASK = 64 * MIB;
|
|
48
|
+
const BYTES_PER_SEARCH_WORKER = 128 * MIB;
|
|
38
49
|
function getAvailableMemory() {
|
|
39
50
|
if (typeof process.availableMemory !== 'function')
|
|
40
51
|
return undefined;
|
|
@@ -61,9 +72,9 @@ function getDefaultSearchWorkers() {
|
|
|
61
72
|
// Hardcoded optimal values (no env override needed)
|
|
62
73
|
export const PARALLEL_CONCURRENCY = getOptimalParallelism();
|
|
63
74
|
// Configurable via environment variables
|
|
64
|
-
export const MAX_SEARCHABLE_FILE_SIZE = parseEnvInt('MAX_SEARCH_SIZE',
|
|
65
|
-
export const MAX_TEXT_FILE_SIZE = parseEnvInt('MAX_FILE_SIZE', 10 *
|
|
66
|
-
export const DEFAULT_READ_MANY_MAX_TOTAL_SIZE = parseEnvInt('MAX_READ_MANY_TOTAL_SIZE', 512 *
|
|
75
|
+
export const MAX_SEARCHABLE_FILE_SIZE = parseEnvInt('MAX_SEARCH_SIZE', MIB, 100 * KIB, 10 * MIB);
|
|
76
|
+
export const MAX_TEXT_FILE_SIZE = parseEnvInt('MAX_FILE_SIZE', 10 * MIB, MIB, 100 * MIB);
|
|
77
|
+
export const DEFAULT_READ_MANY_MAX_TOTAL_SIZE = parseEnvInt('MAX_READ_MANY_TOTAL_SIZE', 512 * KIB, 10 * KIB, 100 * MIB);
|
|
67
78
|
export const DEFAULT_SEARCH_TIMEOUT_MS = parseEnvInt('DEFAULT_SEARCH_TIMEOUT', 5000, 100, 60000);
|
|
68
79
|
const ALLOW_SENSITIVE_FILES = parseEnvBool('FS_CONTEXT_ALLOW_SENSITIVE', false);
|
|
69
80
|
const ENV_DENYLIST = parseEnvList('FS_CONTEXT_DENYLIST');
|
|
@@ -73,7 +84,7 @@ const ENV_ALLOWLIST = parseEnvList('FS_CONTEXT_ALLOWLIST');
|
|
|
73
84
|
* Default: CPU cores (capped at 8 for optimal I/O performance).
|
|
74
85
|
* Configurable via FS_CONTEXT_SEARCH_WORKERS env var.
|
|
75
86
|
*/
|
|
76
|
-
export const SEARCH_WORKERS = parseEnvInt('FS_CONTEXT_SEARCH_WORKERS', getDefaultSearchWorkers(),
|
|
87
|
+
export const SEARCH_WORKERS = parseEnvInt('FS_CONTEXT_SEARCH_WORKERS', getDefaultSearchWorkers(), 1, 16);
|
|
77
88
|
// Hardcoded defaults
|
|
78
89
|
export const DEFAULT_MAX_DEPTH = 10;
|
|
79
90
|
export const DEFAULT_LIST_MAX_ENTRIES = 10000;
|
package/dist/lib/errors.js
CHANGED
|
@@ -27,6 +27,9 @@ function getNodeErrno(error) {
|
|
|
27
27
|
return undefined;
|
|
28
28
|
return errno;
|
|
29
29
|
}
|
|
30
|
+
function messageIncludesAny(message, patterns) {
|
|
31
|
+
return patterns.some((pattern) => message.includes(pattern));
|
|
32
|
+
}
|
|
30
33
|
const ERRNO_CODE_BY_VALUE = new Map();
|
|
31
34
|
const SYSTEM_ERROR_MAP = getSystemErrorMap();
|
|
32
35
|
for (const [name, value] of Object.entries(osConstants.errno)) {
|
|
@@ -187,23 +190,25 @@ const ERROR_SUGGESTIONS = {
|
|
|
187
190
|
[ErrorCode.E_SYMLINK_NOT_ALLOWED]: 'Symbolic links that escape allowed directories are not permitted for security reasons.',
|
|
188
191
|
[ErrorCode.E_UNKNOWN]: 'An unexpected error occurred. Check the error message for details.',
|
|
189
192
|
};
|
|
193
|
+
const NOT_FOUND_PATTERNS = ['enoent', 'no such file or directory'];
|
|
194
|
+
const PERMISSION_DENIED_PATTERNS = [
|
|
195
|
+
'permission denied',
|
|
196
|
+
'not permitted',
|
|
197
|
+
];
|
|
190
198
|
function getDirectErrorCode(error) {
|
|
191
199
|
if (error instanceof McpError) {
|
|
192
200
|
return error.code;
|
|
193
201
|
}
|
|
194
202
|
const code = getNodeErrorCodeLabel(error);
|
|
195
|
-
|
|
196
|
-
return getNodeErrorCode(code);
|
|
197
|
-
}
|
|
198
|
-
return undefined;
|
|
203
|
+
return code ? getNodeErrorCode(code) : undefined;
|
|
199
204
|
}
|
|
200
205
|
function classifyMessageError(error) {
|
|
201
206
|
const message = isNativeError(error) ? error.message : String(error);
|
|
202
207
|
const lower = message.toLowerCase();
|
|
203
|
-
if (
|
|
208
|
+
if (messageIncludesAny(lower, NOT_FOUND_PATTERNS)) {
|
|
204
209
|
return ErrorCode.E_NOT_FOUND;
|
|
205
210
|
}
|
|
206
|
-
if (
|
|
211
|
+
if (messageIncludesAny(lower, PERMISSION_DENIED_PATTERNS)) {
|
|
207
212
|
return ErrorCode.E_PERMISSION_DENIED;
|
|
208
213
|
}
|
|
209
214
|
if (lower.includes('not a directory')) {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function needsStatsForSort(sortBy) {
|
|
2
|
+
return sortBy === 'size' || sortBy === 'modified';
|
|
3
|
+
}
|
|
4
|
+
export function withOptionalStoppedReason(summary, stoppedReason) {
|
|
5
|
+
if (stoppedReason === undefined) {
|
|
6
|
+
return summary;
|
|
7
|
+
}
|
|
8
|
+
return { ...summary, stoppedReason };
|
|
9
|
+
}
|
|
@@ -15,6 +15,7 @@ const PERM_STRINGS = [
|
|
|
15
15
|
'rw-',
|
|
16
16
|
'rwx',
|
|
17
17
|
];
|
|
18
|
+
const UNKNOWN_PATH = '(unknown)';
|
|
18
19
|
function getPermissions(mode) {
|
|
19
20
|
const ownerIndex = (mode >> 6) & 0b111;
|
|
20
21
|
const groupIndex = (mode >> 3) & 0b111;
|
|
@@ -74,14 +75,21 @@ function buildEmptyResult() {
|
|
|
74
75
|
};
|
|
75
76
|
}
|
|
76
77
|
async function processFileInfo(filePath, options) {
|
|
77
|
-
const info = await getFileInfo(filePath,
|
|
78
|
-
includeMimeType: options.includeMimeType,
|
|
79
|
-
signal: options.signal,
|
|
80
|
-
});
|
|
78
|
+
const info = await getFileInfo(filePath, options);
|
|
81
79
|
return { path: filePath, info };
|
|
82
80
|
}
|
|
81
|
+
function buildIndexedPathTasks(paths) {
|
|
82
|
+
const tasks = [];
|
|
83
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
84
|
+
const filePath = paths[index];
|
|
85
|
+
if (filePath === undefined)
|
|
86
|
+
continue;
|
|
87
|
+
tasks.push({ filePath, index });
|
|
88
|
+
}
|
|
89
|
+
return tasks;
|
|
90
|
+
}
|
|
83
91
|
async function readFileInfoInParallel(paths, options) {
|
|
84
|
-
return
|
|
92
|
+
return processInParallel(buildIndexedPathTasks(paths), async ({ filePath, index }) => ({
|
|
85
93
|
index,
|
|
86
94
|
value: await processFileInfo(filePath, options),
|
|
87
95
|
}), PARALLEL_CONCURRENCY, options.signal);
|
|
@@ -94,14 +102,15 @@ function applyResults(output, results) {
|
|
|
94
102
|
function applyErrors(output, errors, paths) {
|
|
95
103
|
for (const failure of errors) {
|
|
96
104
|
const { index } = failure;
|
|
97
|
-
if (index
|
|
105
|
+
if (!isValidOutputIndex(index, output.length))
|
|
98
106
|
continue;
|
|
99
|
-
|
|
100
|
-
continue;
|
|
101
|
-
const filePath = paths[index] ?? '(unknown)';
|
|
107
|
+
const filePath = paths[index] ?? UNKNOWN_PATH;
|
|
102
108
|
output[index] = { path: filePath, error: failure.error.message };
|
|
103
109
|
}
|
|
104
110
|
}
|
|
111
|
+
function isValidOutputIndex(index, length) {
|
|
112
|
+
return index >= 0 && index < length;
|
|
113
|
+
}
|
|
105
114
|
function calculateSummary(results) {
|
|
106
115
|
let succeeded = 0;
|
|
107
116
|
let failed = 0;
|
|
@@ -125,7 +134,10 @@ function calculateSummary(results) {
|
|
|
125
134
|
export async function getMultipleFileInfo(paths, options = {}) {
|
|
126
135
|
if (paths.length === 0)
|
|
127
136
|
return buildEmptyResult();
|
|
128
|
-
const output = paths.
|
|
137
|
+
const output = new Array(paths.length);
|
|
138
|
+
for (let index = 0; index < paths.length; index += 1) {
|
|
139
|
+
output[index] = { path: paths[index] ?? UNKNOWN_PATH };
|
|
140
|
+
}
|
|
129
141
|
const { results, errors } = await readFileInfoInParallel(paths, options);
|
|
130
142
|
applyResults(output, results);
|
|
131
143
|
applyErrors(output, errors, paths);
|
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
import * as fs from 'node:fs/promises';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import ignore, {} from 'ignore';
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
import { isNodeError } from '../errors.js';
|
|
5
|
+
import { toPosixPath } from '../path-format.js';
|
|
6
|
+
function parseGitignoreLines(contents) {
|
|
7
|
+
const lines = [];
|
|
8
|
+
for (const line of contents.split(/\r?\n/u)) {
|
|
9
|
+
const trimmed = line.trim();
|
|
10
|
+
if (trimmed.length > 0) {
|
|
11
|
+
lines.push(trimmed);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return lines;
|
|
6
15
|
}
|
|
7
16
|
export async function loadRootGitignore(root, signal) {
|
|
8
17
|
const gitignorePath = path.join(root, '.gitignore');
|
|
@@ -14,26 +23,20 @@ export async function loadRootGitignore(root, signal) {
|
|
|
14
23
|
});
|
|
15
24
|
}
|
|
16
25
|
catch (error) {
|
|
17
|
-
if (
|
|
18
|
-
error !== null &&
|
|
19
|
-
'code' in error &&
|
|
20
|
-
error.code === 'ENOENT') {
|
|
26
|
+
if (isNodeError(error) && error.code === 'ENOENT') {
|
|
21
27
|
return null;
|
|
22
28
|
}
|
|
23
29
|
throw error;
|
|
24
30
|
}
|
|
25
31
|
const matcher = ignore();
|
|
26
|
-
matcher.add(contents
|
|
27
|
-
.split(/\r?\n/u)
|
|
28
|
-
.map((line) => line.trim())
|
|
29
|
-
.filter((line) => line.length > 0));
|
|
32
|
+
matcher.add(parseGitignoreLines(contents));
|
|
30
33
|
return matcher;
|
|
31
34
|
}
|
|
32
35
|
export function isIgnoredByGitignore(matcher, root, absolutePath, options = {}) {
|
|
33
36
|
const relative = path.relative(root, absolutePath);
|
|
34
37
|
if (relative.length === 0)
|
|
35
38
|
return false;
|
|
36
|
-
const normalized =
|
|
39
|
+
const normalized = toPosixPath(relative);
|
|
37
40
|
if (options.isDirectory) {
|
|
38
41
|
return matcher.ignores(normalized.endsWith('/') ? normalized : `${normalized}/`);
|
|
39
42
|
}
|
|
@@ -2,19 +2,32 @@ import * as fs from 'node:fs/promises';
|
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { glob as fsGlob } from 'node:fs/promises';
|
|
4
4
|
import { getToolContextSnapshot, publishOpsTraceEnd, publishOpsTraceError, publishOpsTraceStart, shouldPublishOpsTrace, startPerfMeasure, } from '../observability.js';
|
|
5
|
+
import { toPosixPath } from '../path-format.js';
|
|
6
|
+
import { isRecord } from '../type-guards.js';
|
|
7
|
+
export function resolveEntryType(dirent) {
|
|
8
|
+
if (dirent.isDirectory())
|
|
9
|
+
return 'directory';
|
|
10
|
+
if (dirent.isSymbolicLink())
|
|
11
|
+
return 'symlink';
|
|
12
|
+
if (dirent.isFile())
|
|
13
|
+
return 'file';
|
|
14
|
+
return 'other';
|
|
15
|
+
}
|
|
5
16
|
const GLOB_MAGIC_RE = /[*?[\]{}!]/u;
|
|
6
17
|
const DEFAULT_MAX_HIDDEN_DEPTH = 10;
|
|
18
|
+
const GLOB_BATCH_CONCURRENCY = 32;
|
|
7
19
|
const SEP = '/';
|
|
8
|
-
const WIN_SEP = '\\';
|
|
9
20
|
const DOT_CHAR_CODE = 46;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
21
|
+
const GLOB_BOOLEAN_OPTION_KEYS = [
|
|
22
|
+
'includeHidden',
|
|
23
|
+
'baseNameMatch',
|
|
24
|
+
'caseSensitiveMatch',
|
|
25
|
+
'followSymbolicLinks',
|
|
26
|
+
'onlyFiles',
|
|
27
|
+
'stats',
|
|
28
|
+
];
|
|
16
29
|
function normalizePattern(pattern, baseNameMatch) {
|
|
17
|
-
const normalized =
|
|
30
|
+
const normalized = toPosixPath(pattern);
|
|
18
31
|
if (!baseNameMatch)
|
|
19
32
|
return normalized;
|
|
20
33
|
if (normalized.includes(SEP))
|
|
@@ -22,7 +35,7 @@ function normalizePattern(pattern, baseNameMatch) {
|
|
|
22
35
|
return `**/${normalized}`;
|
|
23
36
|
}
|
|
24
37
|
function normalizeIgnorePatterns(patterns) {
|
|
25
|
-
return patterns.map(
|
|
38
|
+
return patterns.map(toPosixPath);
|
|
26
39
|
}
|
|
27
40
|
function splitPatternPrefix(normalizedPattern) {
|
|
28
41
|
if (!GLOB_MAGIC_RE.test(normalizedPattern)) {
|
|
@@ -67,12 +80,14 @@ function addDotfileCandidates(patterns, prefix, remainderSegments) {
|
|
|
67
80
|
}
|
|
68
81
|
function addGlobstarCandidates(patterns, prefix, remainder, maxDepth) {
|
|
69
82
|
const afterGlobstar = remainder.slice(3);
|
|
83
|
+
const addDotFile = afterGlobstar.length > 0 && afterGlobstar.charCodeAt(0) !== DOT_CHAR_CODE;
|
|
84
|
+
let depthPrefix = '';
|
|
70
85
|
for (let depth = 0; depth <= maxDepth; depth++) {
|
|
71
|
-
const depthPrefix = depth > 0 ? '*/'.repeat(depth) : '';
|
|
72
86
|
patterns.add(`${prefix}${depthPrefix}.*/**/${afterGlobstar}`);
|
|
73
|
-
if (
|
|
87
|
+
if (addDotFile) {
|
|
74
88
|
patterns.add(`${prefix}${depthPrefix}.${afterGlobstar}`);
|
|
75
89
|
}
|
|
90
|
+
depthPrefix += '*/';
|
|
76
91
|
}
|
|
77
92
|
}
|
|
78
93
|
function buildHiddenPatterns(normalizedPattern, maxDepth) {
|
|
@@ -111,15 +126,7 @@ function assertOptionsShape(options) {
|
|
|
111
126
|
throw new TypeError('globEntries: options.excludePatterns must contain only strings');
|
|
112
127
|
}
|
|
113
128
|
}
|
|
114
|
-
const
|
|
115
|
-
'includeHidden',
|
|
116
|
-
'baseNameMatch',
|
|
117
|
-
'caseSensitiveMatch',
|
|
118
|
-
'followSymbolicLinks',
|
|
119
|
-
'onlyFiles',
|
|
120
|
-
'stats',
|
|
121
|
-
];
|
|
122
|
-
for (const key of boolKeys) {
|
|
129
|
+
for (const key of GLOB_BOOLEAN_OPTION_KEYS) {
|
|
123
130
|
if (typeof o[key] !== 'boolean') {
|
|
124
131
|
throw new TypeError(`globEntries: options.${key} must be a boolean`);
|
|
125
132
|
}
|
|
@@ -167,12 +174,10 @@ function getRelativeDepth(relativePath) {
|
|
|
167
174
|
function isGlobDirentLike(value) {
|
|
168
175
|
if (!isRecord(value))
|
|
169
176
|
return false;
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
typeof candidate.isFile === 'function' &&
|
|
175
|
-
typeof candidate.isSymbolicLink === 'function');
|
|
177
|
+
return (typeof value.name === 'string' &&
|
|
178
|
+
typeof value.isDirectory === 'function' &&
|
|
179
|
+
typeof value.isFile === 'function' &&
|
|
180
|
+
typeof value.isSymbolicLink === 'function');
|
|
176
181
|
}
|
|
177
182
|
function resolveDirentBase(cwd, parentPath) {
|
|
178
183
|
if (!parentPath)
|
|
@@ -228,24 +233,32 @@ async function resolveStringMatch(match, cwd, maxDepth, seen, onlyFiles, followS
|
|
|
228
233
|
}
|
|
229
234
|
async function* processIterable(iterable, context) {
|
|
230
235
|
const { cwd, maxDepth, seen, onlyFiles, followSymlinks, returnStats, suppressErrors, } = context;
|
|
231
|
-
const CONCURRENCY = 32;
|
|
232
236
|
const buffer = [];
|
|
233
237
|
const flush = async function* () {
|
|
234
238
|
if (buffer.length === 0)
|
|
235
239
|
return;
|
|
236
|
-
const
|
|
240
|
+
const batchSize = buffer.length;
|
|
241
|
+
const requests = new Array(batchSize);
|
|
242
|
+
for (let index = 0; index < batchSize; index += 1) {
|
|
243
|
+
const match = buffer[index];
|
|
244
|
+
if (match === undefined) {
|
|
245
|
+
requests[index] = Promise.resolve(null);
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
requests[index] = resolveStringMatch(match, cwd, maxDepth, seen, onlyFiles, followSymlinks, returnStats, suppressErrors);
|
|
249
|
+
}
|
|
237
250
|
buffer.length = 0;
|
|
238
|
-
const results = await Promise.all(
|
|
239
|
-
for (const
|
|
240
|
-
if (
|
|
241
|
-
yield
|
|
251
|
+
const results = await Promise.all(requests);
|
|
252
|
+
for (const entry of results) {
|
|
253
|
+
if (entry !== null)
|
|
254
|
+
yield entry;
|
|
242
255
|
}
|
|
243
256
|
};
|
|
244
257
|
try {
|
|
245
258
|
for await (const match of iterable) {
|
|
246
259
|
if (typeof match === 'string') {
|
|
247
260
|
buffer.push(match);
|
|
248
|
-
if (buffer.length >=
|
|
261
|
+
if (buffer.length >= GLOB_BATCH_CONCURRENCY) {
|
|
249
262
|
yield* flush();
|
|
250
263
|
}
|
|
251
264
|
continue;
|
|
@@ -4,7 +4,8 @@ import { DEFAULT_LIST_MAX_ENTRIES, DEFAULT_MAX_DEPTH, DEFAULT_SEARCH_TIMEOUT_MS,
|
|
|
4
4
|
import { createTimedAbortSignal, processInParallel, withAbort, } from '../fs-helpers.js';
|
|
5
5
|
import { isSensitivePath } from '../path-policy.js';
|
|
6
6
|
import { isPathWithinDirectories, normalizePath, validateExistingDirectory, validateExistingPathDetailed, } from '../path-validation.js';
|
|
7
|
-
import {
|
|
7
|
+
import { needsStatsForSort, withOptionalStoppedReason } from './common.js';
|
|
8
|
+
import { globEntries, resolveEntryType } from './glob-engine.js';
|
|
8
9
|
function normalizePattern(pattern) {
|
|
9
10
|
if (!pattern || pattern.length === 0)
|
|
10
11
|
return undefined;
|
|
@@ -33,9 +34,6 @@ function sortEntries(entries, sortBy) {
|
|
|
33
34
|
}[sortBy];
|
|
34
35
|
entries.sort(compare);
|
|
35
36
|
}
|
|
36
|
-
function needsStatsForSort(sortBy) {
|
|
37
|
-
return sortBy === 'size' || sortBy === 'modified';
|
|
38
|
-
}
|
|
39
37
|
function resolveMaxDepth(normalized) {
|
|
40
38
|
if (!normalized.pattern) {
|
|
41
39
|
return 1;
|
|
@@ -67,33 +65,37 @@ async function* readDirectoryEntries(basePath, normalized, needsStats, signal) {
|
|
|
67
65
|
}
|
|
68
66
|
return;
|
|
69
67
|
}
|
|
70
|
-
const { results, errors } = await processInParallel(entries.map((
|
|
71
|
-
index
|
|
72
|
-
|
|
73
|
-
|
|
68
|
+
const { results, errors } = await processInParallel(entries.map((_, index) => index), async (index) => {
|
|
69
|
+
const candidate = entries[index];
|
|
70
|
+
if (!candidate) {
|
|
71
|
+
throw new Error(`Entry index out of range: ${String(index)}`);
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
index,
|
|
75
|
+
stats: await withAbort(fsp.lstat(candidate.entryPath), signal),
|
|
76
|
+
};
|
|
77
|
+
}, PARALLEL_CONCURRENCY, signal);
|
|
74
78
|
if (errors.length > 0) {
|
|
75
79
|
throw errors[0]?.error ?? new Error('Failed to read entry stats');
|
|
76
80
|
}
|
|
77
|
-
const statsByIndex =
|
|
81
|
+
const statsByIndex = [];
|
|
78
82
|
for (const result of results) {
|
|
79
|
-
statsByIndex
|
|
83
|
+
statsByIndex[result.index] = result.stats;
|
|
80
84
|
}
|
|
81
|
-
let index = 0;
|
|
82
|
-
|
|
83
|
-
|
|
85
|
+
for (let index = 0; index < entries.length; index += 1) {
|
|
86
|
+
const entry = entries[index];
|
|
87
|
+
if (!entry)
|
|
88
|
+
continue;
|
|
89
|
+
const stats = statsByIndex[index];
|
|
84
90
|
yield {
|
|
85
91
|
path: entry.entryPath,
|
|
86
92
|
dirent: entry.dirent,
|
|
87
93
|
...(stats ? { stats } : {}),
|
|
88
94
|
};
|
|
89
|
-
index += 1;
|
|
90
95
|
}
|
|
91
96
|
}
|
|
92
97
|
function createEntryStream(basePath, normalized, maxDepth, needsStats, signal) {
|
|
93
|
-
|
|
94
|
-
normalized.excludePatterns.length === 0 &&
|
|
95
|
-
maxDepth === 1;
|
|
96
|
-
if (canUseFastPath) {
|
|
98
|
+
if (shouldUseFastPath(normalized, maxDepth)) {
|
|
97
99
|
return readDirectoryEntries(basePath, normalized, needsStats, signal);
|
|
98
100
|
}
|
|
99
101
|
return globEntries({
|
|
@@ -109,14 +111,10 @@ function createEntryStream(basePath, normalized, maxDepth, needsStats, signal) {
|
|
|
109
111
|
stats: needsStats,
|
|
110
112
|
});
|
|
111
113
|
}
|
|
112
|
-
function
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
return 'symlink';
|
|
117
|
-
if (dirent.isFile())
|
|
118
|
-
return 'file';
|
|
119
|
-
return 'other';
|
|
114
|
+
function shouldUseFastPath(normalized, maxDepth) {
|
|
115
|
+
return (!normalized.pattern &&
|
|
116
|
+
normalized.excludePatterns.length === 0 &&
|
|
117
|
+
maxDepth === 1);
|
|
120
118
|
}
|
|
121
119
|
function resolveRelativePath(basePath, entryPath) {
|
|
122
120
|
return path.relative(basePath, entryPath) || path.basename(entryPath);
|
|
@@ -125,7 +123,7 @@ async function resolveSymlinkTarget(entryType, includeSymlinkTargets, entryPath)
|
|
|
125
123
|
if (entryType !== 'symlink' || !includeSymlinkTargets) {
|
|
126
124
|
return undefined;
|
|
127
125
|
}
|
|
128
|
-
return
|
|
126
|
+
return fsp.readlink(entryPath).catch(() => undefined);
|
|
129
127
|
}
|
|
130
128
|
function updateTotals(entryType, totals) {
|
|
131
129
|
if (entryType === 'file')
|
|
@@ -151,10 +149,10 @@ function trackSymlink(entryType, includeSymlinkTargets, counters) {
|
|
|
151
149
|
counters.symlinksNotFollowed += 1;
|
|
152
150
|
}
|
|
153
151
|
}
|
|
154
|
-
async function isEntryAccessible(entryPath, entryType,
|
|
152
|
+
async function isEntryAccessible(entryPath, entryType, basePathDirectories, signal, counters) {
|
|
155
153
|
if (entryType !== 'symlink') {
|
|
156
154
|
const normalized = normalizePath(entryPath);
|
|
157
|
-
if (!isPathWithinDirectories(normalized,
|
|
155
|
+
if (!isPathWithinDirectories(normalized, basePathDirectories)) {
|
|
158
156
|
counters.skippedInaccessible += 1;
|
|
159
157
|
return false;
|
|
160
158
|
}
|
|
@@ -198,7 +196,7 @@ async function enqueueAppendEntry(entry, entryType, ctx, pending, flushPending)
|
|
|
198
196
|
}
|
|
199
197
|
}
|
|
200
198
|
function buildSummary(entries, totals, maxDepth, truncated, stoppedReason, counters) {
|
|
201
|
-
const
|
|
199
|
+
const summary = {
|
|
202
200
|
totalEntries: entries.length,
|
|
203
201
|
entriesScanned: entries.length,
|
|
204
202
|
entriesVisible: entries.length,
|
|
@@ -209,15 +207,13 @@ function buildSummary(entries, totals, maxDepth, truncated, stoppedReason, count
|
|
|
209
207
|
skippedInaccessible: counters.skippedInaccessible,
|
|
210
208
|
symlinksNotFollowed: counters.symlinksNotFollowed,
|
|
211
209
|
};
|
|
212
|
-
return
|
|
213
|
-
...baseSummary,
|
|
214
|
-
...(stoppedReason !== undefined ? { stoppedReason } : {}),
|
|
215
|
-
};
|
|
210
|
+
return withOptionalStoppedReason(summary, stoppedReason);
|
|
216
211
|
}
|
|
217
212
|
async function collectEntries(basePath, normalized, signal, needsStats, maxDepth) {
|
|
218
213
|
const entries = [];
|
|
219
214
|
const totals = { files: 0, directories: 0 };
|
|
220
215
|
const counters = { skippedInaccessible: 0, symlinksNotFollowed: 0 };
|
|
216
|
+
const basePathDirectories = [basePath];
|
|
221
217
|
let truncated = false;
|
|
222
218
|
let stoppedReason;
|
|
223
219
|
const pending = [];
|
|
@@ -244,7 +240,7 @@ async function collectEntries(basePath, normalized, signal, needsStats, maxDepth
|
|
|
244
240
|
}
|
|
245
241
|
const entryType = resolveEntryType(entry.dirent);
|
|
246
242
|
trackSymlink(entryType, normalized.includeSymlinkTargets, counters);
|
|
247
|
-
const accessible = await isEntryAccessible(entry.path, entryType,
|
|
243
|
+
const accessible = await isEntryAccessible(entry.path, entryType, basePathDirectories, signal, counters);
|
|
248
244
|
if (!accessible) {
|
|
249
245
|
continue;
|
|
250
246
|
}
|