@j0hanz/fs-context-mcp 2.0.7 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -12
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js.map +1 -1
- package/dist/instructions.md +27 -118
- package/dist/lib/constants.d.ts +1 -0
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +1 -0
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/file-operations/file-info.d.ts.map +1 -1
- package/dist/lib/file-operations/file-info.js +2 -0
- package/dist/lib/file-operations/file-info.js.map +1 -1
- package/dist/lib/file-operations/gitignore.d.ts +6 -0
- package/dist/lib/file-operations/gitignore.d.ts.map +1 -0
- package/dist/lib/file-operations/gitignore.js +42 -0
- package/dist/lib/file-operations/gitignore.js.map +1 -0
- package/dist/lib/file-operations/read-multiple-files.d.ts +5 -1
- package/dist/lib/file-operations/read-multiple-files.d.ts.map +1 -1
- package/dist/lib/file-operations/read-multiple-files.js +48 -17
- package/dist/lib/file-operations/read-multiple-files.js.map +1 -1
- package/dist/lib/file-operations/search-content-scan.d.ts +29 -0
- package/dist/lib/file-operations/search-content-scan.d.ts.map +1 -0
- package/dist/lib/file-operations/search-content-scan.js +468 -0
- package/dist/lib/file-operations/search-content-scan.js.map +1 -0
- package/dist/lib/file-operations/search-content-worker-api.d.ts +56 -0
- package/dist/lib/file-operations/search-content-worker-api.d.ts.map +1 -0
- package/dist/lib/file-operations/search-content-worker-api.js +239 -0
- package/dist/lib/file-operations/search-content-worker-api.js.map +1 -0
- package/dist/lib/file-operations/search-content.d.ts.map +1 -1
- package/dist/lib/file-operations/search-content.js +59 -53
- package/dist/lib/file-operations/search-content.js.map +1 -1
- package/dist/lib/file-operations/search-files.d.ts +1 -0
- package/dist/lib/file-operations/search-files.d.ts.map +1 -1
- package/dist/lib/file-operations/search-files.js +11 -2
- package/dist/lib/file-operations/search-files.js.map +1 -1
- package/dist/lib/file-operations/tree.d.ts +25 -0
- package/dist/lib/file-operations/tree.d.ts.map +1 -0
- package/dist/lib/file-operations/tree.js +172 -0
- package/dist/lib/file-operations/tree.js.map +1 -0
- package/dist/lib/fs-helpers.d.ts +5 -1
- package/dist/lib/fs-helpers.d.ts.map +1 -1
- package/dist/lib/fs-helpers.js +131 -14
- package/dist/lib/fs-helpers.js.map +1 -1
- package/dist/lib/path-validation.d.ts.map +1 -1
- package/dist/lib/path-validation.js +15 -17
- package/dist/lib/path-validation.js.map +1 -1
- package/dist/lib/resource-store.d.ts +23 -0
- package/dist/lib/resource-store.d.ts.map +1 -0
- package/dist/lib/resource-store.js +75 -0
- package/dist/lib/resource-store.js.map +1 -0
- package/dist/lib/strict-stdio-transport.d.ts +31 -0
- package/dist/lib/strict-stdio-transport.d.ts.map +1 -0
- package/dist/lib/strict-stdio-transport.js +158 -0
- package/dist/lib/strict-stdio-transport.js.map +1 -0
- package/dist/resources.d.ts +5 -0
- package/dist/resources.d.ts.map +1 -0
- package/dist/resources.js +43 -0
- package/dist/resources.js.map +1 -0
- package/dist/schemas.d.ts +78 -12
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +125 -24
- package/dist/schemas.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +52 -11
- package/dist/server.js.map +1 -1
- package/dist/tooling/tool-response.d.ts +32 -0
- package/dist/tooling/tool-response.d.ts.map +1 -0
- package/dist/tooling/tool-response.js +63 -0
- package/dist/tooling/tool-response.js.map +1 -0
- package/dist/tooling/tools-registry.d.ts +5 -0
- package/dist/tooling/tools-registry.d.ts.map +1 -0
- package/dist/tooling/tools-registry.js +532 -0
- package/dist/tooling/tools-registry.js.map +1 -0
- package/dist/tools.d.ts +12 -12
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +351 -51
- package/dist/tools.js.map +1 -1
- package/package.json +7 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as fsp from 'node:fs/promises';
|
|
2
|
-
import { MAX_TEXT_FILE_SIZE, PARALLEL_CONCURRENCY } from '../constants.js';
|
|
2
|
+
import { DEFAULT_READ_MANY_MAX_TOTAL_SIZE, MAX_TEXT_FILE_SIZE, PARALLEL_CONCURRENCY, } from '../constants.js';
|
|
3
3
|
import { processInParallel, readFile, readFileWithStats, withAbort, } from '../fs-helpers.js';
|
|
4
4
|
import { validateExistingPath } from '../path-validation.js';
|
|
5
5
|
function buildReadOptions(options) {
|
|
@@ -10,6 +10,12 @@ function buildReadOptions(options) {
|
|
|
10
10
|
if (options.head !== undefined) {
|
|
11
11
|
readOptions.head = options.head;
|
|
12
12
|
}
|
|
13
|
+
if (options.startLine !== undefined) {
|
|
14
|
+
readOptions.startLine = options.startLine;
|
|
15
|
+
}
|
|
16
|
+
if (options.endLine !== undefined) {
|
|
17
|
+
readOptions.endLine = options.endLine;
|
|
18
|
+
}
|
|
13
19
|
return readOptions;
|
|
14
20
|
}
|
|
15
21
|
function buildReadMultipleResult(filePath, result) {
|
|
@@ -23,6 +29,10 @@ function buildReadMultipleResult(filePath, result) {
|
|
|
23
29
|
value.totalLines = result.totalLines;
|
|
24
30
|
if (result.head !== undefined)
|
|
25
31
|
value.head = result.head;
|
|
32
|
+
if (result.startLine !== undefined)
|
|
33
|
+
value.startLine = result.startLine;
|
|
34
|
+
if (result.endLine !== undefined)
|
|
35
|
+
value.endLine = result.endLine;
|
|
26
36
|
if (result.linesRead !== undefined)
|
|
27
37
|
value.linesRead = result.linesRead;
|
|
28
38
|
if (result.hasMoreLines !== undefined) {
|
|
@@ -47,7 +57,7 @@ async function readSingleFile(task, options, signal) {
|
|
|
47
57
|
};
|
|
48
58
|
}
|
|
49
59
|
function isPartialRead(options) {
|
|
50
|
-
return options.head !== undefined;
|
|
60
|
+
return options.head !== undefined || options.startLine !== undefined;
|
|
51
61
|
}
|
|
52
62
|
async function readFilesInParallel(filesToProcess, options, signal) {
|
|
53
63
|
return await processInParallel(filesToProcess, async (task) => readSingleFile(task, options, signal), PARALLEL_CONCURRENCY, signal);
|
|
@@ -56,11 +66,17 @@ function normalizeReadMultipleOptions(options) {
|
|
|
56
66
|
const normalized = {
|
|
57
67
|
encoding: options.encoding ?? 'utf-8',
|
|
58
68
|
maxSize: Math.min(options.maxSize ?? MAX_TEXT_FILE_SIZE, MAX_TEXT_FILE_SIZE),
|
|
59
|
-
maxTotalSize: options.maxTotalSize ??
|
|
69
|
+
maxTotalSize: options.maxTotalSize ?? DEFAULT_READ_MANY_MAX_TOTAL_SIZE,
|
|
60
70
|
};
|
|
61
71
|
if (options.head !== undefined) {
|
|
62
72
|
normalized.head = options.head;
|
|
63
73
|
}
|
|
74
|
+
if (options.startLine !== undefined) {
|
|
75
|
+
normalized.startLine = options.startLine;
|
|
76
|
+
}
|
|
77
|
+
if (options.endLine !== undefined) {
|
|
78
|
+
normalized.endLine = options.endLine;
|
|
79
|
+
}
|
|
64
80
|
return normalized;
|
|
65
81
|
}
|
|
66
82
|
function resolveNormalizedOptions(_filePaths, options) {
|
|
@@ -82,29 +98,44 @@ function estimatePartialSize(stats, maxSize) {
|
|
|
82
98
|
function estimateFullSize(stats) {
|
|
83
99
|
return stats.size;
|
|
84
100
|
}
|
|
85
|
-
function
|
|
101
|
+
function markRemainingSkipped(startIndex, total, skippedBudget) {
|
|
102
|
+
for (let index = startIndex; index < total; index += 1) {
|
|
103
|
+
skippedBudget.add(index);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function tryValidateFile(filePath, index, signal) {
|
|
107
|
+
try {
|
|
108
|
+
return await validateFile(filePath, index, signal);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
async function collectFileBudget(filePaths, partialRead, maxTotalSize, maxSize, signal) {
|
|
86
115
|
const skippedBudget = new Set();
|
|
87
116
|
const validated = new Map();
|
|
117
|
+
const estimateSize = partialRead
|
|
118
|
+
? (stats) => estimatePartialSize(stats, maxSize)
|
|
119
|
+
: estimateFullSize;
|
|
88
120
|
let totalSize = 0;
|
|
89
|
-
for (
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
121
|
+
for (let index = 0; index < filePaths.length; index += 1) {
|
|
122
|
+
const filePath = filePaths[index];
|
|
123
|
+
if (!filePath)
|
|
124
|
+
continue;
|
|
125
|
+
const info = await tryValidateFile(filePath, index, signal);
|
|
126
|
+
if (!info)
|
|
94
127
|
continue;
|
|
128
|
+
validated.set(index, info);
|
|
129
|
+
const estimatedSize = estimateSize(info.stats);
|
|
130
|
+
if (totalSize + estimatedSize > maxTotalSize) {
|
|
131
|
+
skippedBudget.add(index);
|
|
132
|
+
markRemainingSkipped(index + 1, filePaths.length, skippedBudget);
|
|
133
|
+
return { skippedBudget, validated };
|
|
95
134
|
}
|
|
96
135
|
totalSize += estimatedSize;
|
|
97
136
|
}
|
|
98
137
|
return { skippedBudget, validated };
|
|
99
138
|
}
|
|
100
|
-
async function collectFileBudget(filePaths, partialRead, maxTotalSize, maxSize, signal) {
|
|
101
|
-
const { results } = await processInParallel(filePaths.map((filePath, index) => ({ filePath, index })), async ({ filePath, index }) => validateFile(filePath, index, signal), PARALLEL_CONCURRENCY, signal);
|
|
102
|
-
const orderedResults = [...results].sort((a, b) => a.index - b.index);
|
|
103
|
-
const estimateSize = partialRead
|
|
104
|
-
? (stats) => estimatePartialSize(stats, maxSize)
|
|
105
|
-
: estimateFullSize;
|
|
106
|
-
return applyBudget(orderedResults, estimateSize, maxTotalSize);
|
|
107
|
-
}
|
|
108
139
|
function buildOutput(filePaths) {
|
|
109
140
|
return filePaths.map((filePath) => ({ path: filePath }));
|
|
110
141
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read-multiple-files.js","sourceRoot":"","sources":["../../../src/lib/file-operations/read-multiple-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,kBAAkB,CAAC;AAGxC,OAAO,
|
|
1
|
+
{"version":3,"file":"read-multiple-files.js","sourceRoot":"","sources":["../../../src/lib/file-operations/read-multiple-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,kBAAkB,CAAC;AAGxC,OAAO,EACL,gCAAgC,EAChC,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,iBAAiB,EACjB,QAAQ,EACR,iBAAiB,EACjB,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AA0C7D,SAAS,gBAAgB,CAAC,OAAsC;IAO9D,MAAM,WAAW,GAMb;QACF,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,WAAW,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAClC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,WAAW,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAC5C,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,WAAW,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACxC,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAgB,EAChB,MAA4C;IAE5C,MAAM,KAAK,GAAuB;QAChC,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC;IACF,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;QAAE,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IAC1E,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACxD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;QAAE,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IACvE,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;QAAE,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IACjE,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;QAAE,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IACvE,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,IAAkB,EAClB,OAAsC,EACtC,MAAoB;IAEpB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnD,MAAM,WAAW,GAAmC;QAClD,GAAG,gBAAgB,CAAC,OAAO,CAAC;KAC7B,CAAC;IACF,IAAI,MAAM,EAAE,CAAC;QACX,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;IAC9B,CAAC;IACD,MAAM,MAAM,GACV,SAAS,IAAI,KAAK;QAChB,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,CAAC;QAClE,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IAE5C,OAAO;QACL,KAAK;QACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,EAAE,MAAM,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,OAAsC;IAC3D,OAAO,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,cAA8B,EAC9B,OAAsC,EACtC,MAAoB;IAKpB,OAAO,MAAM,iBAAiB,CAC5B,cAAc,EACd,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,EACrD,oBAAoB,EACpB,MAAM,CACP,CAAC;AACJ,CAAC;AAED,SAAS,4BAA4B,CACnC,OAA4B;IAE5B,MAAM,UAAU,GAAkC;QAChD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,OAAO;QACrC,OAAO,EAAE,IAAI,CAAC,GAAG,CACf,OAAO,CAAC,OAAO,IAAI,kBAAkB,EACrC,kBAAkB,CACnB;QACD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,gCAAgC;KACvE,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAC3C,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACvC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,wBAAwB,CAC/B,UAA6B,EAC7B,OAA4B;IAE5B,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACpC,MAAM,QAAQ,GAGV,EAAE,UAAU,EAAE,4BAA4B,CAAC,IAAI,CAAC,EAAE,CAAC;IACvD,IAAI,MAAM,EAAE,CAAC;QACX,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AASD,KAAK,UAAU,YAAY,CACzB,QAAgB,EAChB,KAAa,EACb,MAAoB;IAEpB,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAC3D,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAY,EAAE,OAAe;IACxD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAY;IACpC,OAAO,KAAK,CAAC,IAAI,CAAC;AACpB,CAAC;AAED,SAAS,oBAAoB,CAC3B,UAAkB,EAClB,KAAa,EACb,aAA0B;IAE1B,KAAK,IAAI,KAAK,GAAG,UAAU,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACvD,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,QAAgB,EAChB,KAAa,EACb,MAAoB;IAEpB,IAAI,CAAC;QACH,OAAO,MAAM,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAA4B,EAC5B,WAAoB,EACpB,YAAoB,EACpB,OAAe,EACf,MAAoB;IAKpB,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,GAAG,EAA6B,CAAC;IACvD,MAAM,YAAY,GAAG,WAAW;QAC9B,CAAC,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC;QACvD,CAAC,CAAC,gBAAgB,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAE3B,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,SAAS,GAAG,aAAa,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,oBAAoB,CAAC,KAAK,GAAG,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACjE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;QACtC,CAAC;QAED,SAAS,IAAI,aAAa,CAAC;IAC7B,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;AACtC,CAAC;AAED,SAAS,WAAW,CAAC,SAA4B;IAC/C,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CACnB,MAA4B,EAC5B,OAAuD;IAEvD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAClB,MAA4B,EAC5B,MAAyC,EACzC,cAAmC,EACnC,SAA4B;IAE5B,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,aAAa,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC1C,IAAI,aAAa,GAAG,CAAC;YAAE,SAAS;QAChC,MAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC;QACzD,MAAM,CAAC,aAAa,CAAC,GAAG;YACtB,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;SAC7B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,SAA4B,EAC5B,SAMC,EACD,aAA0B;IAE1B,OAAO,SAAS;SACb,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,MAAM;YACX,CAAC,CAAC;gBACE,QAAQ;gBACR,KAAK;gBACL,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB;YACH,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,kBAAkB,CACzB,MAA4B,EAC5B,aAA0B,EAC1B,SAA4B,EAC5B,YAAoB;IAEpB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,CAAC,KAAK,CAAC,GAAG;YACd,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,+DAA+D,YAAY,SAAS;SAC5F,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,SAA4B,EAC5B,UAA+B,EAAE;IAEjC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,wBAAwB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAE5E,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IAC9C,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,iBAAiB,CAC1D,SAAS,EACT,WAAW,EACX,UAAU,CAAC,YAAY,EACvB,UAAU,CAAC,OAAO,EAClB,MAAM,CACP,CAAC;IAEF,MAAM,cAAc,GAAG,mBAAmB,CACxC,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IAEF,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,mBAAmB,CACnD,cAAc,EACd,UAAU,EACV,MAAM,CACP,CAAC;IAEF,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;IACvD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IAE9E,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { ContentMatch, SearchContentResult } from '../../config.js';
|
|
2
|
+
import { type MatcherOptions, type ScanFileOptions } from './search-content-worker-api.js';
|
|
3
|
+
export interface ResolvedFile {
|
|
4
|
+
resolvedPath: string;
|
|
5
|
+
requestedPath: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ScanSummary {
|
|
8
|
+
filesScanned: number;
|
|
9
|
+
filesMatched: number;
|
|
10
|
+
skippedTooLarge: number;
|
|
11
|
+
skippedBinary: number;
|
|
12
|
+
skippedInaccessible: number;
|
|
13
|
+
truncated: boolean;
|
|
14
|
+
stoppedReason: SearchContentResult['summary']['stoppedReason'];
|
|
15
|
+
}
|
|
16
|
+
interface ScanResultLike {
|
|
17
|
+
matches: readonly ContentMatch[];
|
|
18
|
+
matched: boolean;
|
|
19
|
+
skippedTooLarge: boolean;
|
|
20
|
+
skippedBinary: boolean;
|
|
21
|
+
}
|
|
22
|
+
export declare function createScanSummary(): ScanSummary;
|
|
23
|
+
export declare function shouldStopOnSignalOrLimit(signal: AbortSignal, matchesCount: number, maxResults: number, summary: ScanSummary): boolean;
|
|
24
|
+
export declare function applyScanResult(result: ScanResultLike, matches: ContentMatch[], summary: ScanSummary, remaining: number): void;
|
|
25
|
+
export declare function shouldUseWorkers(): boolean;
|
|
26
|
+
export declare function getSearchEngineName(): 'workers' | 'sequential';
|
|
27
|
+
export declare function scanMatches(files: AsyncIterable<ResolvedFile>, pattern: string, matcherOptions: MatcherOptions, scanOptions: ScanFileOptions, maxResults: number, signal: AbortSignal, summary: ScanSummary): Promise<SearchContentResult['matches']>;
|
|
28
|
+
export {};
|
|
29
|
+
//# sourceMappingURL=search-content-scan.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-content-scan.d.ts","sourceRoot":"","sources":["../../../src/lib/file-operations/search-content-scan.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEzE,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,eAAe,EAKrB,MAAM,gCAAgC,CAAC;AAExC,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,mBAAmB,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC;CAChE;AAED,UAAU,cAAc;IACtB,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,wBAAgB,iBAAiB,IAAI,WAAW,CAU/C;AAED,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,WAAW,EACnB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAYT;AAED,wBAAgB,eAAe,CAC7B,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,EAAE,WAAW,EACpB,SAAS,EAAE,MAAM,GAChB,IAAI,CAON;AAuqBD,wBAAgB,gBAAgB,IAAI,OAAO,CAG1C;AAED,wBAAgB,mBAAmB,IAAI,SAAS,GAAG,YAAY,CAE9D;AAED,wBAAsB,WAAW,CAC/B,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,EAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,eAAe,EAC5B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAsBzC"}
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
import * as path from 'node:path';
|
|
2
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
3
|
+
import { Worker } from 'node:worker_threads';
|
|
4
|
+
import { SEARCH_WORKERS } from '../constants.js';
|
|
5
|
+
import { buildMatcher, scanFileResolved, } from './search-content-worker-api.js';
|
|
6
|
+
export function createScanSummary() {
|
|
7
|
+
return {
|
|
8
|
+
filesScanned: 0,
|
|
9
|
+
filesMatched: 0,
|
|
10
|
+
skippedTooLarge: 0,
|
|
11
|
+
skippedBinary: 0,
|
|
12
|
+
skippedInaccessible: 0,
|
|
13
|
+
truncated: false,
|
|
14
|
+
stoppedReason: undefined,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export function shouldStopOnSignalOrLimit(signal, matchesCount, maxResults, summary) {
|
|
18
|
+
if (signal.aborted) {
|
|
19
|
+
summary.truncated = true;
|
|
20
|
+
summary.stoppedReason = 'timeout';
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
if (matchesCount >= maxResults) {
|
|
24
|
+
summary.truncated = true;
|
|
25
|
+
summary.stoppedReason = 'maxResults';
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
export function applyScanResult(result, matches, summary, remaining) {
|
|
31
|
+
if (result.skippedTooLarge)
|
|
32
|
+
summary.skippedTooLarge++;
|
|
33
|
+
if (result.skippedBinary)
|
|
34
|
+
summary.skippedBinary++;
|
|
35
|
+
if (result.matched)
|
|
36
|
+
summary.filesMatched++;
|
|
37
|
+
if (result.matches.length > 0 && remaining > 0) {
|
|
38
|
+
matches.push(...result.matches.slice(0, remaining));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async function scanSequentialFile(file, matcher, scanOptions, signal, maxResults, matches, summary) {
|
|
42
|
+
try {
|
|
43
|
+
const remaining = maxResults - matches.length;
|
|
44
|
+
const result = await scanFileResolved(file.resolvedPath, file.requestedPath, matcher, scanOptions, signal, remaining);
|
|
45
|
+
applyScanResult(result, matches, summary, remaining);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
summary.skippedInaccessible++;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async function collectSequentialMatches(files, matcher, scanOptions, maxResults, signal, summary) {
|
|
52
|
+
const matches = [];
|
|
53
|
+
for await (const file of files) {
|
|
54
|
+
if (shouldStopOnSignalOrLimit(signal, matches.length, maxResults, summary)) {
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
await scanSequentialFile(file, matcher, scanOptions, signal, maxResults, matches, summary);
|
|
58
|
+
}
|
|
59
|
+
return matches;
|
|
60
|
+
}
|
|
61
|
+
async function scanFilesSequential(files, pattern, matcherOptions, scanOptions, maxResults, signal, summary) {
|
|
62
|
+
const matcher = buildMatcher(pattern, matcherOptions);
|
|
63
|
+
return await collectSequentialMatches(files, matcher, scanOptions, maxResults, signal, summary);
|
|
64
|
+
}
|
|
65
|
+
const currentDir = path.dirname(fileURLToPath(import.meta.url));
|
|
66
|
+
const currentFile = fileURLToPath(import.meta.url);
|
|
67
|
+
const isSourceContext = currentFile.endsWith('.ts');
|
|
68
|
+
const WORKER_SCRIPT_PATH = path.join(currentDir, isSourceContext ? 'search-worker.ts' : 'search-worker.js');
|
|
69
|
+
const WORKER_SCRIPT_URL = pathToFileURL(WORKER_SCRIPT_PATH);
|
|
70
|
+
const MAX_RESPAWNS = 3;
|
|
71
|
+
function handleWorkerMessage(slot, message, log) {
|
|
72
|
+
const pending = slot.pending.get(message.id);
|
|
73
|
+
if (!pending) {
|
|
74
|
+
log(`Received message for unknown request ${String(message.id)}`);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
slot.pending.delete(message.id);
|
|
78
|
+
if (message.type === 'result') {
|
|
79
|
+
pending.resolve(message.result);
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
pending.reject(new Error(message.error));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function handleWorkerError(slot, error, log) {
|
|
86
|
+
log(`Worker ${String(slot.index)} error: ${error.message}`);
|
|
87
|
+
for (const [, pending] of slot.pending) {
|
|
88
|
+
pending.reject(new Error(`Worker error: ${error.message}`));
|
|
89
|
+
}
|
|
90
|
+
slot.pending.clear();
|
|
91
|
+
slot.worker?.terminate().catch(() => { });
|
|
92
|
+
slot.worker = null;
|
|
93
|
+
}
|
|
94
|
+
function handleWorkerExit(slot, code, isClosed, maxRespawns, log) {
|
|
95
|
+
log(`Worker ${String(slot.index)} exited with code ${String(code)}`);
|
|
96
|
+
if (isClosed) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (slot.pending.size > 0) {
|
|
100
|
+
const error = new Error(`Worker exited unexpectedly with code ${String(code)}`);
|
|
101
|
+
for (const [, pending] of slot.pending) {
|
|
102
|
+
pending.reject(error);
|
|
103
|
+
}
|
|
104
|
+
slot.pending.clear();
|
|
105
|
+
}
|
|
106
|
+
slot.worker = null;
|
|
107
|
+
if (code !== 0 && slot.respawnCount < maxRespawns) {
|
|
108
|
+
slot.respawnCount++;
|
|
109
|
+
log(`Worker ${String(slot.index)} will be respawned on next request (attempt ${String(slot.respawnCount)}/${String(maxRespawns)})`);
|
|
110
|
+
}
|
|
111
|
+
else if (slot.respawnCount >= maxRespawns) {
|
|
112
|
+
log(`Worker ${String(slot.index)} exceeded max respawns, slot disabled`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function selectSlot(slots, nextSlotIndex, maxRespawns) {
|
|
116
|
+
let bestSlot = null;
|
|
117
|
+
let bestNextSlotIndex = nextSlotIndex;
|
|
118
|
+
let bestPending = Number.POSITIVE_INFINITY;
|
|
119
|
+
for (let offset = 0; offset < slots.length; offset++) {
|
|
120
|
+
const index = (nextSlotIndex + offset) % slots.length;
|
|
121
|
+
const slot = slots[index];
|
|
122
|
+
if (!slot)
|
|
123
|
+
continue;
|
|
124
|
+
if (!slot.worker && slot.respawnCount >= maxRespawns)
|
|
125
|
+
continue;
|
|
126
|
+
const pendingSize = slot.pending.size;
|
|
127
|
+
if (pendingSize < bestPending) {
|
|
128
|
+
bestSlot = slot;
|
|
129
|
+
bestPending = pendingSize;
|
|
130
|
+
bestNextSlotIndex = (index + 1) % slots.length;
|
|
131
|
+
if (bestPending === 0)
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return { slot: bestSlot, nextSlotIndex: bestNextSlotIndex };
|
|
136
|
+
}
|
|
137
|
+
function attachWorkerHandlers(worker, slot, getClosed, maxRespawns, log) {
|
|
138
|
+
worker.on('message', (message) => {
|
|
139
|
+
handleWorkerMessage(slot, message, log);
|
|
140
|
+
});
|
|
141
|
+
worker.on('error', (error) => {
|
|
142
|
+
handleWorkerError(slot, error, log);
|
|
143
|
+
});
|
|
144
|
+
worker.on('exit', (code) => {
|
|
145
|
+
handleWorkerExit(slot, code, getClosed(), maxRespawns, log);
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
class SearchWorkerPool {
|
|
149
|
+
slots;
|
|
150
|
+
debug;
|
|
151
|
+
nextRequestId = 0;
|
|
152
|
+
nextSlotIndex = 0;
|
|
153
|
+
closed = false;
|
|
154
|
+
constructor(options) {
|
|
155
|
+
this.debug = options.debug ?? false;
|
|
156
|
+
this.slots = [];
|
|
157
|
+
for (let i = 0; i < options.size; i++) {
|
|
158
|
+
this.slots.push({
|
|
159
|
+
worker: null,
|
|
160
|
+
pending: new Map(),
|
|
161
|
+
respawnCount: 0,
|
|
162
|
+
index: i,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
log(message) {
|
|
167
|
+
if (this.debug) {
|
|
168
|
+
console.error(`[SearchWorkerPool] ${message}`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
spawnWorker(slot) {
|
|
172
|
+
this.log(`Spawning worker for slot ${String(slot.index)}`);
|
|
173
|
+
const workerOptions = {
|
|
174
|
+
workerData: {
|
|
175
|
+
debug: this.debug,
|
|
176
|
+
threadId: slot.index,
|
|
177
|
+
},
|
|
178
|
+
type: 'module',
|
|
179
|
+
execArgv: isSourceContext ? ['--import', 'tsx/esm'] : undefined,
|
|
180
|
+
};
|
|
181
|
+
const worker = new Worker(WORKER_SCRIPT_URL, workerOptions);
|
|
182
|
+
worker.unref();
|
|
183
|
+
const logEntry = (entry) => {
|
|
184
|
+
this.log(entry);
|
|
185
|
+
};
|
|
186
|
+
attachWorkerHandlers(worker, slot, () => this.closed, MAX_RESPAWNS, logEntry);
|
|
187
|
+
return worker;
|
|
188
|
+
}
|
|
189
|
+
getWorker(slot) {
|
|
190
|
+
slot.worker ??= this.spawnWorker(slot);
|
|
191
|
+
return slot.worker;
|
|
192
|
+
}
|
|
193
|
+
ensureOpen() {
|
|
194
|
+
if (this.closed) {
|
|
195
|
+
throw new Error('Worker pool is closed');
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
selectAvailableSlot() {
|
|
199
|
+
const selection = selectSlot(this.slots, this.nextSlotIndex, MAX_RESPAWNS);
|
|
200
|
+
this.nextSlotIndex = selection.nextSlotIndex;
|
|
201
|
+
if (!selection.slot) {
|
|
202
|
+
throw new Error('All worker slots are disabled');
|
|
203
|
+
}
|
|
204
|
+
return selection.slot;
|
|
205
|
+
}
|
|
206
|
+
buildScanRequest(id, request) {
|
|
207
|
+
return {
|
|
208
|
+
type: 'scan',
|
|
209
|
+
id,
|
|
210
|
+
...request,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
createScanPromise(slot, worker, scanRequest) {
|
|
214
|
+
let settled = false;
|
|
215
|
+
return new Promise((resolve, reject) => {
|
|
216
|
+
const safeResolve = (result) => {
|
|
217
|
+
if (settled)
|
|
218
|
+
return;
|
|
219
|
+
settled = true;
|
|
220
|
+
resolve(result);
|
|
221
|
+
};
|
|
222
|
+
const safeReject = (error) => {
|
|
223
|
+
if (settled)
|
|
224
|
+
return;
|
|
225
|
+
settled = true;
|
|
226
|
+
reject(error);
|
|
227
|
+
};
|
|
228
|
+
slot.pending.set(scanRequest.id, {
|
|
229
|
+
resolve: safeResolve,
|
|
230
|
+
reject: safeReject,
|
|
231
|
+
request: scanRequest,
|
|
232
|
+
});
|
|
233
|
+
try {
|
|
234
|
+
worker.postMessage(scanRequest);
|
|
235
|
+
}
|
|
236
|
+
catch (error) {
|
|
237
|
+
slot.pending.delete(scanRequest.id);
|
|
238
|
+
safeReject(error instanceof Error ? error : new Error(String(error)));
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
createCancel(slot, worker, id) {
|
|
243
|
+
return () => {
|
|
244
|
+
const pending = slot.pending.get(id);
|
|
245
|
+
if (!pending)
|
|
246
|
+
return;
|
|
247
|
+
slot.pending.delete(id);
|
|
248
|
+
try {
|
|
249
|
+
worker.postMessage({ type: 'cancel', id });
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
// Ignore: cancellation should still reject the local pending promise.
|
|
253
|
+
}
|
|
254
|
+
pending.reject(new Error('Scan cancelled'));
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
scan(request) {
|
|
258
|
+
this.ensureOpen();
|
|
259
|
+
const slot = this.selectAvailableSlot();
|
|
260
|
+
const worker = this.getWorker(slot);
|
|
261
|
+
const id = this.nextRequestId++;
|
|
262
|
+
const scanRequest = this.buildScanRequest(id, request);
|
|
263
|
+
const promise = this.createScanPromise(slot, worker, scanRequest);
|
|
264
|
+
const cancel = this.createCancel(slot, worker, id);
|
|
265
|
+
const outcome = promise.then((result) => ({ task, result }), (error) => ({
|
|
266
|
+
task,
|
|
267
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
268
|
+
}));
|
|
269
|
+
const task = { id, promise, cancel, outcome };
|
|
270
|
+
return task;
|
|
271
|
+
}
|
|
272
|
+
async close() {
|
|
273
|
+
if (this.closed)
|
|
274
|
+
return;
|
|
275
|
+
this.closed = true;
|
|
276
|
+
this.log('Closing worker pool');
|
|
277
|
+
for (const slot of this.slots) {
|
|
278
|
+
for (const [, pending] of slot.pending) {
|
|
279
|
+
pending.reject(new Error('Worker pool closed'));
|
|
280
|
+
}
|
|
281
|
+
slot.pending.clear();
|
|
282
|
+
}
|
|
283
|
+
const terminatePromises = [];
|
|
284
|
+
for (const slot of this.slots) {
|
|
285
|
+
if (slot.worker) {
|
|
286
|
+
try {
|
|
287
|
+
slot.worker.postMessage({ type: 'shutdown' });
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// Ignore: worker may already be terminating.
|
|
291
|
+
}
|
|
292
|
+
terminatePromises.push(slot.worker.terminate());
|
|
293
|
+
slot.worker = null;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
await Promise.allSettled(terminatePromises);
|
|
297
|
+
this.log('Worker pool closed');
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
function isWorkerPoolAvailable() {
|
|
301
|
+
return !isSourceContext;
|
|
302
|
+
}
|
|
303
|
+
let poolInstance = null;
|
|
304
|
+
let poolSize = 0;
|
|
305
|
+
let poolDebug = false;
|
|
306
|
+
function getSearchWorkerPool(size, debug = false) {
|
|
307
|
+
if (size <= 0) {
|
|
308
|
+
throw new Error('Pool size must be positive');
|
|
309
|
+
}
|
|
310
|
+
if (poolInstance && poolSize === size && poolDebug === debug) {
|
|
311
|
+
return poolInstance;
|
|
312
|
+
}
|
|
313
|
+
if (poolInstance) {
|
|
314
|
+
void poolInstance.close();
|
|
315
|
+
}
|
|
316
|
+
poolInstance = new SearchWorkerPool({ size, debug });
|
|
317
|
+
poolSize = size;
|
|
318
|
+
poolDebug = debug;
|
|
319
|
+
return poolInstance;
|
|
320
|
+
}
|
|
321
|
+
function createParallelScanConfig(pattern, matcherOptions, scanOptions, maxResults, signal) {
|
|
322
|
+
const debug = process.env.FS_CONTEXT_SEARCH_WORKERS_DEBUG === '1';
|
|
323
|
+
return {
|
|
324
|
+
pool: getSearchWorkerPool(SEARCH_WORKERS, debug),
|
|
325
|
+
pattern,
|
|
326
|
+
matcherOptions,
|
|
327
|
+
scanOptions,
|
|
328
|
+
maxResults,
|
|
329
|
+
maxInFlight: Math.max(1, SEARCH_WORKERS),
|
|
330
|
+
signal,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
function createParallelScanState(files, summary) {
|
|
334
|
+
return {
|
|
335
|
+
matches: [],
|
|
336
|
+
summary,
|
|
337
|
+
inFlight: new Set(),
|
|
338
|
+
iterator: files[Symbol.asyncIterator](),
|
|
339
|
+
done: false,
|
|
340
|
+
stoppedEarly: false,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
function markTruncated(summary, reason) {
|
|
344
|
+
summary.truncated = true;
|
|
345
|
+
summary.stoppedReason = reason;
|
|
346
|
+
}
|
|
347
|
+
function cancelInFlight(inFlight) {
|
|
348
|
+
for (const task of inFlight) {
|
|
349
|
+
task.cancel();
|
|
350
|
+
void task.promise.catch(() => { });
|
|
351
|
+
}
|
|
352
|
+
inFlight.clear();
|
|
353
|
+
}
|
|
354
|
+
function stopIfSignaledOrLimited(config, state) {
|
|
355
|
+
if (!shouldStopOnSignalOrLimit(config.signal, state.matches.length, config.maxResults, state.summary)) {
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
state.stoppedEarly = true;
|
|
359
|
+
state.done = true;
|
|
360
|
+
cancelInFlight(state.inFlight);
|
|
361
|
+
return true;
|
|
362
|
+
}
|
|
363
|
+
async function awaitNextOutcome(inFlight) {
|
|
364
|
+
function* outcomes() {
|
|
365
|
+
for (const task of inFlight) {
|
|
366
|
+
yield task.outcome;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return await Promise.race(outcomes());
|
|
370
|
+
}
|
|
371
|
+
function handleWorkerOutcome(outcome, config, state) {
|
|
372
|
+
state.inFlight.delete(outcome.task);
|
|
373
|
+
if (outcome.error) {
|
|
374
|
+
if (outcome.error.message !== 'Scan cancelled') {
|
|
375
|
+
state.summary.skippedInaccessible++;
|
|
376
|
+
}
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const { result } = outcome;
|
|
380
|
+
if (!result)
|
|
381
|
+
return;
|
|
382
|
+
const remaining = config.maxResults - state.matches.length;
|
|
383
|
+
if (remaining <= 0) {
|
|
384
|
+
markTruncated(state.summary, 'maxResults');
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
applyScanResult(result, state.matches, state.summary, remaining);
|
|
388
|
+
if (state.matches.length >= config.maxResults) {
|
|
389
|
+
markTruncated(state.summary, 'maxResults');
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
async function enqueueNextTask(config, state) {
|
|
393
|
+
if (stopIfSignaledOrLimited(config, state))
|
|
394
|
+
return;
|
|
395
|
+
const next = await state.iterator.next();
|
|
396
|
+
if (next.done) {
|
|
397
|
+
state.done = true;
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
const remaining = Math.max(1, config.maxResults - state.matches.length);
|
|
401
|
+
const task = config.pool.scan({
|
|
402
|
+
resolvedPath: next.value.resolvedPath,
|
|
403
|
+
requestedPath: next.value.requestedPath,
|
|
404
|
+
pattern: config.pattern,
|
|
405
|
+
matcherOptions: config.matcherOptions,
|
|
406
|
+
scanOptions: config.scanOptions,
|
|
407
|
+
maxMatches: remaining,
|
|
408
|
+
});
|
|
409
|
+
state.inFlight.add(task);
|
|
410
|
+
}
|
|
411
|
+
async function fillInFlight(config, state) {
|
|
412
|
+
while (!state.done && state.inFlight.size < config.maxInFlight) {
|
|
413
|
+
await enqueueNextTask(config, state);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
async function drainInFlight(config, state) {
|
|
417
|
+
await fillInFlight(config, state);
|
|
418
|
+
while (state.inFlight.size > 0) {
|
|
419
|
+
if (stopIfSignaledOrLimited(config, state))
|
|
420
|
+
break;
|
|
421
|
+
handleWorkerOutcome(await awaitNextOutcome(state.inFlight), config, state);
|
|
422
|
+
if (stopIfSignaledOrLimited(config, state))
|
|
423
|
+
break;
|
|
424
|
+
await fillInFlight(config, state);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
async function finalizeParallelScan(state) {
|
|
428
|
+
if (!state.stoppedEarly)
|
|
429
|
+
return;
|
|
430
|
+
cancelInFlight(state.inFlight);
|
|
431
|
+
await state.iterator.return?.();
|
|
432
|
+
}
|
|
433
|
+
function attachAbortHandler(config, state) {
|
|
434
|
+
const onAbort = () => {
|
|
435
|
+
state.stoppedEarly = true;
|
|
436
|
+
markTruncated(state.summary, 'timeout');
|
|
437
|
+
cancelInFlight(state.inFlight);
|
|
438
|
+
};
|
|
439
|
+
config.signal.addEventListener('abort', onAbort, { once: true });
|
|
440
|
+
return onAbort;
|
|
441
|
+
}
|
|
442
|
+
async function scanFilesParallel(files, pattern, matcherOptions, scanOptions, maxResults, signal, summary) {
|
|
443
|
+
const config = createParallelScanConfig(pattern, matcherOptions, scanOptions, maxResults, signal);
|
|
444
|
+
const state = createParallelScanState(files, summary);
|
|
445
|
+
const onAbort = attachAbortHandler(config, state);
|
|
446
|
+
try {
|
|
447
|
+
await drainInFlight(config, state);
|
|
448
|
+
await finalizeParallelScan(state);
|
|
449
|
+
}
|
|
450
|
+
finally {
|
|
451
|
+
signal.removeEventListener('abort', onAbort);
|
|
452
|
+
}
|
|
453
|
+
return state.matches;
|
|
454
|
+
}
|
|
455
|
+
export function shouldUseWorkers() {
|
|
456
|
+
// Multi-CPU only: require at least 2 workers to justify thread overhead.
|
|
457
|
+
return isWorkerPoolAvailable() && SEARCH_WORKERS >= 2;
|
|
458
|
+
}
|
|
459
|
+
export function getSearchEngineName() {
|
|
460
|
+
return shouldUseWorkers() ? 'workers' : 'sequential';
|
|
461
|
+
}
|
|
462
|
+
export async function scanMatches(files, pattern, matcherOptions, scanOptions, maxResults, signal, summary) {
|
|
463
|
+
if (shouldUseWorkers()) {
|
|
464
|
+
return await scanFilesParallel(files, pattern, matcherOptions, scanOptions, maxResults, signal, summary);
|
|
465
|
+
}
|
|
466
|
+
return await scanFilesSequential(files, pattern, matcherOptions, scanOptions, maxResults, signal, summary);
|
|
467
|
+
}
|
|
468
|
+
//# sourceMappingURL=search-content-scan.js.map
|