@cesarandreslopez/occ 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/src/code/build.d.ts +49 -1
- package/dist/src/code/build.js +127 -33
- package/dist/src/code/build.js.map +1 -1
- package/dist/src/code/chunk.d.ts +10 -0
- package/dist/src/code/chunk.js +14 -1
- package/dist/src/code/chunk.js.map +1 -1
- package/dist/src/code/command.js +16 -0
- package/dist/src/code/command.js.map +1 -1
- package/dist/src/code/discover.d.ts +13 -0
- package/dist/src/code/discover.js +49 -2
- package/dist/src/code/discover.js.map +1 -1
- package/dist/src/code/focus.d.ts +31 -0
- package/dist/src/code/focus.js +192 -0
- package/dist/src/code/focus.js.map +1 -0
- package/dist/src/code/incremental.d.ts +75 -0
- package/dist/src/code/incremental.js +257 -0
- package/dist/src/code/incremental.js.map +1 -0
- package/dist/src/code/map.d.ts +36 -1
- package/dist/src/code/map.js +83 -4
- package/dist/src/code/map.js.map +1 -1
- package/dist/src/code/output.js +28 -4
- package/dist/src/code/output.js.map +1 -1
- package/dist/src/code/parsers.d.ts +1 -0
- package/dist/src/code/parsers.js +1 -1
- package/dist/src/code/parsers.js.map +1 -1
- package/dist/src/code/rank.d.ts +14 -0
- package/dist/src/code/rank.js +58 -0
- package/dist/src/code/rank.js.map +1 -1
- package/dist/src/code/store.d.ts +20 -0
- package/dist/src/code/store.js +262 -109
- package/dist/src/code/store.js.map +1 -1
- package/dist/src/doc/chunk.d.ts +17 -0
- package/dist/src/doc/chunk.js +47 -18
- package/dist/src/doc/chunk.js.map +1 -1
- package/dist/src/doc/inspect-docx.js +13 -3
- package/dist/src/doc/inspect-docx.js.map +1 -1
- package/dist/src/doc/inspect-pdf.js +12 -17
- package/dist/src/doc/inspect-pdf.js.map +1 -1
- package/dist/src/index.d.ts +13 -8
- package/dist/src/index.js +9 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/inspect/xlsx-cells.d.ts +8 -0
- package/dist/src/inspect/xlsx-cells.js +17 -2
- package/dist/src/inspect/xlsx-cells.js.map +1 -1
- package/dist/src/markdown/convert.js +45 -27
- package/dist/src/markdown/convert.js.map +1 -1
- package/dist/src/parsers/docx.js +10 -2
- package/dist/src/parsers/docx.js.map +1 -1
- package/dist/src/parsers/index.js +5 -0
- package/dist/src/parsers/index.js.map +1 -1
- package/dist/src/parsers/pdf.js +12 -32
- package/dist/src/parsers/pdf.js.map +1 -1
- package/dist/src/parsers/xlsx.js +11 -1
- package/dist/src/parsers/xlsx.js.map +1 -1
- package/dist/src/progress-event.d.ts +3 -1
- package/dist/src/progress-event.js +31 -1
- package/dist/src/progress-event.js.map +1 -1
- package/dist/src/sheet/inspect.js +20 -10
- package/dist/src/sheet/inspect.js.map +1 -1
- package/dist/src/table/inspect-docx.js +10 -1
- package/dist/src/table/inspect-docx.js.map +1 -1
- package/dist/src/table/inspect-xlsx.js +15 -5
- package/dist/src/table/inspect-xlsx.js.map +1 -1
- package/dist/src/utils.d.ts +24 -0
- package/dist/src/utils.js +91 -2
- package/dist/src/utils.js.map +1 -1
- package/dist/src/workspace/bundle.d.ts +24 -0
- package/dist/src/workspace/bundle.js +303 -97
- package/dist/src/workspace/bundle.js.map +1 -1
- package/dist/src/workspace/documents.d.ts +4 -0
- package/dist/src/workspace/documents.js +15 -2
- package/dist/src/workspace/documents.js.map +1 -1
- package/dist/src/workspace/prepare-types.d.ts +15 -0
- package/dist/src/workspace/prepare-types.js +18 -1
- package/dist/src/workspace/prepare-types.js.map +1 -1
- package/dist/src/workspace/prepare.js +10 -2
- package/dist/src/workspace/prepare.js.map +1 -1
- package/package.json +9 -2
package/dist/src/utils.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import fg from 'fast-glob';
|
|
2
2
|
import ignore from 'ignore';
|
|
3
|
-
import officeparser from 'officeparser';
|
|
4
3
|
import path from 'node:path';
|
|
5
4
|
import { readFile } from 'node:fs/promises';
|
|
6
5
|
import { z } from 'zod';
|
|
@@ -122,6 +121,39 @@ export function abortIfNeeded(signal) {
|
|
|
122
121
|
throw new OccAbortError();
|
|
123
122
|
}
|
|
124
123
|
}
|
|
124
|
+
/**
|
|
125
|
+
* Map `items` through an async `fn` with at most `concurrency` calls in flight.
|
|
126
|
+
*
|
|
127
|
+
* Results are returned in input order (each worker writes into its item's slot).
|
|
128
|
+
* Non-finite `concurrency` (NaN/Infinity) falls back to 1. The signal is checked
|
|
129
|
+
* before each item is dispatched; an abort surfaces as OccAbortError. The first
|
|
130
|
+
* rejection from `fn` propagates and stops further dispatch (in-flight items may
|
|
131
|
+
* still complete) — callers that need per-item error capture should catch inside `fn`.
|
|
132
|
+
*/
|
|
133
|
+
export async function mapWithConcurrency(items, concurrency, fn, signal) {
|
|
134
|
+
const results = new Array(items.length);
|
|
135
|
+
if (items.length === 0)
|
|
136
|
+
return results;
|
|
137
|
+
const safeConcurrency = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
|
|
138
|
+
const workerCount = Math.max(1, Math.min(safeConcurrency, items.length));
|
|
139
|
+
let cursor = 0;
|
|
140
|
+
let failed = false;
|
|
141
|
+
async function worker() {
|
|
142
|
+
while (!failed && cursor < items.length) {
|
|
143
|
+
abortIfNeeded(signal);
|
|
144
|
+
const index = cursor++;
|
|
145
|
+
try {
|
|
146
|
+
results[index] = await fn(items[index], index);
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
failed = true;
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
155
|
+
return results;
|
|
156
|
+
}
|
|
125
157
|
export function sanitizeForkExecArgv(execArgv = process.execArgv) {
|
|
126
158
|
const sanitized = [];
|
|
127
159
|
for (let i = 0; i < execArgv.length; i++) {
|
|
@@ -149,8 +181,65 @@ export async function writeStream(stream, text) {
|
|
|
149
181
|
});
|
|
150
182
|
});
|
|
151
183
|
}
|
|
184
|
+
/** Noisy pdf.js output prefixes suppressed while parsing PDFs (font parsing, deprecated API, etc.). */
|
|
185
|
+
export const PDF_LOG_SUPPRESSED_PREFIXES = ['Warning: ', 'Info: ', 'Deprecated API usage: '];
|
|
186
|
+
// Reference-counted console.log suppression shared by every PDF call site so
|
|
187
|
+
// concurrent (and nested) suppressed sections never clobber each other's restore:
|
|
188
|
+
// the original console.log is captured once when the depth goes 0 → 1 and
|
|
189
|
+
// restored only when the last active section ends (depth back to 0).
|
|
190
|
+
const activeSuppressionPrefixLists = [];
|
|
191
|
+
let consoleLogSuppressionDepth = 0;
|
|
192
|
+
let savedOriginalConsoleLog;
|
|
193
|
+
/**
|
|
194
|
+
* Run `fn` while console.log lines starting with any of `prefixes` are dropped.
|
|
195
|
+
* Lines that match no active prefix list pass through unchanged. Safe to overlap
|
|
196
|
+
* and nest across concurrent callers.
|
|
197
|
+
*/
|
|
198
|
+
export async function suppressConsoleLogWhile(prefixes, fn) {
|
|
199
|
+
if (consoleLogSuppressionDepth++ === 0) {
|
|
200
|
+
const originalLog = console.log;
|
|
201
|
+
savedOriginalConsoleLog = originalLog;
|
|
202
|
+
console.log = (...args) => {
|
|
203
|
+
const first = args[0];
|
|
204
|
+
if (typeof first === 'string'
|
|
205
|
+
&& activeSuppressionPrefixLists.some(list => list.some(prefix => first.startsWith(prefix)))) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
originalLog.apply(console, args);
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
activeSuppressionPrefixLists.push(prefixes);
|
|
212
|
+
try {
|
|
213
|
+
return await fn();
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
const index = activeSuppressionPrefixLists.indexOf(prefixes);
|
|
217
|
+
if (index >= 0)
|
|
218
|
+
activeSuppressionPrefixLists.splice(index, 1);
|
|
219
|
+
if (--consoleLogSuppressionDepth === 0 && savedOriginalConsoleLog) {
|
|
220
|
+
console.log = savedOriginalConsoleLog;
|
|
221
|
+
savedOriginalConsoleLog = undefined;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Rethrow a lazy `import()` failure with the offending package named, so a
|
|
227
|
+
* missing/broken heavy dependency surfaces as an actionable diagnostic instead
|
|
228
|
+
* of a generic per-file parse failure.
|
|
229
|
+
*/
|
|
230
|
+
export function wrapDependencyLoadError(pkg, error) {
|
|
231
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
232
|
+
throw new Error(`occ dependency '${pkg}' failed to load: ${message}`, { cause: error });
|
|
233
|
+
}
|
|
234
|
+
let officeparserModule;
|
|
235
|
+
async function loadOfficeparser() {
|
|
236
|
+
officeparserModule ??= import('officeparser')
|
|
237
|
+
.catch((error) => wrapDependencyLoadError('officeparser', error));
|
|
238
|
+
return officeparserModule;
|
|
239
|
+
}
|
|
152
240
|
export async function extractOfficeText(input) {
|
|
153
|
-
const
|
|
241
|
+
const { OfficeParser } = await loadOfficeparser();
|
|
242
|
+
const raw = await OfficeParser.parseOffice(input);
|
|
154
243
|
if (typeof raw === 'string')
|
|
155
244
|
return raw;
|
|
156
245
|
if (raw && typeof raw.toText === 'function') {
|
package/dist/src/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAA4B;IACvD,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IACzB,OAAO,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoB;IACvD,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,UAAU;YAAE,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,UAAU,EAAE,EAAE,MAAM,UAAU,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAA2B;IACvD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAa,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAE1E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AAE5G,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAmBvD,SAAS,iBAAiB,CAAC,OAAgB,EAAE,MAAqD;IAChG,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,YAAoB,EAAE,QAAgB;IAC7D,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,KAAK,UAAU,2BAA2B,CACxC,OAAe,EACf,UAAoB;IAEpB,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,EAAE;QAC/D,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,mBAAmB,EAAE,KAAK;QAC1B,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC;QACxC,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC7E,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjD,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC1E,CAAC;IACJ,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC7E,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAChF,OAAO,SAAS,GAAG,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,UAAgC,EAAE;IAElC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,WAAW;QAC3C,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,2BAA2B,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,aAAa,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;QAC/E,CAAC,CAAC,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;QAClE,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,cAAc,GAAG,OAAO,CAAC,qBAAqB,IAAI,OAAO,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC;QAC9F,CAAC,CAAC,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,OAAO,CAAC,YAAoB;YAC1B,MAAM,cAAc,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjF,IAAI,WAAW,GAAG,KAAK,CAAC;YAExB,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;gBACtC,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBAC/D,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBAC5C,CAAC,CAAC,cAAc;oBAChB,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,cAAc,EAAE,CAAC;gBACnB,cAAc,GAAG,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACjF,CAAC;YAED,OAAO,WAAW,IAAI,cAAc,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAoB;IAChD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,aAAa,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAmB,EACnB,WAAmB,EACnB,EAA0C,EAC1C,MAAoB;IAEpB,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,UAAU,MAAM;QACnB,OAAO,CAAC,MAAM,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxC,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,WAAqB,OAAO,CAAC,QAAQ;IACxE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,GAAG,KAAK,cAAc,EAAE,CAAC;YAC3B,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;YAAE,SAAS;QAC9C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAA6B,EAAE,IAAY;IAC3E,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAoB,EAAE,EAAE;YAC1C,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;gBACpB,OAAO,EAAE,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uGAAuG;AACvG,MAAM,CAAC,MAAM,2BAA2B,GAAsB,CAAC,WAAW,EAAE,QAAQ,EAAE,wBAAwB,CAAC,CAAC;AAEhH,6EAA6E;AAC7E,kFAAkF;AAClF,0EAA0E;AAC1E,qEAAqE;AACrE,MAAM,4BAA4B,GAA0B,EAAE,CAAC;AAC/D,IAAI,0BAA0B,GAAG,CAAC,CAAC;AACnC,IAAI,uBAAuD,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAA2B,EAC3B,EAAoB;IAEpB,IAAI,0BAA0B,EAAE,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;QAChC,uBAAuB,GAAG,WAAW,CAAC;QACtC,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,OAAO,KAAK,KAAK,QAAQ;mBACxB,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9F,OAAO;YACT,CAAC;YACD,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,GAAG,4BAA4B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,KAAK,IAAI,CAAC;YAAE,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,EAAE,0BAA0B,KAAK,CAAC,IAAI,uBAAuB,EAAE,CAAC;YAClE,OAAO,CAAC,GAAG,GAAG,uBAAuB,CAAC;YACtC,uBAAuB,GAAG,SAAS,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAW,EAAE,KAAc;IACjE,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,qBAAqB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAC1F,CAAC;AAOD,IAAI,kBAA2D,CAAC;AAEhE,KAAK,UAAU,gBAAgB;IAC7B,kBAAkB,KAAK,MAAM,CAAC,cAAc,CAAC;SAC1C,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,uBAAuB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7E,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAsB;IAC5D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,GAAG,IAAI,OAAQ,GAA4B,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAE,GAAgC,CAAC,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACxE,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type CodeIndexStore, type StoreStrategy } from '../code/store.js';
|
|
1
2
|
import { type DocumentChunk, type DocumentChunkOptions } from '../doc/chunk.js';
|
|
2
3
|
import type { CodebaseIndex, ContentMode } from '../code/types.js';
|
|
3
4
|
import type { CodebaseIndexSlim } from '../code/slim.js';
|
|
@@ -26,6 +27,29 @@ export interface WorkspaceBundleOptions {
|
|
|
26
27
|
maxDocumentChunks?: number;
|
|
27
28
|
maxCodeDocumentReferences?: number;
|
|
28
29
|
chunk?: Pick<DocumentChunkOptions, 'maxTokens' | 'overlapTokens' | 'countTokens'>;
|
|
30
|
+
/**
|
|
31
|
+
* Persistent code-index cache directory. When set (and no `store` is
|
|
32
|
+
* provided), the bundle resolves its code index through
|
|
33
|
+
* `openCodeIndexStore` instead of `buildCodebaseIndexIsolated`: cache hits
|
|
34
|
+
* skip parsing entirely, and cache misses build **in-process** (no
|
|
35
|
+
* subprocess isolation — this is an explicit opt-in). Cache entries are
|
|
36
|
+
* keyed by `(contentMode, build-option fingerprint)`, so the bundle options
|
|
37
|
+
* must match the consumer's own store options to share an entry — when in
|
|
38
|
+
* doubt, pass `store` directly (recommended).
|
|
39
|
+
*/
|
|
40
|
+
cacheDir?: string;
|
|
41
|
+
/** Store strategy used with `cacheDir`/`store` (default `'ensure-fresh'`). */
|
|
42
|
+
cacheStrategy?: StoreStrategy;
|
|
43
|
+
/**
|
|
44
|
+
* Bring-your-own code index store. Takes precedence over `cacheDir`; the
|
|
45
|
+
* bundle never disposes a caller-provided store.
|
|
46
|
+
*/
|
|
47
|
+
store?: CodeIndexStore;
|
|
48
|
+
/**
|
|
49
|
+
* Run the description / analysis / documents / code phases concurrently
|
|
50
|
+
* (default `true`). `false` restores the literal sequential path.
|
|
51
|
+
*/
|
|
52
|
+
concurrentPhases?: boolean;
|
|
29
53
|
signal?: AbortSignal;
|
|
30
54
|
onProgress?: OnProgress;
|
|
31
55
|
}
|
|
@@ -4,9 +4,20 @@ import { describeWorkspace } from './describe.js';
|
|
|
4
4
|
import { inspectWorkspaceDocumentSet } from './documents.js';
|
|
5
5
|
import { buildCodebaseIndexIsolated } from '../code/isolated.js';
|
|
6
6
|
import { previewCodebaseSize } from '../code/preview.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { openCodeIndexStore } from '../code/store.js';
|
|
8
|
+
import { slimifyIndex } from '../code/slim.js';
|
|
9
|
+
import { chunkDocument, chunkDocumentFromMarkdown } from '../doc/chunk.js';
|
|
10
|
+
import { abortIfNeeded, mapWithConcurrency, normalizeRelativePath } from '../utils.js';
|
|
11
|
+
import { isOccAbortError } from '../errors.js';
|
|
9
12
|
const DEFAULT_EXCLUDE_DIR = ['node_modules', '.git', 'dist', 'vendor', 'build', 'coverage', 'target'];
|
|
13
|
+
// Step descriptors shared by the sequential and concurrent phase paths.
|
|
14
|
+
const STEP_DESCRIPTION = { phase: 'description', detail: 'Describe workspace' };
|
|
15
|
+
const STEP_ANALYSIS = { phase: 'stats', detail: 'Analyze workspace totals' };
|
|
16
|
+
const STEP_DOCUMENTS = { phase: 'inspect', detail: 'Inspect documents' };
|
|
17
|
+
const STEP_CODE_PREVIEW = { phase: 'code-preview', detail: 'Preview code index size' };
|
|
18
|
+
const STEP_CODE_INDEX = { phase: 'index', detail: 'Build code index' };
|
|
19
|
+
const STEP_DOCUMENT_CHUNK = { phase: 'document-chunk', detail: 'Chunk documents' };
|
|
20
|
+
const STEP_OUTLINE = { phase: 'outline', detail: 'Build workspace outline' };
|
|
10
21
|
function reportStep(onProgress, step, completed, total, startedAt) {
|
|
11
22
|
onProgress?.({
|
|
12
23
|
phase: step.phase,
|
|
@@ -87,6 +98,9 @@ function buildOutline(rootDir, description, documents, codeIndex) {
|
|
|
87
98
|
}
|
|
88
99
|
return root;
|
|
89
100
|
}
|
|
101
|
+
function escapeRegExp(value) {
|
|
102
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
103
|
+
}
|
|
90
104
|
function detectCodeDocumentReferences(codeIndex, documents, limit) {
|
|
91
105
|
if (!codeIndex || !documents || limit <= 0)
|
|
92
106
|
return [];
|
|
@@ -101,6 +115,23 @@ function detectCodeDocumentReferences(codeIndex, documents, limit) {
|
|
|
101
115
|
seenNames.add(node.name);
|
|
102
116
|
return true;
|
|
103
117
|
}).slice(0, 1_000);
|
|
118
|
+
// Compile each symbol pattern once instead of per line.
|
|
119
|
+
const compiled = symbols.map((symbol) => ({
|
|
120
|
+
symbol,
|
|
121
|
+
pattern: new RegExp(`\\b${escapeRegExp(symbol.name)}\\b`),
|
|
122
|
+
}));
|
|
123
|
+
// One alternation over all (longest-first, already-escaped) names, used
|
|
124
|
+
// purely as a per-line skip filter — extraction still runs the per-symbol
|
|
125
|
+
// patterns, so output order and the limit early-return are unchanged.
|
|
126
|
+
let prescreen = null;
|
|
127
|
+
if (symbols.length > 0) {
|
|
128
|
+
try {
|
|
129
|
+
prescreen = new RegExp(`\\b(?:${symbols.map((symbol) => escapeRegExp(symbol.name)).join('|')})\\b`);
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
prescreen = null; // Fall back to scanning every line with the compiled patterns.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
104
135
|
const references = [];
|
|
105
136
|
for (const doc of documents.documents) {
|
|
106
137
|
if (!doc.markdownContent)
|
|
@@ -110,8 +141,9 @@ function detectCodeDocumentReferences(codeIndex, documents, limit) {
|
|
|
110
141
|
const line = lines[lineIndex];
|
|
111
142
|
if (!line.trim())
|
|
112
143
|
continue;
|
|
113
|
-
|
|
114
|
-
|
|
144
|
+
if (prescreen && !prescreen.test(line))
|
|
145
|
+
continue;
|
|
146
|
+
for (const { symbol, pattern } of compiled) {
|
|
115
147
|
if (!pattern.test(line))
|
|
116
148
|
continue;
|
|
117
149
|
references.push({
|
|
@@ -134,32 +166,56 @@ async function buildDocumentChunks(documents, options, onProgress) {
|
|
|
134
166
|
if (!documents)
|
|
135
167
|
return [];
|
|
136
168
|
const maxChunks = options.maxDocumentChunks ?? 500;
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
169
|
+
const total = documents.documents.length;
|
|
170
|
+
let completed = 0;
|
|
171
|
+
let accumulatedChunks = 0;
|
|
172
|
+
// Workers skip documents once the running chunk total reaches the cap, so
|
|
173
|
+
// conversion stops near it instead of chunking every document (documents
|
|
174
|
+
// already in flight still finish — up to the concurrency width of overshoot).
|
|
175
|
+
// Items are dispatched in document order, so every document that contributed
|
|
176
|
+
// to the total precedes any skipped one; flattening in document order plus
|
|
177
|
+
// the post-join slice stays the authoritative cap, and the returned chunks
|
|
178
|
+
// are identical to chunking everything.
|
|
179
|
+
const perDocument = await mapWithConcurrency(documents.documents, 5, async (doc) => {
|
|
180
|
+
if (accumulatedChunks >= maxChunks)
|
|
181
|
+
return undefined;
|
|
182
|
+
let docChunks;
|
|
141
183
|
try {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
184
|
+
docChunks = doc.markdownContent != null
|
|
185
|
+
// Markdown was already produced by document inspection — reuse it
|
|
186
|
+
// instead of re-reading and re-converting the file. Per-document
|
|
187
|
+
// values are spread last so caller chunk options cannot clobber them.
|
|
188
|
+
? chunkDocumentFromMarkdown(doc.markdownContent, {
|
|
189
|
+
...options.chunk,
|
|
190
|
+
filePath: doc.filePath,
|
|
191
|
+
relativePath: doc.relativePath,
|
|
192
|
+
})
|
|
193
|
+
: await chunkDocument(doc.filePath, {
|
|
194
|
+
...options.chunk,
|
|
195
|
+
relativePath: doc.relativePath,
|
|
196
|
+
});
|
|
149
197
|
}
|
|
150
198
|
catch {
|
|
151
199
|
// Document conversion errors are already represented in inspectDocuments.
|
|
152
200
|
}
|
|
201
|
+
if (docChunks)
|
|
202
|
+
accumulatedChunks += docChunks.length;
|
|
153
203
|
onProgress?.({
|
|
154
204
|
phase: 'document-chunk',
|
|
155
|
-
total
|
|
156
|
-
completed:
|
|
205
|
+
total,
|
|
206
|
+
completed: ++completed,
|
|
157
207
|
detail: doc.relativePath,
|
|
158
208
|
currentPath: doc.filePath,
|
|
159
209
|
scope: 'document',
|
|
160
210
|
});
|
|
211
|
+
return docChunks;
|
|
212
|
+
}, options.signal);
|
|
213
|
+
const chunks = [];
|
|
214
|
+
for (const docChunks of perDocument) {
|
|
215
|
+
if (docChunks)
|
|
216
|
+
chunks.push(...docChunks);
|
|
161
217
|
}
|
|
162
|
-
return chunks;
|
|
218
|
+
return chunks.slice(0, maxChunks);
|
|
163
219
|
}
|
|
164
220
|
export async function bundleWorkspace(rootDir, options = {}) {
|
|
165
221
|
const startedAt = Date.now();
|
|
@@ -170,103 +226,253 @@ export async function bundleWorkspace(rootDir, options = {}) {
|
|
|
170
226
|
const includeCode = options.includeCode !== false;
|
|
171
227
|
const includeDocumentChunks = options.includeDocumentChunks ?? false;
|
|
172
228
|
const includeCodeDocumentReferences = options.includeCodeDocumentReferences ?? true;
|
|
229
|
+
const concurrentPhases = options.concurrentPhases !== false;
|
|
173
230
|
const onProgress = options.onProgress;
|
|
174
231
|
const errors = [];
|
|
175
232
|
const shared = commonWorkspaceOptions(options);
|
|
176
233
|
const steps = [
|
|
177
|
-
...(includeDescription ? [
|
|
178
|
-
...(includeAnalysis ? [
|
|
179
|
-
...(includeDocuments ? [
|
|
180
|
-
...(includeCode ? [
|
|
181
|
-
...(includeCode ? [
|
|
182
|
-
...(includeDocumentChunks ? [
|
|
183
|
-
|
|
234
|
+
...(includeDescription ? [STEP_DESCRIPTION] : []),
|
|
235
|
+
...(includeAnalysis ? [STEP_ANALYSIS] : []),
|
|
236
|
+
...(includeDocuments ? [STEP_DOCUMENTS] : []),
|
|
237
|
+
...(includeCode ? [STEP_CODE_PREVIEW] : []),
|
|
238
|
+
...(includeCode ? [STEP_CODE_INDEX] : []),
|
|
239
|
+
...(includeDocumentChunks ? [STEP_DOCUMENT_CHUNK] : []),
|
|
240
|
+
STEP_OUTLINE,
|
|
184
241
|
];
|
|
185
242
|
let completedSteps = 0;
|
|
243
|
+
const scopedCodeProgress = (event) => onProgress?.({ ...event, scope: event.scope ?? 'code' });
|
|
244
|
+
// Resolves the code index either through a persistent store (cacheDir/store
|
|
245
|
+
// options — cache hits skip parsing, misses build in-process) or through the
|
|
246
|
+
// pre-existing isolated subprocess build.
|
|
247
|
+
const acquireCodeIndex = async () => {
|
|
248
|
+
const slim = options.slimCodeIndex ?? true;
|
|
249
|
+
if (options.store || options.cacheDir) {
|
|
250
|
+
const store = options.store ?? openCodeIndexStore({
|
|
251
|
+
repoRoot: resolvedRoot,
|
|
252
|
+
cacheDir: options.cacheDir,
|
|
253
|
+
excludeDir: shared.excludeDir,
|
|
254
|
+
noGitignore: shared.noGitignore,
|
|
255
|
+
ignorePatterns: shared.ignorePatterns,
|
|
256
|
+
overlayIgnorePatterns: shared.overlayIgnorePatterns,
|
|
257
|
+
contentMode: options.contentMode ?? (slim ? 'none' : 'excerpt'),
|
|
258
|
+
maxFiles: options.maxCodeFiles,
|
|
259
|
+
maxBytes: options.maxCodeBytes,
|
|
260
|
+
onBudgetExceeded: 'truncate',
|
|
261
|
+
});
|
|
262
|
+
const session = await store.getSession({
|
|
263
|
+
strategy: options.cacheStrategy ?? 'ensure-fresh',
|
|
264
|
+
signal: options.signal,
|
|
265
|
+
onProgress: scopedCodeProgress,
|
|
266
|
+
});
|
|
267
|
+
return slim ? slimifyIndex(session.getIndex()) : session.getIndex();
|
|
268
|
+
}
|
|
269
|
+
return buildCodebaseIndexIsolated({
|
|
270
|
+
repoRoot: resolvedRoot,
|
|
271
|
+
...shared,
|
|
272
|
+
contentMode: options.contentMode ?? (slim ? 'none' : 'excerpt'),
|
|
273
|
+
maxFiles: options.maxCodeFiles,
|
|
274
|
+
maxBytes: options.maxCodeBytes,
|
|
275
|
+
onBudgetExceeded: 'truncate',
|
|
276
|
+
slim,
|
|
277
|
+
signal: options.signal,
|
|
278
|
+
}, scopedCodeProgress);
|
|
279
|
+
};
|
|
186
280
|
reportStep(onProgress, { phase: 'bundle', detail: 'Workspace bundle started' }, completedSteps, steps.length, startedAt);
|
|
187
281
|
let description;
|
|
188
|
-
if (includeDescription) {
|
|
189
|
-
abortIfNeeded(options.signal);
|
|
190
|
-
try {
|
|
191
|
-
description = await describeWorkspace(resolvedRoot, shared);
|
|
192
|
-
}
|
|
193
|
-
catch (err) {
|
|
194
|
-
const error = err;
|
|
195
|
-
errors.push({ scope: 'description', path: resolvedRoot, message: error.message });
|
|
196
|
-
}
|
|
197
|
-
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
198
|
-
}
|
|
199
282
|
let analysis;
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
283
|
+
let documents;
|
|
284
|
+
let codePreview;
|
|
285
|
+
let codeIndex;
|
|
286
|
+
if (concurrentPhases) {
|
|
287
|
+
// Phases run concurrently as error-capturing tasks. Errors are collected
|
|
288
|
+
// per phase and assembled post-join in the canonical sequential order
|
|
289
|
+
// (description, analysis, documents, code-preview, code) so errors[]
|
|
290
|
+
// content and order match the concurrentPhases: false path. Step
|
|
291
|
+
// completion order may vary; the shared counter stays monotonic and inner
|
|
292
|
+
// per-phase events are already scope-tagged. Once the signal aborts, tasks
|
|
293
|
+
// skip un-started phase bodies and suppress further step emissions; the
|
|
294
|
+
// post-join abortIfNeeded turns the abort into a rejection.
|
|
295
|
+
let caughtAbort;
|
|
296
|
+
const phaseErrors = {
|
|
297
|
+
'description': [],
|
|
298
|
+
'analysis': [],
|
|
299
|
+
'documents': [],
|
|
300
|
+
'code-preview': [],
|
|
301
|
+
'code': [],
|
|
302
|
+
};
|
|
303
|
+
const capture = (scope, err) => {
|
|
304
|
+
if (isOccAbortError(err)) {
|
|
305
|
+
caughtAbort ??= err;
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
210
308
|
const error = err;
|
|
211
|
-
|
|
309
|
+
phaseErrors[scope].push({ scope, path: resolvedRoot, message: error.message });
|
|
310
|
+
};
|
|
311
|
+
const tasks = [];
|
|
312
|
+
if (includeDescription) {
|
|
313
|
+
tasks.push((async () => {
|
|
314
|
+
if (options.signal?.aborted)
|
|
315
|
+
return; // already aborted — skip the phase entirely
|
|
316
|
+
try {
|
|
317
|
+
description = await describeWorkspace(resolvedRoot, shared);
|
|
318
|
+
}
|
|
319
|
+
catch (err) {
|
|
320
|
+
capture('description', err);
|
|
321
|
+
}
|
|
322
|
+
if (options.signal?.aborted)
|
|
323
|
+
return; // aborted mid-phase — suppress the completion event
|
|
324
|
+
reportStep(onProgress, STEP_DESCRIPTION, ++completedSteps, steps.length, startedAt);
|
|
325
|
+
})());
|
|
212
326
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
327
|
+
if (includeAnalysis) {
|
|
328
|
+
tasks.push((async () => {
|
|
329
|
+
if (options.signal?.aborted)
|
|
330
|
+
return; // already aborted — skip the phase entirely
|
|
331
|
+
try {
|
|
332
|
+
analysis = await analyzeWorkspace(resolvedRoot, {
|
|
333
|
+
...shared,
|
|
334
|
+
includeDocuments,
|
|
335
|
+
includeCode,
|
|
336
|
+
}, (event) => onProgress?.({ ...event, scope: event.scope ?? 'workspace' }));
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
capture('analysis', err);
|
|
340
|
+
}
|
|
341
|
+
if (options.signal?.aborted)
|
|
342
|
+
return; // aborted mid-phase — suppress the completion event
|
|
343
|
+
reportStep(onProgress, STEP_ANALYSIS, ++completedSteps, steps.length, startedAt);
|
|
344
|
+
})());
|
|
226
345
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
346
|
+
if (includeDocuments) {
|
|
347
|
+
tasks.push((async () => {
|
|
348
|
+
if (options.signal?.aborted)
|
|
349
|
+
return; // already aborted — skip the phase entirely
|
|
350
|
+
try {
|
|
351
|
+
documents = await inspectWorkspaceDocumentSet(resolvedRoot, {
|
|
352
|
+
...shared,
|
|
353
|
+
includeMarkdown: includeCodeDocumentReferences,
|
|
354
|
+
includeDataFiles: options.includeDataFiles,
|
|
355
|
+
maxFiles: options.maxDocumentFiles,
|
|
356
|
+
maxReferenceFiles: options.maxReferenceFiles,
|
|
357
|
+
}, (event) => onProgress?.({ ...event, scope: event.scope ?? 'document' }));
|
|
358
|
+
}
|
|
359
|
+
catch (err) {
|
|
360
|
+
capture('documents', err);
|
|
361
|
+
}
|
|
362
|
+
if (options.signal?.aborted)
|
|
363
|
+
return; // aborted mid-phase — suppress the completion event
|
|
364
|
+
reportStep(onProgress, STEP_DOCUMENTS, ++completedSteps, steps.length, startedAt);
|
|
365
|
+
})());
|
|
230
366
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
367
|
+
if (includeCode) {
|
|
368
|
+
// Code preview then index — sequential within this task.
|
|
369
|
+
tasks.push((async () => {
|
|
370
|
+
if (options.signal?.aborted)
|
|
371
|
+
return; // already aborted — skip the phase entirely
|
|
372
|
+
try {
|
|
373
|
+
codePreview = await previewCodebaseSize({
|
|
374
|
+
repoRoot: resolvedRoot,
|
|
375
|
+
...shared,
|
|
376
|
+
maxFiles: options.maxCodeFiles,
|
|
377
|
+
maxBytes: options.maxCodeBytes,
|
|
378
|
+
signal: options.signal,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
catch (err) {
|
|
382
|
+
capture('code-preview', err);
|
|
383
|
+
}
|
|
384
|
+
if (options.signal?.aborted)
|
|
385
|
+
return; // aborted mid-phase — suppress events and skip the index build
|
|
386
|
+
reportStep(onProgress, STEP_CODE_PREVIEW, ++completedSteps, steps.length, startedAt);
|
|
387
|
+
if (options.signal?.aborted)
|
|
388
|
+
return; // abort fired from the preview step emission
|
|
389
|
+
try {
|
|
390
|
+
codeIndex = await acquireCodeIndex();
|
|
391
|
+
}
|
|
392
|
+
catch (err) {
|
|
393
|
+
capture('code', err);
|
|
394
|
+
}
|
|
395
|
+
if (options.signal?.aborted)
|
|
396
|
+
return; // aborted mid-phase — suppress the completion event
|
|
397
|
+
reportStep(onProgress, STEP_CODE_INDEX, ++completedSteps, steps.length, startedAt);
|
|
398
|
+
})());
|
|
399
|
+
}
|
|
400
|
+
await Promise.all(tasks);
|
|
401
|
+
errors.push(...phaseErrors.description, ...phaseErrors.analysis, ...phaseErrors.documents, ...phaseErrors['code-preview'], ...phaseErrors.code);
|
|
236
402
|
abortIfNeeded(options.signal);
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
403
|
+
if (caughtAbort)
|
|
404
|
+
throw caughtAbort;
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
if (includeDescription) {
|
|
408
|
+
abortIfNeeded(options.signal);
|
|
409
|
+
try {
|
|
410
|
+
description = await describeWorkspace(resolvedRoot, shared);
|
|
411
|
+
}
|
|
412
|
+
catch (err) {
|
|
413
|
+
const error = err;
|
|
414
|
+
errors.push({ scope: 'description', path: resolvedRoot, message: error.message });
|
|
415
|
+
}
|
|
416
|
+
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
245
417
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
418
|
+
if (includeAnalysis) {
|
|
419
|
+
abortIfNeeded(options.signal);
|
|
420
|
+
try {
|
|
421
|
+
analysis = await analyzeWorkspace(resolvedRoot, {
|
|
422
|
+
...shared,
|
|
423
|
+
includeDocuments,
|
|
424
|
+
includeCode,
|
|
425
|
+
}, (event) => onProgress?.({ ...event, scope: event.scope ?? 'workspace' }));
|
|
426
|
+
}
|
|
427
|
+
catch (err) {
|
|
428
|
+
const error = err;
|
|
429
|
+
errors.push({ scope: 'analysis', path: resolvedRoot, message: error.message });
|
|
430
|
+
}
|
|
431
|
+
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
249
432
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
433
|
+
if (includeDocuments) {
|
|
434
|
+
abortIfNeeded(options.signal);
|
|
435
|
+
try {
|
|
436
|
+
documents = await inspectWorkspaceDocumentSet(resolvedRoot, {
|
|
437
|
+
...shared,
|
|
438
|
+
includeMarkdown: includeCodeDocumentReferences,
|
|
439
|
+
includeDataFiles: options.includeDataFiles,
|
|
440
|
+
maxFiles: options.maxDocumentFiles,
|
|
441
|
+
maxReferenceFiles: options.maxReferenceFiles,
|
|
442
|
+
}, (event) => onProgress?.({ ...event, scope: event.scope ?? 'document' }));
|
|
443
|
+
}
|
|
444
|
+
catch (err) {
|
|
445
|
+
const error = err;
|
|
446
|
+
errors.push({ scope: 'documents', path: resolvedRoot, message: error.message });
|
|
447
|
+
}
|
|
448
|
+
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
264
449
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
450
|
+
if (includeCode) {
|
|
451
|
+
abortIfNeeded(options.signal);
|
|
452
|
+
try {
|
|
453
|
+
codePreview = await previewCodebaseSize({
|
|
454
|
+
repoRoot: resolvedRoot,
|
|
455
|
+
...shared,
|
|
456
|
+
maxFiles: options.maxCodeFiles,
|
|
457
|
+
maxBytes: options.maxCodeBytes,
|
|
458
|
+
signal: options.signal,
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
catch (err) {
|
|
462
|
+
const error = err;
|
|
463
|
+
errors.push({ scope: 'code-preview', path: resolvedRoot, message: error.message });
|
|
464
|
+
}
|
|
465
|
+
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
466
|
+
abortIfNeeded(options.signal);
|
|
467
|
+
try {
|
|
468
|
+
codeIndex = await acquireCodeIndex();
|
|
469
|
+
}
|
|
470
|
+
catch (err) {
|
|
471
|
+
const error = err;
|
|
472
|
+
errors.push({ scope: 'code', path: resolvedRoot, message: error.message });
|
|
473
|
+
}
|
|
474
|
+
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
268
475
|
}
|
|
269
|
-
reportStep(onProgress, steps[completedSteps], ++completedSteps, steps.length, startedAt);
|
|
270
476
|
}
|
|
271
477
|
const documentChunks = includeDocumentChunks
|
|
272
478
|
? await buildDocumentChunks(documents, options, onProgress)
|