@cesarandreslopez/occ 0.8.10 → 0.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/dist/src/code/build.d.ts +49 -1
- package/dist/src/code/build.js +127 -33
- package/dist/src/code/build.js.map +1 -1
- package/dist/src/code/chunk.d.ts +10 -0
- package/dist/src/code/chunk.js +14 -1
- package/dist/src/code/chunk.js.map +1 -1
- package/dist/src/code/discover.d.ts +13 -0
- package/dist/src/code/discover.js +49 -2
- package/dist/src/code/discover.js.map +1 -1
- package/dist/src/code/incremental.d.ts +75 -0
- package/dist/src/code/incremental.js +257 -0
- package/dist/src/code/incremental.js.map +1 -0
- package/dist/src/code/map.d.ts +8 -1
- package/dist/src/code/map.js +13 -3
- package/dist/src/code/map.js.map +1 -1
- package/dist/src/code/parsers.d.ts +1 -0
- package/dist/src/code/parsers.js +1 -1
- package/dist/src/code/parsers.js.map +1 -1
- package/dist/src/code/rank.d.ts +14 -0
- package/dist/src/code/rank.js +58 -0
- package/dist/src/code/rank.js.map +1 -1
- package/dist/src/code/store.d.ts +20 -0
- package/dist/src/code/store.js +262 -109
- package/dist/src/code/store.js.map +1 -1
- package/dist/src/doc/chunk.d.ts +17 -0
- package/dist/src/doc/chunk.js +47 -18
- package/dist/src/doc/chunk.js.map +1 -1
- package/dist/src/doc/inspect-docx.js +13 -3
- package/dist/src/doc/inspect-docx.js.map +1 -1
- package/dist/src/doc/inspect-pdf.js +12 -17
- package/dist/src/doc/inspect-pdf.js.map +1 -1
- package/dist/src/index.d.ts +13 -8
- package/dist/src/index.js +9 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/inspect/xlsx-cells.d.ts +8 -0
- package/dist/src/inspect/xlsx-cells.js +17 -2
- package/dist/src/inspect/xlsx-cells.js.map +1 -1
- package/dist/src/markdown/convert.js +45 -27
- package/dist/src/markdown/convert.js.map +1 -1
- package/dist/src/parsers/docx.js +10 -2
- package/dist/src/parsers/docx.js.map +1 -1
- package/dist/src/parsers/index.js +5 -0
- package/dist/src/parsers/index.js.map +1 -1
- package/dist/src/parsers/pdf.js +12 -32
- package/dist/src/parsers/pdf.js.map +1 -1
- package/dist/src/parsers/xlsx.js +11 -1
- package/dist/src/parsers/xlsx.js.map +1 -1
- package/dist/src/progress-event.d.ts +3 -1
- package/dist/src/progress-event.js +32 -1
- package/dist/src/progress-event.js.map +1 -1
- package/dist/src/sheet/inspect.js +20 -10
- package/dist/src/sheet/inspect.js.map +1 -1
- package/dist/src/table/inspect-docx.js +10 -1
- package/dist/src/table/inspect-docx.js.map +1 -1
- package/dist/src/table/inspect-xlsx.js +15 -5
- package/dist/src/table/inspect-xlsx.js.map +1 -1
- package/dist/src/utils.d.ts +24 -0
- package/dist/src/utils.js +91 -2
- package/dist/src/utils.js.map +1 -1
- package/dist/src/workspace/bundle.d.ts +51 -0
- package/dist/src/workspace/bundle.js +405 -93
- package/dist/src/workspace/bundle.js.map +1 -1
- package/dist/src/workspace/documents.d.ts +4 -0
- package/dist/src/workspace/documents.js +15 -2
- package/dist/src/workspace/documents.js.map +1 -1
- package/dist/src/workspace/prepare-types.d.ts +15 -0
- package/dist/src/workspace/prepare-types.js +18 -1
- package/dist/src/workspace/prepare-types.js.map +1 -1
- package/dist/src/workspace/prepare.js +10 -2
- package/dist/src/workspace/prepare.js.map +1 -1
- package/package.json +9 -2
package/dist/src/utils.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import fg from 'fast-glob';
|
|
2
2
|
import ignore from 'ignore';
|
|
3
|
-
import officeparser from 'officeparser';
|
|
4
3
|
import path from 'node:path';
|
|
5
4
|
import { readFile } from 'node:fs/promises';
|
|
6
5
|
import { z } from 'zod';
|
|
@@ -122,6 +121,39 @@ export function abortIfNeeded(signal) {
|
|
|
122
121
|
throw new OccAbortError();
|
|
123
122
|
}
|
|
124
123
|
}
|
|
124
|
+
/**
|
|
125
|
+
* Map `items` through an async `fn` with at most `concurrency` calls in flight.
|
|
126
|
+
*
|
|
127
|
+
* Results are returned in input order (each worker writes into its item's slot).
|
|
128
|
+
* Non-finite `concurrency` (NaN/Infinity) falls back to 1. The signal is checked
|
|
129
|
+
* before each item is dispatched; an abort surfaces as OccAbortError. The first
|
|
130
|
+
* rejection from `fn` propagates and stops further dispatch (in-flight items may
|
|
131
|
+
* still complete) — callers that need per-item error capture should catch inside `fn`.
|
|
132
|
+
*/
|
|
133
|
+
export async function mapWithConcurrency(items, concurrency, fn, signal) {
|
|
134
|
+
const results = new Array(items.length);
|
|
135
|
+
if (items.length === 0)
|
|
136
|
+
return results;
|
|
137
|
+
const safeConcurrency = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
|
|
138
|
+
const workerCount = Math.max(1, Math.min(safeConcurrency, items.length));
|
|
139
|
+
let cursor = 0;
|
|
140
|
+
let failed = false;
|
|
141
|
+
async function worker() {
|
|
142
|
+
while (!failed && cursor < items.length) {
|
|
143
|
+
abortIfNeeded(signal);
|
|
144
|
+
const index = cursor++;
|
|
145
|
+
try {
|
|
146
|
+
results[index] = await fn(items[index], index);
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
failed = true;
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
155
|
+
return results;
|
|
156
|
+
}
|
|
125
157
|
export function sanitizeForkExecArgv(execArgv = process.execArgv) {
|
|
126
158
|
const sanitized = [];
|
|
127
159
|
for (let i = 0; i < execArgv.length; i++) {
|
|
@@ -149,8 +181,65 @@ export async function writeStream(stream, text) {
|
|
|
149
181
|
});
|
|
150
182
|
});
|
|
151
183
|
}
|
|
184
|
+
/** Noisy pdf.js output prefixes suppressed while parsing PDFs (font parsing, deprecated API, etc.). */
|
|
185
|
+
export const PDF_LOG_SUPPRESSED_PREFIXES = ['Warning: ', 'Info: ', 'Deprecated API usage: '];
|
|
186
|
+
// Reference-counted console.log suppression shared by every PDF call site so
|
|
187
|
+
// concurrent (and nested) suppressed sections never clobber each other's restore:
|
|
188
|
+
// the original console.log is captured once when the depth goes 0 → 1 and
|
|
189
|
+
// restored only when the last active section ends (depth back to 0).
|
|
190
|
+
const activeSuppressionPrefixLists = [];
|
|
191
|
+
let consoleLogSuppressionDepth = 0;
|
|
192
|
+
let savedOriginalConsoleLog;
|
|
193
|
+
/**
|
|
194
|
+
* Run `fn` while console.log lines starting with any of `prefixes` are dropped.
|
|
195
|
+
* Lines that match no active prefix list pass through unchanged. Safe to overlap
|
|
196
|
+
* and nest across concurrent callers.
|
|
197
|
+
*/
|
|
198
|
+
export async function suppressConsoleLogWhile(prefixes, fn) {
|
|
199
|
+
if (consoleLogSuppressionDepth++ === 0) {
|
|
200
|
+
const originalLog = console.log;
|
|
201
|
+
savedOriginalConsoleLog = originalLog;
|
|
202
|
+
console.log = (...args) => {
|
|
203
|
+
const first = args[0];
|
|
204
|
+
if (typeof first === 'string'
|
|
205
|
+
&& activeSuppressionPrefixLists.some(list => list.some(prefix => first.startsWith(prefix)))) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
originalLog.apply(console, args);
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
activeSuppressionPrefixLists.push(prefixes);
|
|
212
|
+
try {
|
|
213
|
+
return await fn();
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
const index = activeSuppressionPrefixLists.indexOf(prefixes);
|
|
217
|
+
if (index >= 0)
|
|
218
|
+
activeSuppressionPrefixLists.splice(index, 1);
|
|
219
|
+
if (--consoleLogSuppressionDepth === 0 && savedOriginalConsoleLog) {
|
|
220
|
+
console.log = savedOriginalConsoleLog;
|
|
221
|
+
savedOriginalConsoleLog = undefined;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Rethrow a lazy `import()` failure with the offending package named, so a
|
|
227
|
+
* missing/broken heavy dependency surfaces as an actionable diagnostic instead
|
|
228
|
+
* of a generic per-file parse failure.
|
|
229
|
+
*/
|
|
230
|
+
export function wrapDependencyLoadError(pkg, error) {
|
|
231
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
232
|
+
throw new Error(`occ dependency '${pkg}' failed to load: ${message}`, { cause: error });
|
|
233
|
+
}
|
|
234
|
+
let officeparserModule;
|
|
235
|
+
async function loadOfficeparser() {
|
|
236
|
+
officeparserModule ??= import('officeparser')
|
|
237
|
+
.catch((error) => wrapDependencyLoadError('officeparser', error));
|
|
238
|
+
return officeparserModule;
|
|
239
|
+
}
|
|
152
240
|
export async function extractOfficeText(input) {
|
|
153
|
-
const
|
|
241
|
+
const { OfficeParser } = await loadOfficeparser();
|
|
242
|
+
const raw = await OfficeParser.parseOffice(input);
|
|
154
243
|
if (typeof raw === 'string')
|
|
155
244
|
return raw;
|
|
156
245
|
if (raw && typeof raw.toText === 'function') {
|
package/dist/src/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAA4B;IACvD,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IACzB,OAAO,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoB;IACvD,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,UAAU;YAAE,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,UAAU,EAAE,EAAE,MAAM,UAAU,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAA2B;IACvD,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAa,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAE1E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AAE5G,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAmBvD,SAAS,iBAAiB,CAAC,OAAgB,EAAE,MAAqD;IAChG,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,YAAoB,EAAE,QAAgB;IAC7D,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,KAAK,UAAU,2BAA2B,CACxC,OAAe,EACf,UAAoB;IAEpB,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,EAAE;QAC/D,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,mBAAmB,EAAE,KAAK;QAC1B,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC;QACxC,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC7E,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjD,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;SAC1E,CAAC;IACJ,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC7E,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAChF,OAAO,SAAS,GAAG,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,UAAgC,EAAE;IAElC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,WAAW;QAC3C,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,2BAA2B,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,aAAa,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;QAC/E,CAAC,CAAC,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC;QAClE,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,cAAc,GAAG,OAAO,CAAC,qBAAqB,IAAI,OAAO,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC;QAC9F,CAAC,CAAC,MAAM,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,OAAO,CAAC,YAAoB;YAC1B,MAAM,cAAc,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACjF,IAAI,WAAW,GAAG,KAAK,CAAC;YAExB,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;gBACtC,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBAC/D,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBAC5C,CAAC,CAAC,cAAc;oBAChB,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,cAAc,EAAE,CAAC;gBACnB,cAAc,GAAG,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACjF,CAAC;YAED,OAAO,WAAW,IAAI,cAAc,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAoB;IAChD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,aAAa,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAmB,EACnB,WAAmB,EACnB,EAA0C,EAC1C,MAAoB;IAEpB,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEvC,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,UAAU,MAAM;QACnB,OAAO,CAAC,MAAM,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxC,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,WAAqB,OAAO,CAAC,QAAQ;IACxE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,GAAG,KAAK,cAAc,EAAE,CAAC;YAC3B,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;YAAE,SAAS;QAC9C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAA6B,EAAE,IAAY;IAC3E,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAoB,EAAE,EAAE;YAC1C,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;;gBACpB,OAAO,EAAE,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uGAAuG;AACvG,MAAM,CAAC,MAAM,2BAA2B,GAAsB,CAAC,WAAW,EAAE,QAAQ,EAAE,wBAAwB,CAAC,CAAC;AAEhH,6EAA6E;AAC7E,kFAAkF;AAClF,0EAA0E;AAC1E,qEAAqE;AACrE,MAAM,4BAA4B,GAA0B,EAAE,CAAC;AAC/D,IAAI,0BAA0B,GAAG,CAAC,CAAC;AACnC,IAAI,uBAAuD,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAA2B,EAC3B,EAAoB;IAEpB,IAAI,0BAA0B,EAAE,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;QAChC,uBAAuB,GAAG,WAAW,CAAC;QACtC,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,OAAO,KAAK,KAAK,QAAQ;mBACxB,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9F,OAAO;YACT,CAAC;YACD,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE5C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,GAAG,4BAA4B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,KAAK,IAAI,CAAC;YAAE,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,EAAE,0BAA0B,KAAK,CAAC,IAAI,uBAAuB,EAAE,CAAC;YAClE,OAAO,CAAC,GAAG,GAAG,uBAAuB,CAAC;YACtC,uBAAuB,GAAG,SAAS,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAW,EAAE,KAAc;IACjE,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,qBAAqB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AAC1F,CAAC;AAOD,IAAI,kBAA2D,CAAC;AAEhE,KAAK,UAAU,gBAAgB;IAC7B,kBAAkB,KAAK,MAAM,CAAC,cAAc,CAAC;SAC1C,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,uBAAuB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7E,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAsB;IAC5D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,GAAG,IAAI,OAAQ,GAA4B,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAE,GAAgC,CAAC,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACxE,CAAC"}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { type CodeIndexStore, type StoreStrategy } from '../code/store.js';
|
|
2
|
+
import { type CodeChunk, type CodeChunkOptions } from '../code/chunk.js';
|
|
1
3
|
import { type DocumentChunk, type DocumentChunkOptions } from '../doc/chunk.js';
|
|
2
4
|
import type { CodebaseIndex, ContentMode } from '../code/types.js';
|
|
3
5
|
import type { CodebaseIndexSlim } from '../code/slim.js';
|
|
@@ -15,6 +17,8 @@ export interface WorkspaceBundleOptions {
|
|
|
15
17
|
includeDocuments?: boolean;
|
|
16
18
|
includeCode?: boolean;
|
|
17
19
|
includeDocumentChunks?: boolean;
|
|
20
|
+
includeCodeChunks?: boolean;
|
|
21
|
+
includeCorpusChunks?: boolean;
|
|
18
22
|
includeCodeDocumentReferences?: boolean;
|
|
19
23
|
includeDataFiles?: boolean;
|
|
20
24
|
contentMode?: ContentMode;
|
|
@@ -24,8 +28,34 @@ export interface WorkspaceBundleOptions {
|
|
|
24
28
|
maxCodeFiles?: number;
|
|
25
29
|
maxCodeBytes?: number;
|
|
26
30
|
maxDocumentChunks?: number;
|
|
31
|
+
maxCodeChunks?: number;
|
|
32
|
+
maxCorpusChunks?: number;
|
|
27
33
|
maxCodeDocumentReferences?: number;
|
|
28
34
|
chunk?: Pick<DocumentChunkOptions, 'maxTokens' | 'overlapTokens' | 'countTokens'>;
|
|
35
|
+
codeChunk?: Pick<CodeChunkOptions, 'maxTokens' | 'overlapTokens' | 'countTokens' | 'files'>;
|
|
36
|
+
/**
|
|
37
|
+
* Persistent code-index cache directory. When set (and no `store` is
|
|
38
|
+
* provided), the bundle resolves its code index through
|
|
39
|
+
* `openCodeIndexStore` instead of `buildCodebaseIndexIsolated`: cache hits
|
|
40
|
+
* skip parsing entirely, and cache misses build **in-process** (no
|
|
41
|
+
* subprocess isolation — this is an explicit opt-in). Cache entries are
|
|
42
|
+
* keyed by `(contentMode, build-option fingerprint)`, so the bundle options
|
|
43
|
+
* must match the consumer's own store options to share an entry — when in
|
|
44
|
+
* doubt, pass `store` directly (recommended).
|
|
45
|
+
*/
|
|
46
|
+
cacheDir?: string;
|
|
47
|
+
/** Store strategy used with `cacheDir`/`store` (default `'ensure-fresh'`). */
|
|
48
|
+
cacheStrategy?: StoreStrategy;
|
|
49
|
+
/**
|
|
50
|
+
* Bring-your-own code index store. Takes precedence over `cacheDir`; the
|
|
51
|
+
* bundle never disposes a caller-provided store.
|
|
52
|
+
*/
|
|
53
|
+
store?: CodeIndexStore;
|
|
54
|
+
/**
|
|
55
|
+
* Run the description / analysis / documents / code phases concurrently
|
|
56
|
+
* (default `true`). `false` restores the literal sequential path.
|
|
57
|
+
*/
|
|
58
|
+
concurrentPhases?: boolean;
|
|
29
59
|
signal?: AbortSignal;
|
|
30
60
|
onProgress?: OnProgress;
|
|
31
61
|
}
|
|
@@ -46,6 +76,25 @@ export interface WorkspaceCodeDocumentReference {
|
|
|
46
76
|
line: number;
|
|
47
77
|
context: string;
|
|
48
78
|
}
|
|
79
|
+
export interface WorkspaceCorpusChunk {
|
|
80
|
+
kind: 'code' | 'document';
|
|
81
|
+
chunkId: string;
|
|
82
|
+
filePath: string;
|
|
83
|
+
relativePath: string;
|
|
84
|
+
title: string;
|
|
85
|
+
content: string;
|
|
86
|
+
chunkIndex: number;
|
|
87
|
+
startLine?: number;
|
|
88
|
+
endLine?: number;
|
|
89
|
+
language?: string;
|
|
90
|
+
format?: string;
|
|
91
|
+
headingPath?: string[];
|
|
92
|
+
anchor?: string;
|
|
93
|
+
symbolNames?: string[];
|
|
94
|
+
symbolTypes?: string[];
|
|
95
|
+
tokenEstimate?: number;
|
|
96
|
+
wordCount?: number;
|
|
97
|
+
}
|
|
49
98
|
export interface WorkspaceBundle {
|
|
50
99
|
schemaVersion: 1;
|
|
51
100
|
rootDir: string;
|
|
@@ -56,6 +105,8 @@ export interface WorkspaceBundle {
|
|
|
56
105
|
codePreview?: CodebaseSizePreview;
|
|
57
106
|
codeIndex?: CodebaseIndex | CodebaseIndexSlim;
|
|
58
107
|
documentChunks?: DocumentChunk[];
|
|
108
|
+
codeChunks?: CodeChunk[];
|
|
109
|
+
corpusChunks?: WorkspaceCorpusChunk[];
|
|
59
110
|
outline: WorkspaceOutlineNode;
|
|
60
111
|
codeDocumentReferences: WorkspaceCodeDocumentReference[];
|
|
61
112
|
errors: Array<{
|