@cesarandreslopez/occ 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/src/code/build.d.ts +49 -1
- package/dist/src/code/build.js +127 -33
- package/dist/src/code/build.js.map +1 -1
- package/dist/src/code/chunk.d.ts +10 -0
- package/dist/src/code/chunk.js +14 -1
- package/dist/src/code/chunk.js.map +1 -1
- package/dist/src/code/command.js +16 -0
- package/dist/src/code/command.js.map +1 -1
- package/dist/src/code/discover.d.ts +13 -0
- package/dist/src/code/discover.js +49 -2
- package/dist/src/code/discover.js.map +1 -1
- package/dist/src/code/focus.d.ts +31 -0
- package/dist/src/code/focus.js +192 -0
- package/dist/src/code/focus.js.map +1 -0
- package/dist/src/code/incremental.d.ts +75 -0
- package/dist/src/code/incremental.js +257 -0
- package/dist/src/code/incremental.js.map +1 -0
- package/dist/src/code/map.d.ts +36 -1
- package/dist/src/code/map.js +83 -4
- package/dist/src/code/map.js.map +1 -1
- package/dist/src/code/output.js +28 -4
- package/dist/src/code/output.js.map +1 -1
- package/dist/src/code/parsers.d.ts +1 -0
- package/dist/src/code/parsers.js +1 -1
- package/dist/src/code/parsers.js.map +1 -1
- package/dist/src/code/rank.d.ts +14 -0
- package/dist/src/code/rank.js +58 -0
- package/dist/src/code/rank.js.map +1 -1
- package/dist/src/code/store.d.ts +20 -0
- package/dist/src/code/store.js +262 -109
- package/dist/src/code/store.js.map +1 -1
- package/dist/src/doc/chunk.d.ts +17 -0
- package/dist/src/doc/chunk.js +47 -18
- package/dist/src/doc/chunk.js.map +1 -1
- package/dist/src/doc/inspect-docx.js +13 -3
- package/dist/src/doc/inspect-docx.js.map +1 -1
- package/dist/src/doc/inspect-pdf.js +12 -17
- package/dist/src/doc/inspect-pdf.js.map +1 -1
- package/dist/src/index.d.ts +13 -8
- package/dist/src/index.js +9 -5
- package/dist/src/index.js.map +1 -1
- package/dist/src/inspect/xlsx-cells.d.ts +8 -0
- package/dist/src/inspect/xlsx-cells.js +17 -2
- package/dist/src/inspect/xlsx-cells.js.map +1 -1
- package/dist/src/markdown/convert.js +45 -27
- package/dist/src/markdown/convert.js.map +1 -1
- package/dist/src/parsers/docx.js +10 -2
- package/dist/src/parsers/docx.js.map +1 -1
- package/dist/src/parsers/index.js +5 -0
- package/dist/src/parsers/index.js.map +1 -1
- package/dist/src/parsers/pdf.js +12 -32
- package/dist/src/parsers/pdf.js.map +1 -1
- package/dist/src/parsers/xlsx.js +11 -1
- package/dist/src/parsers/xlsx.js.map +1 -1
- package/dist/src/progress-event.d.ts +3 -1
- package/dist/src/progress-event.js +31 -1
- package/dist/src/progress-event.js.map +1 -1
- package/dist/src/sheet/inspect.js +20 -10
- package/dist/src/sheet/inspect.js.map +1 -1
- package/dist/src/table/inspect-docx.js +10 -1
- package/dist/src/table/inspect-docx.js.map +1 -1
- package/dist/src/table/inspect-xlsx.js +15 -5
- package/dist/src/table/inspect-xlsx.js.map +1 -1
- package/dist/src/utils.d.ts +24 -0
- package/dist/src/utils.js +91 -2
- package/dist/src/utils.js.map +1 -1
- package/dist/src/workspace/bundle.d.ts +24 -0
- package/dist/src/workspace/bundle.js +303 -97
- package/dist/src/workspace/bundle.js.map +1 -1
- package/dist/src/workspace/documents.d.ts +4 -0
- package/dist/src/workspace/documents.js +15 -2
- package/dist/src/workspace/documents.js.map +1 -1
- package/dist/src/workspace/prepare-types.d.ts +15 -0
- package/dist/src/workspace/prepare-types.js +18 -1
- package/dist/src/workspace/prepare-types.js.map +1 -1
- package/dist/src/workspace/prepare.js +10 -2
- package/dist/src/workspace/prepare.js.map +1 -1
- package/package.json +9 -2
package/dist/src/doc/chunk.js
CHANGED
|
@@ -33,15 +33,31 @@ function headingPathFor(node, nodesById) {
|
|
|
33
33
|
}
|
|
34
34
|
return pathParts;
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Split a section's content into line-aligned chunks that respect a token budget.
|
|
38
|
+
*
|
|
39
|
+
* Per-line token counts are precomputed once and budget decisions use a running
|
|
40
|
+
* sum, so `countTokens` runs O(lines + chunks) times instead of re-counting the
|
|
41
|
+
* accumulated chunk on every line. Each precomputed count includes the line's
|
|
42
|
+
* trailing `'\n'` separator (except the seed's final line, which has none), so
|
|
43
|
+
* the running sum upper-bounds the joined+trimmed chunk under tokenizers that
|
|
44
|
+
* charge for newlines — emitted chunks stay within `maxTokens`, at the cost of
|
|
45
|
+
* slightly conservative boundaries that may land a line earlier than an exact
|
|
46
|
+
* count would allow. The emitted `tokenEstimate` is still the exact
|
|
47
|
+
* `countTokens(content)` of each joined+trimmed chunk.
|
|
48
|
+
*/
|
|
36
49
|
function splitByTokenBudget(seed, maxTokens, overlapTokens, countTokens) {
|
|
37
50
|
const lines = seed.content.split('\n');
|
|
51
|
+
const lineTokens = lines.map((line, index) => countTokens(index < lines.length - 1 ? `${line}\n` : line));
|
|
38
52
|
const chunks = [];
|
|
39
53
|
let currentLines = [];
|
|
40
54
|
let currentStartLine = seed.startLine;
|
|
55
|
+
let runningTokens = 0;
|
|
41
56
|
const emit = () => {
|
|
42
57
|
const content = currentLines.join('\n').trim();
|
|
43
58
|
if (!content) {
|
|
44
59
|
currentLines = [];
|
|
60
|
+
runningTokens = 0;
|
|
45
61
|
return;
|
|
46
62
|
}
|
|
47
63
|
const emittedStartLine = currentStartLine;
|
|
@@ -56,35 +72,36 @@ function splitByTokenBudget(seed, maxTokens, overlapTokens, countTokens) {
|
|
|
56
72
|
});
|
|
57
73
|
if (overlapTokens <= 0 || (currentLines.length === 1 && tokenEstimate > overlapTokens)) {
|
|
58
74
|
currentLines = [];
|
|
75
|
+
runningTokens = 0;
|
|
59
76
|
return;
|
|
60
77
|
}
|
|
78
|
+
// Walk backwards over the precomputed per-line counts: always carry the
|
|
79
|
+
// last line, then keep extending the overlap while it still fits.
|
|
80
|
+
const firstLineIndex = emittedStartLine - seed.startLine;
|
|
61
81
|
const overlapLines = [];
|
|
62
82
|
let overlapEstimate = 0;
|
|
63
83
|
for (let index = currentLines.length - 1; index >= 0; index--) {
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
if (overlapEstimate > 0 && nextEstimate > overlapTokens)
|
|
84
|
+
const lineEstimate = lineTokens[firstLineIndex + index];
|
|
85
|
+
if (overlapEstimate > 0 && overlapEstimate + lineEstimate > overlapTokens)
|
|
67
86
|
break;
|
|
68
87
|
overlapLines.unshift(currentLines[index]);
|
|
69
|
-
overlapEstimate
|
|
88
|
+
overlapEstimate += lineEstimate;
|
|
70
89
|
}
|
|
71
90
|
currentStartLine = emittedStartLine + emittedLineCount - overlapLines.length;
|
|
72
91
|
currentLines = overlapLines;
|
|
92
|
+
runningTokens = overlapEstimate;
|
|
73
93
|
};
|
|
74
94
|
for (let index = 0; index < lines.length; index++) {
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
const nextContent = nextLines.join('\n').trim();
|
|
78
|
-
const nextEstimate = countTokens(nextContent);
|
|
79
|
-
if (currentLines.length > 0 && nextEstimate > maxTokens) {
|
|
95
|
+
const lineEstimate = lineTokens[index];
|
|
96
|
+
if (currentLines.length > 0 && runningTokens + lineEstimate > maxTokens) {
|
|
80
97
|
emit();
|
|
81
98
|
}
|
|
82
99
|
if (currentLines.length === 0) {
|
|
83
100
|
currentStartLine = seed.startLine + index;
|
|
84
101
|
}
|
|
85
|
-
currentLines.push(
|
|
86
|
-
|
|
87
|
-
if (currentLines.length === 1 &&
|
|
102
|
+
currentLines.push(lines[index]);
|
|
103
|
+
runningTokens += lineEstimate;
|
|
104
|
+
if (currentLines.length === 1 && runningTokens > maxTokens) {
|
|
88
105
|
emit();
|
|
89
106
|
}
|
|
90
107
|
}
|
|
@@ -130,16 +147,21 @@ function buildChunkSeeds(markdown, filePath) {
|
|
|
130
147
|
startLine: 1,
|
|
131
148
|
}];
|
|
132
149
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
150
|
+
/**
|
|
151
|
+
* Chunk already-converted markdown without touching disk.
|
|
152
|
+
*
|
|
153
|
+
* Useful when the markdown was just produced elsewhere (for example
|
|
154
|
+
* `inspectWorkspaceDocumentSet` with `includeMarkdown`) — it skips the
|
|
155
|
+
* re-read + re-convert that `chunkDocument` performs. For the same markdown
|
|
156
|
+
* and options the output is identical to `chunkDocument`.
|
|
157
|
+
*/
|
|
158
|
+
export function chunkDocumentFromMarkdown(markdown, options) {
|
|
159
|
+
const filePath = options.filePath;
|
|
138
160
|
const maxTokens = options.maxTokens ?? 800;
|
|
139
161
|
const overlapTokens = Math.min(options.overlapTokens ?? 80, Math.max(0, maxTokens - 1));
|
|
140
162
|
const countTokens = options.countTokens ?? estimateTokens;
|
|
141
163
|
const relativePath = options.relativePath ?? normalizeRelativePath(filePath);
|
|
142
|
-
const format = getExtension(filePath);
|
|
164
|
+
const format = options.format ?? getExtension(filePath);
|
|
143
165
|
const chunks = [];
|
|
144
166
|
const seeds = buildChunkSeeds(markdown, filePath);
|
|
145
167
|
for (const seed of seeds) {
|
|
@@ -166,4 +188,11 @@ export async function chunkDocument(filePath, options = {}) {
|
|
|
166
188
|
}
|
|
167
189
|
return chunks;
|
|
168
190
|
}
|
|
191
|
+
export async function chunkDocument(filePath, options = {}) {
|
|
192
|
+
const markdown = await documentToMarkdown(filePath);
|
|
193
|
+
if (markdown == null) {
|
|
194
|
+
throw new Error(`Unsupported document format for chunking: ${filePath}`);
|
|
195
|
+
}
|
|
196
|
+
return chunkDocumentFromMarkdown(markdown, { ...options, filePath });
|
|
197
|
+
}
|
|
169
198
|
//# sourceMappingURL=chunk.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/doc/chunk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA0C9E,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,IAAI,GAAG,KAAK;SACf,WAAW,EAAE;SACb,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3B,OAAO,IAAI,IAAI,SAAS,CAAC;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,OAAe,EAAE,SAAiB;IACpD,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACzE,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,SAAqC;IAChF,MAAM,SAAS,GAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAe,EACf,SAAiB,EACjB,aAAqB,EACrB,WAAwC;IAExC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/doc/chunk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA0C9E,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,IAAI,GAAG,KAAK;SACf,WAAW,EAAE;SACb,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3B,OAAO,IAAI,IAAI,SAAS,CAAC;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,OAAe,EAAE,SAAiB;IACpD,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACzE,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,SAAqC;IAChF,MAAM,SAAS,GAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,OAAO,OAAO,CAAC,YAAY,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CACzB,IAAe,EACf,SAAiB,EACjB,aAAqB,EACrB,WAAwC;IAExC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;IACtC,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,YAAY,GAAG,EAAE,CAAC;YAClB,aAAa,GAAG,CAAC,CAAC;YAClB,OAAO;QACT,CAAC;QAED,MAAM,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC;QAC7C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC;YACV,OAAO;YACP,SAAS,EAAE,gBAAgB;YAC3B,OAAO,EAAE,gBAAgB,GAAG,gBAAgB,GAAG,CAAC;YAChD,aAAa;YACb,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC;SAC/B,CAAC,CAAC;QAEH,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,GAAG,aAAa,CAAC,EAAE,CAAC;YACvF,YAAY,GAAG,EAAE,CAAC;YAClB,aAAa,GAAG,CAAC,CAAC;YAClB,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,kEAAkE;QAClE,MAAM,cAAc,GAAG,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;QACzD,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,KAAK,IAAI,KAAK,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9D,MAAM,YAAY,GAAG,UAAU,CAAC,cAAc,GAAG,KAAK,CAAC,CAAC;YACxD,IAAI,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,YAAY,GAAG,aAAa;gBAAE,MAAM;YACjF,YAAY,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1C,eAAe,IAAI,YAAY,CAAC;QAClC,CAAC;QAED,gBAAgB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC;QAC7E,YAAY,GAAG,YAAY,CAAC;QAC5B,aAAa,GAAG,eAAe,CAAC;IAClC,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,GAAG,YAAY,GAAG,SAAS,EAAE,CAAC;YACxE,IAAI,EAAE,CAAC;QACT,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5C,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAChC,aAAa,IAAI,YAAY,CAAC;QAE9B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,GAAG,SAAS,EAAE,CAAC;YAC3D,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,IAAI,EAAE,CAAC;IACT,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,QAAgB,EAAE,QAAgB;IACzD,MAAM,SAAS,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAE3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC;gBACN,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9B,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,QAAQ,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE;gBAC7C,WAAW,EAAE,EAAE;gBACf,SAAS,EAAE,CAAC;aACb,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;YAAE,SAAS;QAE9B,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,WAAW;YACX,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC;QACrB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,CAAC;gBACD,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9B,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,QAAQ,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE;gBAC7C,WAAW,EAAE,EAAE;gBACf,SAAS,EAAE,CAAC;aACb,CAAC,CAAC;AACP,CAAC;AAWD;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAgB,EAAE,OAAyC;IACnG,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,cAAc,CAAC;IAC1D,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QACnF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACvD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,GAAG,YAAY,IAAI,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE;gBACtD,QAAQ;gBACR,YAAY;gBACZ,MAAM;gBACN,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE;gBACrC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,UAAU,EAAE,SAAS,CAAC,SAAS;gBAC/B,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,aAAa,EAAE,SAAS,CAAC,aAAa;gBACtC,SAAS,EAAE,SAAS,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,UAAgC,EAAE;IACtF,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,6CAA6C,QAAQ,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,yBAAyB,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import JSZip from 'jszip';
|
|
2
|
-
import
|
|
3
|
-
import { countWords } from '../utils.js';
|
|
2
|
+
import { countWords, wrapDependencyLoadError } from '../utils.js';
|
|
4
3
|
import { asOptionalString, formatDateLike } from '../inspect/shared.js';
|
|
4
|
+
// Lazy-loaded so importing this module never pulls in mammoth (CJS default = the lib).
|
|
5
|
+
let mammothModule;
|
|
6
|
+
async function loadMammoth() {
|
|
7
|
+
mammothModule ??= import('mammoth')
|
|
8
|
+
.then((mod) => mod.default)
|
|
9
|
+
.catch((error) => wrapDependencyLoadError('mammoth', error));
|
|
10
|
+
return mammothModule;
|
|
11
|
+
}
|
|
5
12
|
function parseXmlText(xml, tag) {
|
|
6
13
|
const regex = new RegExp(`<${tag}[^>]*>([^<]*)</${tag}>`, 'i');
|
|
7
14
|
const match = xml.match(regex);
|
|
@@ -81,6 +88,7 @@ export async function extractDocxRiskFlags(zip) {
|
|
|
81
88
|
return flags;
|
|
82
89
|
}
|
|
83
90
|
export async function extractDocxContentStats(filePath, zip) {
|
|
91
|
+
const mammoth = await loadMammoth();
|
|
84
92
|
const result = await mammoth.extractRawText({ path: filePath });
|
|
85
93
|
const text = result.value || '';
|
|
86
94
|
const words = countWords(text);
|
|
@@ -118,13 +126,15 @@ export async function extractDocxContentStats(filePath, zip) {
|
|
|
118
126
|
return { words, pages, pagesEstimated, paragraphs, characters, tables, images };
|
|
119
127
|
}
|
|
120
128
|
export async function extractDocxPreview(filePath, sampleParagraphs) {
|
|
129
|
+
const mammoth = await loadMammoth();
|
|
121
130
|
const result = await mammoth.convertToHtml({ path: filePath });
|
|
122
131
|
const html = result.value || '';
|
|
123
132
|
if (!html.trim()) {
|
|
124
133
|
return { truncated: false, paragraphs: [] };
|
|
125
134
|
}
|
|
126
135
|
// Use turndown to convert to markdown for heading detection
|
|
127
|
-
const TurndownService = (await import('turndown')
|
|
136
|
+
const TurndownService = (await import('turndown')
|
|
137
|
+
.catch((error) => wrapDependencyLoadError('turndown', error))).default;
|
|
128
138
|
const turndown = new TurndownService({ headingStyle: 'atx' });
|
|
129
139
|
const markdown = turndown.turndown(html);
|
|
130
140
|
return buildPreviewFromMarkdown(markdown, sampleParagraphs);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspect-docx.js","sourceRoot":"","sources":["../../../src/doc/inspect-docx.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,
|
|
1
|
+
{"version":3,"file":"inspect-docx.js","sourceRoot":"","sources":["../../../src/doc/inspect-docx.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAIxE,uFAAuF;AACvF,IAAI,aAAyE,CAAC;AAE9E,KAAK,UAAU,WAAW;IACxB,aAAa,KAAK,MAAM,CAAC,SAAS,CAAC;SAChC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC;SAC1B,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CAAC,GAAW,EAAE,GAAW;IAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,kBAAkB,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/B,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,GAAU;IACpD,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnE,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;QAClE,KAAK,CAAC,OAAO,GAAG,gBAAgB,CAAC,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACtE,KAAK,CAAC,MAAM,GAAG,gBAAgB,CAAC,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACrE,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;QACxE,KAAK,CAAC,cAAc,GAAG,gBAAgB,CAAC,YAAY,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpF,KAAK,CAAC,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC7E,KAAK,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjE,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,OAAO,GAAG,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAU;IACnD,MAAM,KAAK,GAAiB;QAC1B,QAAQ,EAAE,KAAK;QACf,cAAc,EAAE,KAAK;QACrB,UAAU,EAAE,KAAK;QACjB,eAAe,EAAE,KAAK;QACtB,SAAS,EAAE,KAAK;QAChB,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,KAAK;QACb,YAAY,EAAE,KAAK;QACnB,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,iBAAiB;IACjB,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvE,IAAI,WAAW,IAAI,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QACpD,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,6DAA6D;IAC7D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;QACpF,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;QACjD,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,IAAI,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;IAEzD,mBAAmB;IACnB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC;IACzF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;IAExD,2BAA2B;IAC3B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzE,IAAI,YAAY,IAAI,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QACvD,iGAAiG;QACjG,MAAM,aAAa,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC1E,IAAI,aAAa,GAAG,CAAC;YAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,sBAAsB;IACvE,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvE,IAAI,WAAW,IAAI,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QACpD,MAAM,YAAY,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACvE,IAAI,YAAY,GAAG,CAAC;YAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC9C,CAAC;IAED,wBAAwB;IACxB,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACxG,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC;IAEtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,QAAgB,EAAE,GAAU;IACxE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAE/B,kCAAkC;IAClC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACtC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,KAAK,GAAG,MAAM,CAAC;gBACf,cAAc,GAAG,KAAK,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;QAC5C,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,eAAe;IACf,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAEjC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAClF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,gBAAwB;IACjF,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAEhC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IAC9C,CAAC;IAED,4DAA4D;IAC5D,MAAM,eAAe,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC;SAC9C,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,uBAAuB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAClF,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,wBAAwB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAgB,EAAE,gBAAwB;IACjF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAiC,EAAE,CAAC;IAClD,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAC9D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,cAAc;gBACrB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;gBACrB,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM;aACrC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,sDAAsD;YACtD,IAAI,IAAI,GAAG,IAAI,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9F,CAAC,EAAE,CAAC;gBACJ,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChC,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,cAAc;gBACrB,IAAI;gBACJ,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,CAAC;QACjB,CAAC,EAAE,CAAC;IACN,CAAC;IAED,8BAA8B;IAC9B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAc,EACd,gBAAwB;IAOxB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACvE,qBAAqB,CAAC,GAAG,CAAC;QAC1B,oBAAoB,CAAC,GAAG,CAAC;QACzB,uBAAuB,CAAC,QAAQ,EAAE,GAAG,CAAC;QACtC,kBAAkB,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAC/C,CAAC,CAAC;IAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AAC1D,CAAC"}
|
|
@@ -1,12 +1,19 @@
|
|
|
1
|
-
import pdf from 'pdf-parse';
|
|
2
1
|
import { z } from 'zod';
|
|
3
|
-
import { countWords } from '../utils.js';
|
|
2
|
+
import { countWords, PDF_LOG_SUPPRESSED_PREFIXES, suppressConsoleLogWhile, wrapDependencyLoadError } from '../utils.js';
|
|
4
3
|
import { asOptionalString, formatDateLike } from '../inspect/shared.js';
|
|
5
4
|
const PdfParseResultSchema = z.object({
|
|
6
5
|
text: z.string(),
|
|
7
6
|
numpages: z.number(),
|
|
8
7
|
info: z.record(z.string(), z.unknown()).default({}),
|
|
9
8
|
});
|
|
9
|
+
// Lazy-loaded so importing this module never pulls in pdf-parse (CJS default = the parse fn).
|
|
10
|
+
let pdfParseModule;
|
|
11
|
+
async function loadPdfParse() {
|
|
12
|
+
pdfParseModule ??= import('pdf-parse')
|
|
13
|
+
.then((mod) => mod.default)
|
|
14
|
+
.catch((error) => wrapDependencyLoadError('pdf-parse', error));
|
|
15
|
+
return pdfParseModule;
|
|
16
|
+
}
|
|
10
17
|
function parsePdfDate(value) {
|
|
11
18
|
if (!value)
|
|
12
19
|
return undefined;
|
|
@@ -19,21 +26,9 @@ function parsePdfDate(value) {
|
|
|
19
26
|
return formatDateLike(str);
|
|
20
27
|
}
|
|
21
28
|
export async function inspectPdf(filePath, buffer, sampleParagraphs) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (typeof args[0] === 'string' && (args[0].startsWith('Warning: ') || args[0].startsWith('Info: ') || args[0].startsWith('Deprecated API usage: '))) {
|
|
26
|
-
return;
|
|
27
|
-
}
|
|
28
|
-
originalLog.apply(console, args);
|
|
29
|
-
};
|
|
30
|
-
let data;
|
|
31
|
-
try {
|
|
32
|
-
data = PdfParseResultSchema.parse(await pdf(buffer));
|
|
33
|
-
}
|
|
34
|
-
finally {
|
|
35
|
-
console.log = originalLog;
|
|
36
|
-
}
|
|
29
|
+
const pdf = await loadPdfParse();
|
|
30
|
+
// Suppress pdf.js warnings (shared ref-counted helper — safe under concurrency)
|
|
31
|
+
const data = await suppressConsoleLogWhile(PDF_LOG_SUPPRESSED_PREFIXES, async () => PdfParseResultSchema.parse(await pdf(buffer)));
|
|
37
32
|
const info = data.info;
|
|
38
33
|
const properties = {
|
|
39
34
|
title: asOptionalString(info.Title),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inspect-pdf.js","sourceRoot":"","sources":["../../../src/doc/inspect-pdf.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"inspect-pdf.js","sourceRoot":"","sources":["../../../src/doc/inspect-pdf.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,2BAA2B,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACxH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAIxE,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACpD,CAAC,CAAC;AAEH,8FAA8F;AAC9F,IAAI,cAA4E,CAAC;AAEjF,KAAK,UAAU,YAAY;IACzB,cAAc,KAAK,MAAM,CAAC,WAAW,CAAC;SACnC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC;SAC1B,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,uBAAuB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;IAC1E,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,mDAAmD;IACnD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACzE,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;IACpF,CAAC;IACD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,MAAc,EACd,gBAAwB;IAOxB,MAAM,GAAG,GAAG,MAAM,YAAY,EAAE,CAAC;IAEjC,gFAAgF;IAChF,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE,CACjF,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAC9C,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IAEvB,MAAM,UAAU,GAAuB;QACrC,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;QACnC,OAAO,EAAE,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC;QACvC,MAAM,EAAE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC;QACrC,QAAQ,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC;QACzC,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC;QAC5C,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;KACzC,CAAC;IAEF,wCAAwC;IACxC,MAAM,SAAS,GAAiB;QAC9B,QAAQ,EAAE,KAAK;QACf,cAAc,EAAE,KAAK;QACrB,UAAU,EAAE,KAAK;QACjB,eAAe,EAAE,KAAK;QACtB,SAAS,EAAE,KAAK;QAChB,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,KAAK;QACb,YAAY,EAAE,KAAK;QACnB,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,IAAK,IAAgC,CAAC,SAAS,KAAK,IAAI;KAC5F,CAAC;IAEF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE/E,MAAM,YAAY,GAAoB;QACpC,KAAK;QACL,KAAK,EAAE,IAAI,CAAC,QAAQ;QACpB,cAAc,EAAE,KAAK;QACrB,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,MAAM;QACvB,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;KACV,CAAC;IAEF,0BAA0B;IAC1B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5E,MAAM,iBAAiB,GAAiC,EAAE,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,iBAAiB,CAAC,IAAI,CAAC;YACrB,KAAK,EAAE,CAAC;YACR,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YACnD,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAmB;QAC9B,SAAS,EAAE,cAAc,CAAC,MAAM,GAAG,gBAAgB;QACnD,UAAU,EAAE,iBAAiB;KAC9B,CAAC;IAEF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AAC1D,CAAC"}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildCodebaseIndex } from './code/build.js';
|
|
1
|
+
import { buildCodebaseIndex, parseCodeFiles, assembleCodebaseIndex, DEFAULT_PARSE_CONCURRENCY } from './code/build.js';
|
|
2
2
|
import { createCodeQuerySession, createCodeQuerySessionFromIndex } from './code/session.js';
|
|
3
3
|
import { openChunkCodeIndexStore, openCodeIndexStore } from './code/store.js';
|
|
4
4
|
import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
|
|
@@ -8,10 +8,10 @@ import { buildCodebaseIndexIsolated } from './code/isolated.js';
|
|
|
8
8
|
import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
|
|
9
9
|
import { summarizeModule, toMermaid } from './code/query.js';
|
|
10
10
|
import { buildRepoMap } from './code/map.js';
|
|
11
|
-
import { rankNodes } from './code/rank.js';
|
|
11
|
+
import { rankNodes, rankNodesCached } from './code/rank.js';
|
|
12
12
|
import { inspectDocument } from './doc/inspect.js';
|
|
13
13
|
import { discoverDocuments } from './doc/discover.js';
|
|
14
|
-
import { chunkDocument } from './doc/chunk.js';
|
|
14
|
+
import { chunkDocument, chunkDocumentFromMarkdown } from './doc/chunk.js';
|
|
15
15
|
import { inspectWorkbook } from './sheet/inspect.js';
|
|
16
16
|
import { inspectPresentation } from './slide/inspect.js';
|
|
17
17
|
import { inspectTables } from './table/inspect.js';
|
|
@@ -22,17 +22,18 @@ import { prepareWorkspaceContext } from './workspace/prepare.js';
|
|
|
22
22
|
import { bundleWorkspace } from './workspace/bundle.js';
|
|
23
23
|
import { health } from './health.js';
|
|
24
24
|
export type { CodebaseIndex, CodeNode, CodeEdge, CodeNodeType, CodeEdgeType, IndexTruncation, NormalizedSymbolKind } from './code/types.js';
|
|
25
|
+
export type { ManifestDiff, ManifestEntry, ChangedFileStat } from './code/incremental.js';
|
|
25
26
|
export type { CodeQuerySession, CodeSessionChunkOptions } from './code/session.js';
|
|
26
27
|
export type { CodeIndexStore, CodeIndexStoreOptions, StoreStrategy, StoreMetadata, StoreSessionOptions } from './code/store.js';
|
|
27
28
|
export type { CodeChunk, CodeChunkOptions } from './code/chunk.js';
|
|
28
|
-
export type { BuildCodebaseOptions, CodeIndexBudget, BudgetExceededBehavior } from './code/build.js';
|
|
29
|
+
export type { BuildCodebaseOptions, CodeIndexBudget, BudgetExceededBehavior, ParseCodeFilesOptions, ParseCodeFilesResult, AssembleCodebaseIndexInput } from './code/build.js';
|
|
29
30
|
export type { BuildCodebaseIndexIsolatedOptions } from './code/isolated.js';
|
|
30
31
|
export type { CodebaseIndexSlim, SlimParsedFile } from './code/slim.js';
|
|
31
32
|
export type { CodebaseSizePreview, PreviewCodebaseSizeOptions } from './code/preview.js';
|
|
32
33
|
export type { ModuleSummary, ModuleSummaryOptions, MermaidKind, MermaidOptions } from './code/query.js';
|
|
33
|
-
export type { RepoMapOptions, RepoMapResult, RepoMapEntry, RepoMapSymbol, RepoMapFormat, RepoMapMode } from './code/map.js';
|
|
34
|
+
export type { RepoMapOptions, RepoMapResult, RepoMapEntry, RepoMapSymbol, RepoMapFormat, RepoMapMode, RepoMapFocus, RepoMapFocusResult } from './code/map.js';
|
|
34
35
|
export type { RankOptions, RankResult, RankedNode, RankedFile, RankEdgeWeights } from './code/rank.js';
|
|
35
|
-
export type { DocumentChunk, DocumentChunkOptions } from './doc/chunk.js';
|
|
36
|
+
export type { DocumentChunk, DocumentChunkOptions, ChunkDocumentFromMarkdownOptions } from './doc/chunk.js';
|
|
36
37
|
export type { OnProgress, ProgressEvent, ProgressPhase } from './progress-event.js';
|
|
37
38
|
export type { WorkspacePrepareOptions, WorkspacePreparedContext, WorkspacePreparationProgressEvent, OnPrepareProgress } from './workspace/prepare-types.js';
|
|
38
39
|
export type { AnalyzeWorkspaceOptions } from './workspace/analyze.js';
|
|
@@ -41,12 +42,14 @@ export type { InspectWorkspaceDocumentSetOptions } from './workspace/documents.j
|
|
|
41
42
|
export type { WorkspaceBundle, WorkspaceBundleOptions, WorkspaceOutlineNode, WorkspaceCodeDocumentReference } from './workspace/bundle.js';
|
|
42
43
|
export type { WorkspaceDescription, WorkspaceRecommendedCall } from './workspace/types.js';
|
|
43
44
|
export type { OccHealth } from './health.js';
|
|
44
|
-
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
45
|
+
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, rankNodesCached, parseCodeFiles, assembleCodebaseIndex, DEFAULT_PARSE_CONCURRENCY, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
45
46
|
export { CodeIndexBudgetExceededError } from './code/build.js';
|
|
46
47
|
export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
|
|
47
48
|
export { health } from './health.js';
|
|
48
49
|
export { inspectTables } from './table/inspect.js';
|
|
49
|
-
export { chunkDocument } from './doc/chunk.js';
|
|
50
|
+
export { chunkDocument, chunkDocumentFromMarkdown } from './doc/chunk.js';
|
|
51
|
+
export { PROGRESS_PHASES, isProgressPhase } from './progress-event.js';
|
|
52
|
+
export { prepareProgressToProgressEvent } from './workspace/prepare-types.js';
|
|
50
53
|
export { bundleWorkspace } from './workspace/bundle.js';
|
|
51
54
|
export { toNormalizedSymbolKind } from './code/types.js';
|
|
52
55
|
export interface OccCodeNamespace {
|
|
@@ -64,6 +67,7 @@ export interface OccCodeNamespace {
|
|
|
64
67
|
toMermaid: typeof toMermaid;
|
|
65
68
|
map: typeof buildRepoMap;
|
|
66
69
|
rank: typeof rankNodes;
|
|
70
|
+
rankCached: typeof rankNodesCached;
|
|
67
71
|
slimifyIndex: typeof slimifyIndex;
|
|
68
72
|
parseSlimIndex: typeof parseSlimIndex;
|
|
69
73
|
tryParseSlimIndex: typeof tryParseSlimIndex;
|
|
@@ -80,6 +84,7 @@ export interface OccDocNamespace {
|
|
|
80
84
|
inspect: typeof inspectDocument;
|
|
81
85
|
discover: typeof discoverDocuments;
|
|
82
86
|
chunk: typeof chunkDocument;
|
|
87
|
+
chunkFromMarkdown: typeof chunkDocumentFromMarkdown;
|
|
83
88
|
}
|
|
84
89
|
export interface OccSheetNamespace {
|
|
85
90
|
inspect: typeof inspectWorkbook;
|
package/dist/src/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildCodebaseIndex } from './code/build.js';
|
|
1
|
+
import { buildCodebaseIndex, parseCodeFiles, assembleCodebaseIndex, DEFAULT_PARSE_CONCURRENCY, } from './code/build.js';
|
|
2
2
|
import { createCodeQuerySession, createCodeQuerySessionFromIndex, } from './code/session.js';
|
|
3
3
|
import { openChunkCodeIndexStore, openCodeIndexStore } from './code/store.js';
|
|
4
4
|
import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
|
|
@@ -8,10 +8,10 @@ import { buildCodebaseIndexIsolated } from './code/isolated.js';
|
|
|
8
8
|
import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
|
|
9
9
|
import { summarizeModule, toMermaid } from './code/query.js';
|
|
10
10
|
import { buildRepoMap } from './code/map.js';
|
|
11
|
-
import { rankNodes } from './code/rank.js';
|
|
11
|
+
import { rankNodes, rankNodesCached } from './code/rank.js';
|
|
12
12
|
import { inspectDocument } from './doc/inspect.js';
|
|
13
13
|
import { discoverDocuments } from './doc/discover.js';
|
|
14
|
-
import { chunkDocument } from './doc/chunk.js';
|
|
14
|
+
import { chunkDocument, chunkDocumentFromMarkdown } from './doc/chunk.js';
|
|
15
15
|
import { inspectWorkbook } from './sheet/inspect.js';
|
|
16
16
|
import { inspectPresentation } from './slide/inspect.js';
|
|
17
17
|
import { inspectTables } from './table/inspect.js';
|
|
@@ -21,12 +21,14 @@ import { inspectWorkspaceDocumentSet } from './workspace/documents.js';
|
|
|
21
21
|
import { prepareWorkspaceContext } from './workspace/prepare.js';
|
|
22
22
|
import { bundleWorkspace } from './workspace/bundle.js';
|
|
23
23
|
import { health } from './health.js';
|
|
24
|
-
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
24
|
+
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, rankNodesCached, parseCodeFiles, assembleCodebaseIndex, DEFAULT_PARSE_CONCURRENCY, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
25
25
|
export { CodeIndexBudgetExceededError } from './code/build.js';
|
|
26
26
|
export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
|
|
27
27
|
export { health } from './health.js';
|
|
28
28
|
export { inspectTables } from './table/inspect.js';
|
|
29
|
-
export { chunkDocument } from './doc/chunk.js';
|
|
29
|
+
export { chunkDocument, chunkDocumentFromMarkdown } from './doc/chunk.js';
|
|
30
|
+
export { PROGRESS_PHASES, isProgressPhase } from './progress-event.js';
|
|
31
|
+
export { prepareProgressToProgressEvent } from './workspace/prepare-types.js';
|
|
30
32
|
export { bundleWorkspace } from './workspace/bundle.js';
|
|
31
33
|
export { toNormalizedSymbolKind } from './code/types.js';
|
|
32
34
|
export function createOcc() {
|
|
@@ -46,6 +48,7 @@ export function createOcc() {
|
|
|
46
48
|
toMermaid,
|
|
47
49
|
map: buildRepoMap,
|
|
48
50
|
rank: rankNodes,
|
|
51
|
+
rankCached: rankNodesCached,
|
|
49
52
|
slimifyIndex,
|
|
50
53
|
parseSlimIndex,
|
|
51
54
|
tryParseSlimIndex,
|
|
@@ -62,6 +65,7 @@ export function createOcc() {
|
|
|
62
65
|
inspect: inspectDocument,
|
|
63
66
|
discover: discoverDocuments,
|
|
64
67
|
chunk: chunkDocument,
|
|
68
|
+
chunkFromMarkdown: chunkDocumentFromMarkdown,
|
|
65
69
|
},
|
|
66
70
|
sheet: {
|
|
67
71
|
inspect: inspectWorkbook,
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,yBAAyB,GAC1B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,sBAAsB,EACtB,+BAA+B,GAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyBrC,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,YAAY,EACZ,SAAS,EACT,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,yBAAyB,EACzB,aAAa,EACb,cAAc,EACd,sBAAsB,EACtB,+BAA+B,EAC/B,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,IAAI,cAAc,EACzC,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,iBAAiB,EACjB,iBAAiB,GAClB,CAAC;AACF,OAAO,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,8BAA8B,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AA6DzD,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,IAAI,EAAE;YACJ,UAAU,EAAE,kBAAkB;YAC9B,kBAAkB,EAAE,0BAA0B;YAC9C,aAAa,EAAE,sBAAsB;YACrC,sBAAsB,EAAE,+BAA+B;YACvD,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,uBAAuB;YACvC,WAAW,EAAE,mBAAmB;YAChC,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;YACpB,cAAc,EAAE,cAAc;YAC9B,eAAe;YACf,SAAS;YACT,GAAG,EAAE,YAAY;YACjB,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,eAAe;YAC3B,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,iBAAiB;SAClB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,gBAAgB;YACzB,QAAQ,EAAE,iBAAiB;YAC3B,gBAAgB,EAAE,2BAA2B;YAC7C,OAAO,EAAE,uBAAuB;YAChC,MAAM,EAAE,eAAe;SACxB;QACD,GAAG,EAAE;YACH,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;YACpB,iBAAiB,EAAE,yBAAyB;SAC7C;QACD,KAAK,EAAE;YACL,OAAO,EAAE,eAAe;SACzB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,mBAAmB;SAC7B;QACD,KAAK,EAAE;YACL,OAAO,EAAE,aAAa;SACvB;QACD,MAAM;KACP,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import type { CellObject, WorkSheet } from 'xlsx';
|
|
2
|
+
/**
|
|
3
|
+
* Local equivalent of XLSX.utils.encode_col (0 → A, 25 → Z, 26 → AA) so these
|
|
4
|
+
* synchronous helpers never need the xlsx runtime — callers lazy-load xlsx for
|
|
5
|
+
* parsing only, and cell addressing stays dependency-free.
|
|
6
|
+
*/
|
|
7
|
+
export declare function encodeColumnLetter(col: number): string;
|
|
8
|
+
/** Local equivalent of XLSX.utils.encode_cell ({r: 1, c: 1} → "B2"). */
|
|
9
|
+
export declare function encodeCellAddress(row: number, col: number): string;
|
|
2
10
|
export declare function getCell(sheet: WorkSheet, row: number, col: number): CellObject | undefined;
|
|
3
11
|
export declare function renderCell(cell: CellObject | undefined): string;
|
|
4
12
|
export declare function isNonEmptyCell(cell: CellObject): boolean;
|
|
@@ -1,6 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Local equivalent of XLSX.utils.encode_col (0 → A, 25 → Z, 26 → AA) so these
|
|
3
|
+
* synchronous helpers never need the xlsx runtime — callers lazy-load xlsx for
|
|
4
|
+
* parsing only, and cell addressing stays dependency-free.
|
|
5
|
+
*/
|
|
6
|
+
export function encodeColumnLetter(col) {
|
|
7
|
+
let letters = '';
|
|
8
|
+
for (let n = col + 1; n > 0; n = Math.floor((n - 1) / 26)) {
|
|
9
|
+
letters = String.fromCharCode(((n - 1) % 26) + 65) + letters;
|
|
10
|
+
}
|
|
11
|
+
return letters;
|
|
12
|
+
}
|
|
13
|
+
/** Local equivalent of XLSX.utils.encode_cell ({r: 1, c: 1} → "B2"). */
|
|
14
|
+
export function encodeCellAddress(row, col) {
|
|
15
|
+
return `${encodeColumnLetter(col)}${row + 1}`;
|
|
16
|
+
}
|
|
2
17
|
export function getCell(sheet, row, col) {
|
|
3
|
-
return sheet[
|
|
18
|
+
return sheet[encodeCellAddress(row, col)];
|
|
4
19
|
}
|
|
5
20
|
export function renderCell(cell) {
|
|
6
21
|
if (!cell)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xlsx-cells.js","sourceRoot":"","sources":["../../../src/inspect/xlsx-cells.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"xlsx-cells.js","sourceRoot":"","sources":["../../../src/inspect/xlsx-cells.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC;IAC/D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW;IACxD,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAgB,EAAE,GAAW,EAAE,GAAW;IAChE,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAA2B,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAA4B;IACrD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtE,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACxD,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IACtD,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,IAAI,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,IAAI,CAAC,CAAC,EAAE,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,IAAI,CAAC,CAAC,EAAE,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACrC,CAAC"}
|
|
@@ -1,12 +1,39 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import mammoth from 'mammoth';
|
|
3
|
-
import pdf from 'pdf-parse';
|
|
4
2
|
import JSZip from 'jszip';
|
|
5
|
-
import XLSX from 'xlsx';
|
|
6
|
-
import TurndownService from 'turndown';
|
|
7
3
|
import { z } from 'zod';
|
|
8
|
-
import { extractOfficeText, getExtension } from '../utils.js';
|
|
9
|
-
|
|
4
|
+
import { extractOfficeText, getExtension, PDF_LOG_SUPPRESSED_PREFIXES, suppressConsoleLogWhile, wrapDependencyLoadError } from '../utils.js';
|
|
5
|
+
// Heavy converters are lazy-loaded (and memoized) so importing this module
|
|
6
|
+
// never pulls in mammoth / pdf-parse / xlsx / turndown.
|
|
7
|
+
let mammothModule;
|
|
8
|
+
async function loadMammoth() {
|
|
9
|
+
mammothModule ??= import('mammoth')
|
|
10
|
+
.then((mod) => mod.default)
|
|
11
|
+
.catch((error) => wrapDependencyLoadError('mammoth', error));
|
|
12
|
+
return mammothModule;
|
|
13
|
+
}
|
|
14
|
+
let pdfParseModule;
|
|
15
|
+
async function loadPdfParse() {
|
|
16
|
+
pdfParseModule ??= import('pdf-parse')
|
|
17
|
+
.then((mod) => mod.default)
|
|
18
|
+
.catch((error) => wrapDependencyLoadError('pdf-parse', error));
|
|
19
|
+
return pdfParseModule;
|
|
20
|
+
}
|
|
21
|
+
// CJS interop: prefer .default (full API object) and fall back to the namespace.
|
|
22
|
+
let xlsxModule;
|
|
23
|
+
async function loadXlsx() {
|
|
24
|
+
xlsxModule ??= import('xlsx')
|
|
25
|
+
.then((mod) => mod.default ?? mod)
|
|
26
|
+
.catch((error) => wrapDependencyLoadError('xlsx', error));
|
|
27
|
+
return xlsxModule;
|
|
28
|
+
}
|
|
29
|
+
// Memoizes the configured TurndownService instance alongside the module load.
|
|
30
|
+
let turndownInstance;
|
|
31
|
+
async function getTurndown() {
|
|
32
|
+
turndownInstance ??= import('turndown')
|
|
33
|
+
.then((mod) => new mod.default({ headingStyle: 'atx' }))
|
|
34
|
+
.catch((error) => wrapDependencyLoadError('turndown', error));
|
|
35
|
+
return turndownInstance;
|
|
36
|
+
}
|
|
10
37
|
const PdfTextContentSchema = z.object({ items: z.array(z.object({ str: z.string() }).passthrough()) });
|
|
11
38
|
const RAW_TEXT_EXTENSIONS = new Set([
|
|
12
39
|
'md',
|
|
@@ -24,36 +51,26 @@ const RAW_TEXT_EXTENSIONS = new Set([
|
|
|
24
51
|
]);
|
|
25
52
|
/** Convert a DOCX file to markdown via mammoth → HTML → turndown */
|
|
26
53
|
async function docxToMarkdown(filePath) {
|
|
54
|
+
const mammoth = await loadMammoth();
|
|
27
55
|
const result = await mammoth.convertToHtml({ path: filePath });
|
|
28
56
|
const html = result.value || '';
|
|
29
57
|
if (!html.trim())
|
|
30
58
|
return '';
|
|
59
|
+
const turndown = await getTurndown();
|
|
31
60
|
return turndown.turndown(html);
|
|
32
61
|
}
|
|
33
62
|
/** Convert a PDF to markdown with [Page N] markers */
|
|
34
63
|
async function pdfToMarkdown(filePath) {
|
|
35
64
|
const buffer = await readFile(filePath);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
try {
|
|
46
|
-
data = await pdf(buffer, {
|
|
47
|
-
pagerender: async (pageData) => {
|
|
48
|
-
const textContent = PdfTextContentSchema.parse(await pageData.getTextContent());
|
|
49
|
-
const strings = textContent.items.map(item => item.str);
|
|
50
|
-
return `[Page ${pageData.pageIndex + 1}]\n${strings.join(' ')}`;
|
|
51
|
-
},
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
finally {
|
|
55
|
-
console.log = originalLog;
|
|
56
|
-
}
|
|
65
|
+
const pdf = await loadPdfParse();
|
|
66
|
+
// Suppress pdf.js warnings (shared ref-counted helper — safe under concurrency)
|
|
67
|
+
const data = await suppressConsoleLogWhile(PDF_LOG_SUPPRESSED_PREFIXES, () => pdf(buffer, {
|
|
68
|
+
pagerender: async (pageData) => {
|
|
69
|
+
const textContent = PdfTextContentSchema.parse(await pageData.getTextContent());
|
|
70
|
+
const strings = textContent.items.map(item => item.str);
|
|
71
|
+
return `[Page ${pageData.pageIndex + 1}]\n${strings.join(' ')}`;
|
|
72
|
+
},
|
|
73
|
+
}));
|
|
57
74
|
return data.text;
|
|
58
75
|
}
|
|
59
76
|
/** Convert a PPTX to markdown with slide headers */
|
|
@@ -117,6 +134,7 @@ async function odpToMarkdown(filePath) {
|
|
|
117
134
|
/** Convert an XLSX file to markdown with table formatting per sheet */
|
|
118
135
|
async function xlsxToMarkdown(filePath, maxRows = 20) {
|
|
119
136
|
const buffer = await readFile(filePath);
|
|
137
|
+
const XLSX = await loadXlsx();
|
|
120
138
|
const workbook = XLSX.read(buffer, {
|
|
121
139
|
type: 'buffer',
|
|
122
140
|
cellDates: true,
|