dsh-shared-kb 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -2
- package/cordis.patch.yml +11 -0
- package/lib/client.js +61 -0
- package/lib/core/atomic.js +87 -0
- package/lib/core/errors.js +39 -0
- package/lib/core/hubconfig.js +151 -0
- package/lib/core/paths.js +129 -0
- package/lib/icon.svg +7 -0
- package/lib/index.js +202 -0
- package/lib/indexer/embedder.js +325 -0
- package/lib/indexer/sqlite-index.js +431 -0
- package/lib/indexer/tokenize.js +151 -0
- package/lib/locale/zh-CN.json +4 -0
- package/lib/parse/chunk.js +202 -0
- package/lib/parse/parser.js +643 -0
- package/lib/search/search.js +194 -0
- package/lib/service/hub.js +791 -0
- package/lib/store/docs.js +906 -0
- package/lib/store/lease.js +251 -0
- package/lib/store/manifest.js +263 -0
- package/lib/tools/index.js +759 -0
- package/lib/ui/app.js +682 -0
- package/lib/ui/icon.svg +7 -0
- package/lib/ui/index.html +133 -0
- package/lib/ui/style.css +292 -0
- package/lib/web/api.js +225 -0
- package/lib/web/http.js +114 -0
- package/lib/web/server.js +149 -0
- package/package.json +83 -4
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 文档解析:把各种来源统一成 { blocks, pages, type, warnings, outline }。
|
|
3
|
+
*
|
|
4
|
+
* 原则:
|
|
5
|
+
* - 重量级依赖(mupdf)首次使用才动态 import,没装就只解析纯文本类,
|
|
6
|
+
* 不因为某个 PDF 让整个插件起不来;
|
|
7
|
+
* - 每个解析器都给出 page 号(能给的给),因为「第几页」是复核的锚点;
|
|
8
|
+
* - 解析失败不吞异常,返回 warnings + 已解析出的部分,让上层决定是否标记 partial。
|
|
9
|
+
*/
|
|
10
|
+
import { readFile } from "node:fs/promises";
|
|
11
|
+
import { extOf, isTextualExt } from "../core/paths.js";
|
|
12
|
+
import { badRequest } from "../core/errors.js";
|
|
13
|
+
|
|
14
|
+
export const PARSER_VERSION = 4;
|
|
15
|
+
|
|
16
|
+
/** 支持扩展名 → 解析器名 */
|
|
17
|
+
export const PARSERS = Object.freeze({
|
|
18
|
+
".pdf": "pdf",
|
|
19
|
+
".docx": "docx",
|
|
20
|
+
".xlsx": "xlsx",
|
|
21
|
+
".xlsm": "xlsx",
|
|
22
|
+
".pptx": "pptx",
|
|
23
|
+
".epub": "epub",
|
|
24
|
+
".md": "markdown",
|
|
25
|
+
".markdown": "markdown",
|
|
26
|
+
".txt": "text",
|
|
27
|
+
".csv": "csv",
|
|
28
|
+
".json": "text",
|
|
29
|
+
".html": "html",
|
|
30
|
+
".htm": "html",
|
|
31
|
+
".log": "text",
|
|
32
|
+
".yml": "text",
|
|
33
|
+
".yaml": "text",
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
export const SUPPORTED_EXTS = Object.freeze(Object.keys(PARSERS));
|
|
37
|
+
|
|
38
|
+
export function kindForPath(path) {
|
|
39
|
+
const ext = extOf(path);
|
|
40
|
+
const parser = PARSERS[ext];
|
|
41
|
+
if (!parser) {
|
|
42
|
+
throw badRequest(
|
|
43
|
+
`不支持的文件类型 ${ext || "(无扩展名)"};支持:${SUPPORTED_EXTS.join(" ")}`,
|
|
44
|
+
{ supported: SUPPORTED_EXTS }
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
return parser;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function decodeText(buffer) {
|
|
51
|
+
// BOM 优先
|
|
52
|
+
if (buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) {
|
|
53
|
+
return new TextDecoder("utf-8").decode(buffer.subarray(3));
|
|
54
|
+
}
|
|
55
|
+
if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) {
|
|
56
|
+
return new TextDecoder("utf-16le").decode(buffer.subarray(2));
|
|
57
|
+
}
|
|
58
|
+
if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) {
|
|
59
|
+
return new TextDecoder("utf-16be").decode(buffer.subarray(2));
|
|
60
|
+
}
|
|
61
|
+
// 先按 UTF-8 严格解码;失败再按 GB18030(中文 Windows 纯文本常见)
|
|
62
|
+
try {
|
|
63
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(buffer);
|
|
64
|
+
} catch {
|
|
65
|
+
try {
|
|
66
|
+
return new TextDecoder("gb18030").decode(buffer);
|
|
67
|
+
} catch {
|
|
68
|
+
return new TextDecoder("utf-8").decode(buffer);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// Markdown / 纯文本
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
/** 按 ATX 标题切段:# ## ### …,代码块内的 # 不算标题。 */
|
|
78
|
+
export function markdownToBlocks(markdown) {
|
|
79
|
+
const lines = String(markdown ?? "").split(/\r?\n/);
|
|
80
|
+
const blocks = [];
|
|
81
|
+
let heading = "";
|
|
82
|
+
let level = 0;
|
|
83
|
+
let buffer = [];
|
|
84
|
+
let inFence = false;
|
|
85
|
+
let fenceMark = "";
|
|
86
|
+
|
|
87
|
+
const push = () => {
|
|
88
|
+
const text = buffer.join("\n").trim();
|
|
89
|
+
buffer = [];
|
|
90
|
+
if (text === "") return;
|
|
91
|
+
blocks.push({ text, heading, level });
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
for (const line of lines) {
|
|
95
|
+
const fence = line.match(/^\s*(```+|~~~+)/);
|
|
96
|
+
if (fence) {
|
|
97
|
+
if (!inFence) {
|
|
98
|
+
inFence = true;
|
|
99
|
+
fenceMark = fence[1][0];
|
|
100
|
+
} else if (fence[1][0] === fenceMark) {
|
|
101
|
+
inFence = false;
|
|
102
|
+
}
|
|
103
|
+
buffer.push(line);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (!inFence) {
|
|
107
|
+
const atx = line.match(/^(#{1,6})\s+(.*)$/);
|
|
108
|
+
if (atx) {
|
|
109
|
+
push();
|
|
110
|
+
heading = atx[2].trim();
|
|
111
|
+
level = atx[1].length;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
buffer.push(line);
|
|
116
|
+
}
|
|
117
|
+
push();
|
|
118
|
+
return blocks;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export async function parseMarkdown(path) {
|
|
122
|
+
const text = decodeText(await readFile(path));
|
|
123
|
+
const blocks = markdownToBlocks(text);
|
|
124
|
+
return {
|
|
125
|
+
blocks: blocks.length > 0 ? blocks : [{ text, heading: "", level: 0 }],
|
|
126
|
+
pages: null,
|
|
127
|
+
type: "markdown",
|
|
128
|
+
warnings: [],
|
|
129
|
+
outline: blocks.filter((b) => b.heading).map((b) => ({ title: b.heading, level: b.level })),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export async function parseText(path) {
|
|
134
|
+
const text = decodeText(await readFile(path));
|
|
135
|
+
// 纯文本没有结构,按空行分段,每段不超过 2000 字
|
|
136
|
+
const paras = text.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p !== "");
|
|
137
|
+
const blocks = [];
|
|
138
|
+
for (const para of paras) {
|
|
139
|
+
if (para.length <= 2000) {
|
|
140
|
+
blocks.push({ text: para, heading: "", level: 0 });
|
|
141
|
+
} else {
|
|
142
|
+
for (let i = 0; i < para.length; i += 2000) {
|
|
143
|
+
blocks.push({ text: para.slice(i, i + 2000), heading: "", level: 0 });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return { blocks, pages: null, type: "text", warnings: [], outline: [] };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export async function parseCsv(path) {
|
|
151
|
+
const text = decodeText(await readFile(path));
|
|
152
|
+
const lines = text.split(/\r?\n/).filter((line) => line.trim() !== "");
|
|
153
|
+
if (lines.length === 0) return { blocks: [], pages: null, type: "csv", warnings: [], outline: [] };
|
|
154
|
+
const header = lines[0];
|
|
155
|
+
const cols = header.split(",").length;
|
|
156
|
+
const blocks = [{ text: `表头:${header}`, heading: "表头", level: 1 }];
|
|
157
|
+
// 每 40 行一组,组头带上表头,检索命中时能知道每列是什么
|
|
158
|
+
const GROUP = 40;
|
|
159
|
+
for (let i = 1; i < lines.length; i += GROUP) {
|
|
160
|
+
const slice = lines.slice(i, i + GROUP);
|
|
161
|
+
const body = slice.join("\n");
|
|
162
|
+
blocks.push({
|
|
163
|
+
text: `列:${header}\n${body}`,
|
|
164
|
+
heading: `第 ${i}–${i + slice.length - 1} 行`,
|
|
165
|
+
level: 2,
|
|
166
|
+
page: Math.ceil(i / GROUP),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
return { blocks, pages: Math.ceil(lines.length / GROUP), type: "csv", warnings: [], outline: [] };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export async function parseHtml(path) {
|
|
173
|
+
const text = decodeText(await readFile(path));
|
|
174
|
+
const blocks = htmlToBlocks(text);
|
|
175
|
+
return { blocks, pages: null, type: "html", warnings: [], outline: [] };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** 极简 HTML → 块:去掉 script/style,按块级标签断段,标题标签转 heading。 */
|
|
179
|
+
export function htmlToBlocks(html) {
|
|
180
|
+
const cleaned = String(html ?? "")
|
|
181
|
+
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
|
182
|
+
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
|
183
|
+
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
184
|
+
.replace(/<\/(p|div|section|article|li|tr|h[1-6]|br)\s*>/gi, "\n\n")
|
|
185
|
+
.replace(/<h([1-6])[^>]*>/gi, (_m, lv) => `\n\n__H${lv}__`)
|
|
186
|
+
.replace(/<[^>]+>/g, " ");
|
|
187
|
+
const decoded = cleaned
|
|
188
|
+
.replace(/ /g, " ")
|
|
189
|
+
.replace(/</g, "<")
|
|
190
|
+
.replace(/>/g, ">")
|
|
191
|
+
.replace(/"/g, "\"")
|
|
192
|
+
.replace(/'/g, "'")
|
|
193
|
+
.replace(/&/g, "&");
|
|
194
|
+
const blocks = [];
|
|
195
|
+
let heading = "";
|
|
196
|
+
let level = 0;
|
|
197
|
+
for (const rawChunk of decoded.split(/\n{2,}/)) {
|
|
198
|
+
const chunk = rawChunk.trim();
|
|
199
|
+
if (chunk === "") continue;
|
|
200
|
+
const h = chunk.match(/^__H([1-6])__([\s\S]*)$/);
|
|
201
|
+
if (h) {
|
|
202
|
+
heading = h[2].replace(/\s+/g, " ").trim();
|
|
203
|
+
level = Number(h[1]);
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const text = chunk.replace(/\s+/g, " ").trim();
|
|
207
|
+
if (text !== "") blocks.push({ text, heading, level });
|
|
208
|
+
}
|
|
209
|
+
return blocks;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ---------------------------------------------------------------------------
|
|
213
|
+
// DOCX / PPTX / XLSX(jszip + fast-xml-parser,零额外重依赖)
|
|
214
|
+
// ---------------------------------------------------------------------------
|
|
215
|
+
|
|
216
|
+
async function loadZip(path) {
|
|
217
|
+
const { default: JSZip } = await import("jszip");
|
|
218
|
+
const data = await readFile(path);
|
|
219
|
+
return JSZip.loadAsync(data);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function xmlParser() {
|
|
223
|
+
return { ignoreAttributes: false, attributeNamePrefix: "@_", trimValues: true };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** 从 DOCX 的 word/document.xml 抽段落、表格与标题层级。 */
|
|
227
|
+
export async function parseDocx(path) {
|
|
228
|
+
const zip = await loadZip(path);
|
|
229
|
+
const docFile = zip.file("word/document.xml");
|
|
230
|
+
if (!docFile) throw new Error("不是有效的 docx:缺少 word/document.xml");
|
|
231
|
+
const xml = await docFile.async("string");
|
|
232
|
+
const { XMLParser } = await import("fast-xml-parser");
|
|
233
|
+
const parser = new XMLParser({ ...xmlParser(), isArray: (name) => ["w:p", "w:tbl", "w:tr"].includes(name) });
|
|
234
|
+
const tree = parser.parse(xml);
|
|
235
|
+
|
|
236
|
+
const blocks = [];
|
|
237
|
+
const warnings = [];
|
|
238
|
+
const outline = [];
|
|
239
|
+
// 标题样式:w:pStyle 的 w:val 形如 Heading1 / 1 / 标题 1
|
|
240
|
+
const headingLevel = (styleId) => {
|
|
241
|
+
if (!styleId) return 0;
|
|
242
|
+
const s = String(styleId);
|
|
243
|
+
const m = s.match(/(?:heading|标题)\s*([1-6])/i) || s.match(/^([1-6])$/);
|
|
244
|
+
return m ? Number(m[1]) : 0;
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
const textOfNode = (node, out) => {
|
|
248
|
+
if (node == null) return;
|
|
249
|
+
if (Array.isArray(node)) {
|
|
250
|
+
for (const item of node) textOfNode(item, out);
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (typeof node !== "object") return;
|
|
254
|
+
for (const [key, value] of Object.entries(node)) {
|
|
255
|
+
if (key === "w:t") {
|
|
256
|
+
const parts = Array.isArray(value) ? value : [value];
|
|
257
|
+
for (const part of parts) {
|
|
258
|
+
if (typeof part === "string") out.push(part);
|
|
259
|
+
else if (part && typeof part["#text"] === "string") out.push(part["#text"]);
|
|
260
|
+
}
|
|
261
|
+
} else if (key === "w:tab") {
|
|
262
|
+
out.push("\t");
|
|
263
|
+
} else if (key === "w:br") {
|
|
264
|
+
out.push("\n");
|
|
265
|
+
} else if (key !== "w:pPr" && key !== "w:rPr") {
|
|
266
|
+
textOfNode(value, out);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
const walk = (node, heading, level) => {
|
|
272
|
+
if (node == null) return;
|
|
273
|
+
if (Array.isArray(node)) {
|
|
274
|
+
for (const item of node) walk(item, heading, level);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
if (typeof node !== "object") return;
|
|
278
|
+
|
|
279
|
+
if (Array.isArray(node["w:p"])) {
|
|
280
|
+
for (const p of node["w:p"]) {
|
|
281
|
+
const styleId = p?.["w:pPr"]?.["w:pStyle"]?.["@_w:val"];
|
|
282
|
+
const lv = headingLevel(styleId);
|
|
283
|
+
const out = [];
|
|
284
|
+
textOfNode(p, out);
|
|
285
|
+
const text = out.join("").trim();
|
|
286
|
+
if (text === "") continue;
|
|
287
|
+
if (lv > 0) {
|
|
288
|
+
outline.push({ title: text, level: lv });
|
|
289
|
+
blocks.push({ text, heading: text, level: lv });
|
|
290
|
+
} else {
|
|
291
|
+
blocks.push({ text, heading, level });
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (Array.isArray(node["w:tbl"])) {
|
|
296
|
+
for (const tbl of node["w:tbl"]) {
|
|
297
|
+
const rows = [];
|
|
298
|
+
for (const tr of tbl?.["w:tr"] ?? []) {
|
|
299
|
+
const cells = [];
|
|
300
|
+
for (const tc of tr?.["w:tc"] ?? []) {
|
|
301
|
+
const out = [];
|
|
302
|
+
textOfNode(tc, out);
|
|
303
|
+
cells.push(out.join("").replace(/\s+/g, " ").trim());
|
|
304
|
+
}
|
|
305
|
+
if (cells.some((c) => c !== "")) rows.push(cells.join(" | "));
|
|
306
|
+
}
|
|
307
|
+
if (rows.length > 0) {
|
|
308
|
+
// 表格每 30 行一块,避免超大表把一块撑爆
|
|
309
|
+
for (let i = 0; i < rows.length; i += 30) {
|
|
310
|
+
const slice = rows.slice(i, i + 30);
|
|
311
|
+
blocks.push({ text: slice.join("\n"), heading: heading || "表格", level: Math.max(level, 1) });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
for (const [key, value] of Object.entries(node)) {
|
|
317
|
+
if (key === "w:p" || key === "w:tbl") continue;
|
|
318
|
+
if (key.startsWith("@_") || key === "#text") continue;
|
|
319
|
+
walk(value, heading, level);
|
|
320
|
+
}
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
walk(tree, "", 0);
|
|
324
|
+
if (blocks.length === 0) warnings.push("文档中没有可提取的文字(可能是纯图片文档)");
|
|
325
|
+
return { blocks, pages: null, type: "docx", warnings, outline };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** PPTX:每张幻灯片一块,标题单独作为 heading。 */
|
|
329
|
+
export async function parsePptx(path) {
|
|
330
|
+
const zip = await loadZip(path);
|
|
331
|
+
const slides = Object.keys(zip.files)
|
|
332
|
+
.filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
|
|
333
|
+
.sort((a, b) => Number(a.match(/slide(\d+)/)[1]) - Number(b.match(/slide(\d+)/)[1]));
|
|
334
|
+
if (slides.length === 0) throw new Error("不是有效的 pptx:找不到 ppt/slides");
|
|
335
|
+
const { XMLParser } = await import("fast-xml-parser");
|
|
336
|
+
const parser = new XMLParser({ ...xmlParser(), isArray: (name) => name === "a:p" });
|
|
337
|
+
const blocks = [];
|
|
338
|
+
const outline = [];
|
|
339
|
+
|
|
340
|
+
for (let i = 0; i < slides.length; i += 1) {
|
|
341
|
+
const xml = await zip.file(slides[i]).async("string");
|
|
342
|
+
const tree = parser.parse(xml);
|
|
343
|
+
const texts = [];
|
|
344
|
+
const collect = (node) => {
|
|
345
|
+
if (node == null) return;
|
|
346
|
+
if (Array.isArray(node)) {
|
|
347
|
+
for (const item of node) collect(item);
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
if (typeof node !== "object") return;
|
|
351
|
+
if (typeof node["a:t"] === "string") texts.push(node["a:t"]);
|
|
352
|
+
else if (Array.isArray(node["a:t"])) for (const t of node["a:t"]) if (typeof t === "string") texts.push(t);
|
|
353
|
+
for (const [key, value] of Object.entries(node)) {
|
|
354
|
+
if (key === "a:t") continue;
|
|
355
|
+
collect(value);
|
|
356
|
+
}
|
|
357
|
+
};
|
|
358
|
+
collect(tree);
|
|
359
|
+
const lines = texts.map((t) => t.trim()).filter((t) => t !== "");
|
|
360
|
+
if (lines.length === 0) continue;
|
|
361
|
+
const title = lines[0].length <= 80 ? lines[0] : "";
|
|
362
|
+
const body = lines.slice(title ? 1 : 0).join("\n");
|
|
363
|
+
const label = `第 ${i + 1} 张幻灯片`;
|
|
364
|
+
if (title) outline.push({ title, level: 1 });
|
|
365
|
+
blocks.push({
|
|
366
|
+
text: `${label}${title ? `\n标题:${title}` : ""}${body ? `\n${body}` : ""}`,
|
|
367
|
+
heading: title || label,
|
|
368
|
+
level: 1,
|
|
369
|
+
page: i + 1,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
return { blocks, pages: slides.length, type: "pptx", warnings: [], outline };
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** XLSX:每个工作表一块组,每 40 行一组带表头。 */
|
|
376
|
+
export async function parseXlsx(path) {
|
|
377
|
+
const zip = await loadZip(path);
|
|
378
|
+
const { XMLParser } = await import("fast-xml-parser");
|
|
379
|
+
const parser = new XMLParser(xmlParser());
|
|
380
|
+
|
|
381
|
+
const sharedFile = zip.file("xl/sharedStrings.xml");
|
|
382
|
+
const shared = [];
|
|
383
|
+
if (sharedFile) {
|
|
384
|
+
const tree = parser.parse(await sharedFile.async("string"));
|
|
385
|
+
for (const si of tree?.["sst"]?.["si"] ?? []) {
|
|
386
|
+
const parts = [];
|
|
387
|
+
const collect = (node) => {
|
|
388
|
+
if (node == null) return;
|
|
389
|
+
if (typeof node === "string") {
|
|
390
|
+
parts.push(node);
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
if (Array.isArray(node)) {
|
|
394
|
+
for (const item of node) collect(item);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (typeof node !== "object") return;
|
|
398
|
+
if (typeof node["t"] === "string") parts.push(node["t"]);
|
|
399
|
+
for (const [key, value] of Object.entries(node)) {
|
|
400
|
+
if (key === "t" || key === "rPh") continue;
|
|
401
|
+
collect(value);
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
collect(si);
|
|
405
|
+
shared.push(parts.join(""));
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// 工作表名
|
|
410
|
+
const workbookFile = zip.file("xl/workbook.xml");
|
|
411
|
+
const sheetNames = [];
|
|
412
|
+
if (workbookFile) {
|
|
413
|
+
const tree = parser.parse(await workbookFile.async("string"));
|
|
414
|
+
for (const sheet of tree?.["workbook"]?.["sheets"]?.["sheet"] ?? []) {
|
|
415
|
+
sheetNames.push(String(sheet?.["@_name"] ?? `Sheet${sheetNames.length + 1}`));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const sheetFiles = Object.keys(zip.files)
|
|
420
|
+
.filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name))
|
|
421
|
+
.sort((a, b) => Number(a.match(/sheet(\d+)/)[1]) - Number(b.match(/sheet(\d+)/)[1]));
|
|
422
|
+
|
|
423
|
+
const blocks = [];
|
|
424
|
+
const outline = [];
|
|
425
|
+
for (let s = 0; s < sheetFiles.length; s += 1) {
|
|
426
|
+
const name = sheetNames[s] ?? `Sheet${s + 1}`;
|
|
427
|
+
outline.push({ title: name, level: 1 });
|
|
428
|
+
const tree = parser.parse(await zip.file(sheetFiles[s]).async("string"));
|
|
429
|
+
const rows = tree?.["worksheet"]?.["sheetData"]?.["row"] ?? [];
|
|
430
|
+
const lines = [];
|
|
431
|
+
for (const row of rows) {
|
|
432
|
+
const cells = [];
|
|
433
|
+
for (const c of row?.["c"] ?? []) {
|
|
434
|
+
const type = c?.["@_t"];
|
|
435
|
+
let value = "";
|
|
436
|
+
if (type === "s") {
|
|
437
|
+
const idx = Number(c?.["v"]);
|
|
438
|
+
value = Number.isInteger(idx) ? shared[idx] ?? "" : "";
|
|
439
|
+
} else if (type === "inlineStr") {
|
|
440
|
+
value = String(c?.["is"]?.["t"] ?? "");
|
|
441
|
+
} else {
|
|
442
|
+
value = String(c?.["v"] ?? "");
|
|
443
|
+
}
|
|
444
|
+
if (value !== "") cells.push(value);
|
|
445
|
+
}
|
|
446
|
+
if (cells.length > 0) lines.push(cells.join(" | "));
|
|
447
|
+
}
|
|
448
|
+
if (lines.length === 0) continue;
|
|
449
|
+
const header = lines[0];
|
|
450
|
+
const GROUP = 40;
|
|
451
|
+
for (let i = 1; i < lines.length; i += GROUP) {
|
|
452
|
+
const slice = lines.slice(i, i + GROUP);
|
|
453
|
+
blocks.push({
|
|
454
|
+
text: `工作表:${name}\n表头:${header}\n${slice.join("\n")}`,
|
|
455
|
+
heading: name,
|
|
456
|
+
level: 2,
|
|
457
|
+
page: Math.floor(i / GROUP) + 1,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
if (lines.length === 1) {
|
|
461
|
+
blocks.push({ text: `工作表:${name}\n表头:${header}`, heading: name, level: 2, page: 1 });
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return { blocks, pages: sheetFiles.length, type: "xlsx", warnings: [], outline };
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** EPUB:容器 → spine 顺序读 XHTML。 */
|
|
468
|
+
export async function parseEpub(path) {
|
|
469
|
+
const zip = await loadZip(path);
|
|
470
|
+
const containerFile = zip.file("META-INF/container.xml");
|
|
471
|
+
if (!containerFile) throw new Error("不是有效的 epub:缺少 META-INF/container.xml");
|
|
472
|
+
const { XMLParser } = await import("fast-xml-parser");
|
|
473
|
+
const parser = new XMLParser(xmlParser());
|
|
474
|
+
const container = parser.parse(await containerFile.async("string"));
|
|
475
|
+
const opfPath = container?.["container"]?.["rootfiles"]?.["rootfile"]?.["@_full-path"];
|
|
476
|
+
if (!opfPath) throw new Error("epub 缺少 OPF 路径");
|
|
477
|
+
const opf = parser.parse(await zip.file(opfPath).async("string"));
|
|
478
|
+
const baseDir = opfPath.includes("/") ? opfPath.slice(0, opfPath.lastIndexOf("/") + 1) : "";
|
|
479
|
+
const manifest = new Map();
|
|
480
|
+
for (const item of opf?.["package"]?.["manifest"]?.["item"] ?? []) {
|
|
481
|
+
manifest.set(String(item?.["@_id"]), String(item?.["@_href"] ?? ""));
|
|
482
|
+
}
|
|
483
|
+
const blocks = [];
|
|
484
|
+
const outline = [];
|
|
485
|
+
let page = 0;
|
|
486
|
+
for (const itemref of opf?.["package"]?.["spine"]?.["itemref"] ?? []) {
|
|
487
|
+
const href = manifest.get(String(itemref?.["@_idref"]));
|
|
488
|
+
if (!href) continue;
|
|
489
|
+
const full = decodeURIComponent(baseDir + href);
|
|
490
|
+
const file = zip.file(full);
|
|
491
|
+
if (!file) continue;
|
|
492
|
+
page += 1;
|
|
493
|
+
const html = await file.async("string");
|
|
494
|
+
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
495
|
+
const chapterTitle = titleMatch ? titleMatch[1].replace(/<[^>]+>/g, "").trim() : "";
|
|
496
|
+
if (chapterTitle) outline.push({ title: chapterTitle, level: 1 });
|
|
497
|
+
for (const block of htmlToBlocks(html)) {
|
|
498
|
+
blocks.push({ ...block, page, heading: block.heading || chapterTitle, level: block.level || (chapterTitle ? 1 : 0) });
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
return { blocks, pages: page, type: "epub", warnings: [], outline };
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// ---------------------------------------------------------------------------
|
|
505
|
+
// PDF(mupdf,可选依赖)
|
|
506
|
+
// ---------------------------------------------------------------------------
|
|
507
|
+
|
|
508
|
+
let mupdfPromise = null;
|
|
509
|
+
async function loadMupdf() {
|
|
510
|
+
if (!mupdfPromise) {
|
|
511
|
+
mupdfPromise = import("mupdf").catch(() => null);
|
|
512
|
+
}
|
|
513
|
+
return mupdfPromise;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** 判定一页文字层是否可信:字符数过少或大量替换字符 → 视为扫描页。 */
|
|
517
|
+
export function pageTextQuality(text) {
|
|
518
|
+
const s = String(text ?? "");
|
|
519
|
+
const printable = (s.match(/[\p{L}\p{N}\p{P}\s]/gu) ?? []).length;
|
|
520
|
+
const replacement = (s.match(/\uFFFD/g) ?? []).length;
|
|
521
|
+
const ratio = s.length === 0 ? 0 : printable / s.length;
|
|
522
|
+
return {
|
|
523
|
+
chars: s.replace(/\s+/g, "").length,
|
|
524
|
+
replacementRatio: s.length === 0 ? 0 : replacement / s.length,
|
|
525
|
+
printableRatio: ratio,
|
|
526
|
+
suspicious: s.replace(/\s+/g, "").length < 20 || ratio < 0.6,
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
export async function parsePdf(path) {
|
|
531
|
+
const mupdfModule = await loadMupdf();
|
|
532
|
+
if (!mupdfModule) {
|
|
533
|
+
throw new Error("解析 PDF 需要 mupdf(可选 peer 依赖),当前未安装。请运行 npm i mupdf,或改用文字类格式导入");
|
|
534
|
+
}
|
|
535
|
+
const mupdf = mupdfModule.default ?? mupdfModule;
|
|
536
|
+
const data = await readFile(path);
|
|
537
|
+
const doc = mupdf.Document.openDocument(
|
|
538
|
+
new Uint8Array(data.buffer, data.byteOffset, data.byteLength),
|
|
539
|
+
"application/pdf"
|
|
540
|
+
);
|
|
541
|
+
const pageCount = doc.countPages();
|
|
542
|
+
const blocks = [];
|
|
543
|
+
const warnings = [];
|
|
544
|
+
const scannedPages = [];
|
|
545
|
+
let outline = [];
|
|
546
|
+
try {
|
|
547
|
+
const raw = doc.loadOutline();
|
|
548
|
+
outline = (Array.isArray(raw) ? raw : [])
|
|
549
|
+
.map((o) => (typeof o === "string" ? o : o?.title))
|
|
550
|
+
.filter((t) => typeof t === "string" && t.trim() !== "")
|
|
551
|
+
.slice(0, 500)
|
|
552
|
+
.map((title) => ({ title: title.trim(), level: 1 }));
|
|
553
|
+
} catch {
|
|
554
|
+
outline = [];
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
for (let i = 0; i < pageCount; i += 1) {
|
|
558
|
+
const page = doc.loadPage(i);
|
|
559
|
+
let text = "";
|
|
560
|
+
try {
|
|
561
|
+
const structured = page.toStructuredText("preserve-whitespace");
|
|
562
|
+
text = structured.asText();
|
|
563
|
+
} catch {
|
|
564
|
+
text = "";
|
|
565
|
+
}
|
|
566
|
+
const quality = pageTextQuality(text);
|
|
567
|
+
if (quality.suspicious) scannedPages.push(i + 1);
|
|
568
|
+
const trimmed = text.trim();
|
|
569
|
+
if (trimmed !== "") {
|
|
570
|
+
for (const paragraph of trimmed.split(/\n{2,}/)) {
|
|
571
|
+
const p = paragraph.trim();
|
|
572
|
+
if (p !== "") blocks.push({ text: p, heading: "", level: 0, page: i + 1 });
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
page.destroy?.();
|
|
576
|
+
}
|
|
577
|
+
doc.destroy?.();
|
|
578
|
+
|
|
579
|
+
if (scannedPages.length > 0) {
|
|
580
|
+
warnings.push(
|
|
581
|
+
`${scannedPages.length}/${pageCount} 页没有可信文字层(第 ${scannedPages.slice(0, 20).join("、")}${scannedPages.length > 20 ? "…" : ""} 页),这些页需要 OCR 或查看原页图片`
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
if (blocks.length === 0) warnings.push("PDF 中没有提取到文字层;可能是纯扫描件");
|
|
585
|
+
|
|
586
|
+
return { blocks, pages: pageCount, type: "pdf", warnings, outline, scannedPages };
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/** 渲染某一页为 PNG(供查看原页 / 外部 OCR)。 */
|
|
590
|
+
export async function renderPdfPage(path, pageNo, { dpi = 150 } = {}) {
|
|
591
|
+
const mupdfModule = await loadMupdf();
|
|
592
|
+
if (!mupdfModule) throw new Error("渲染 PDF 需要 mupdf,当前未安装");
|
|
593
|
+
const mupdf = mupdfModule.default ?? mupdfModule;
|
|
594
|
+
const data = await readFile(path);
|
|
595
|
+
const doc = mupdf.Document.openDocument(
|
|
596
|
+
new Uint8Array(data.buffer, data.byteOffset, data.byteLength),
|
|
597
|
+
"application/pdf"
|
|
598
|
+
);
|
|
599
|
+
const index = Math.max(0, Math.min(doc.countPages() - 1, Number(pageNo) - 1));
|
|
600
|
+
const page = doc.loadPage(index);
|
|
601
|
+
const zoom = dpi / 72;
|
|
602
|
+
const pixmap = page.toPixmap(mupdf.Matrix.scale(zoom, zoom), mupdf.ColorSpace.DeviceRGB, false, true);
|
|
603
|
+
const png = pixmap.asPNG();
|
|
604
|
+
page.destroy?.();
|
|
605
|
+
doc.destroy?.();
|
|
606
|
+
return Buffer.from(png);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
// ---------------------------------------------------------------------------
|
|
610
|
+
// 统一入口
|
|
611
|
+
// ---------------------------------------------------------------------------
|
|
612
|
+
|
|
613
|
+
const PARSER_FNS = {
|
|
614
|
+
pdf: parsePdf,
|
|
615
|
+
docx: parseDocx,
|
|
616
|
+
xlsx: parseXlsx,
|
|
617
|
+
pptx: parsePptx,
|
|
618
|
+
epub: parseEpub,
|
|
619
|
+
markdown: parseMarkdown,
|
|
620
|
+
text: parseText,
|
|
621
|
+
csv: parseCsv,
|
|
622
|
+
html: parseHtml,
|
|
623
|
+
};
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* 解析任意受支持文件。
|
|
627
|
+
* @returns {Promise<{blocks, pages, type, warnings, outline, scannedPages?}>}
|
|
628
|
+
*/
|
|
629
|
+
export async function parseFile(path) {
|
|
630
|
+
const kind = kindForPath(path);
|
|
631
|
+
const fn = PARSER_FNS[kind];
|
|
632
|
+
if (!fn) throw badRequest(`没有为 ${kind} 注册解析器`);
|
|
633
|
+
const result = await fn(path);
|
|
634
|
+
return {
|
|
635
|
+
blocks: Array.isArray(result.blocks) ? result.blocks : [],
|
|
636
|
+
pages: result.pages ?? null,
|
|
637
|
+
type: result.type ?? kind,
|
|
638
|
+
warnings: Array.isArray(result.warnings) ? result.warnings : [],
|
|
639
|
+
outline: Array.isArray(result.outline) ? result.outline : [],
|
|
640
|
+
scannedPages: result.scannedPages ?? [],
|
|
641
|
+
parserVersion: PARSER_VERSION,
|
|
642
|
+
};
|
|
643
|
+
}
|