@xiaofanzhao/dsh-plugin-knowledge-base 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +300 -2
- package/client.js +1809 -0
- package/cordis.patch.yml +20 -0
- package/docs/DESIGN.md +662 -0
- package/docs/DETAIL.md +60 -0
- package/docs/images/architecture.svg +111 -0
- package/docs/images/ui-bigscreen.svg +59 -0
- package/docs/images/ui-overview.svg +152 -0
- package/icon.svg +13 -0
- package/index.js +131 -0
- package/lib/chunker.mjs +121 -0
- package/lib/extract.mjs +434 -0
- package/lib/index.mjs +383 -0
- package/lib/pdf.mjs +88 -0
- package/lib/pipeline.mjs +116 -0
- package/lib/preview.mjs +217 -0
- package/lib/routes.mjs +546 -0
- package/lib/settings.mjs +107 -0
- package/lib/store.mjs +445 -0
- package/lib/tools.mjs +246 -0
- package/lib/util.mjs +129 -0
- package/locale/en.json +33 -0
- package/locale/zh.json +33 -0
- package/package.json +63 -4
package/lib/chunker.mjs
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// 分块:把抽取出来的块序列渲染成正文文本,同时切成检索用的片段。
|
|
3
|
+
//
|
|
4
|
+
// 渲染出来的 text 会存进 doc_text,阅读器按字符偏移分页读;片段只存
|
|
5
|
+
// [charStart, charEnd) 与正文副本,用于检索与命中定位。
|
|
6
|
+
// ============================================================================
|
|
7
|
+
|
|
8
|
+
const TARGET_CHARS = 600;
|
|
9
|
+
const MAX_CHUNK_CHARS = 1600;
|
|
10
|
+
const MIN_CHUNK_CHARS = 80;
|
|
11
|
+
|
|
12
|
+
/** 渲染后的正文里,标题行会带 '#',分页会插一行〔第 N 页〕,方便人和模型读。 */
|
|
13
|
+
function lineOf(block, level) {
|
|
14
|
+
if (block.kind === 'page') return `\u3014第 ${block.page ?? '?'} 页\u3015`;
|
|
15
|
+
if (block.kind === 'heading') {
|
|
16
|
+
const depth = Math.min(6, Math.max(1, block.level || 1));
|
|
17
|
+
return `${'#'.repeat(depth)} ${block.text}`;
|
|
18
|
+
}
|
|
19
|
+
return block.text;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function renderDocument(blocks) {
|
|
23
|
+
const pieces = [];
|
|
24
|
+
let offset = 0;
|
|
25
|
+
let page = 0;
|
|
26
|
+
const headingStack = [];
|
|
27
|
+
const lines = [];
|
|
28
|
+
|
|
29
|
+
const push = (text, meta) => {
|
|
30
|
+
lines.push({
|
|
31
|
+
start: offset,
|
|
32
|
+
end: offset + text.length,
|
|
33
|
+
text,
|
|
34
|
+
heading: meta.heading,
|
|
35
|
+
page: meta.page,
|
|
36
|
+
kind: meta.kind,
|
|
37
|
+
});
|
|
38
|
+
pieces.push(text);
|
|
39
|
+
offset += text.length + 1;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
for (const block of blocks) {
|
|
43
|
+
if (!block) continue;
|
|
44
|
+
if (block.kind === 'page') {
|
|
45
|
+
page = Number(block.page) || page;
|
|
46
|
+
push(lineOf(block, 0), { heading: headingStack.map((item) => item.text).join(' > '), page, kind: 'page' });
|
|
47
|
+
push('', { heading: headingStack.map((item) => item.text).join(' > '), page, kind: 'text' });
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (block.kind === 'heading') {
|
|
51
|
+
const level = Math.min(6, Math.max(1, block.level || 1));
|
|
52
|
+
while (headingStack.length && headingStack[headingStack.length - 1].level >= level) headingStack.pop();
|
|
53
|
+
headingStack.push({ level, text: block.text });
|
|
54
|
+
push(lineOf(block, level), { heading: headingStack.map((item) => item.text).join(' > '), page, kind: 'heading' });
|
|
55
|
+
push('', { heading: headingStack.map((item) => item.text).join(' > '), page, kind: 'text' });
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const text = String(block.text ?? '');
|
|
59
|
+
if (!text.trim()) continue;
|
|
60
|
+
push(text, { heading: headingStack.map((item) => item.text).join(' > '), page: Number(block.page) || page, kind: block.kind });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const text = pieces.join('\n').replace(/\n{4,}/g, '\n\n\n');
|
|
64
|
+
return { text, lines };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** 把渲染后的行切成检索片段。 */
|
|
68
|
+
export function chunkDocument(lines) {
|
|
69
|
+
const chunks = [];
|
|
70
|
+
let buffer = [];
|
|
71
|
+
let bufferChars = 0;
|
|
72
|
+
let start = 0;
|
|
73
|
+
let end = 0;
|
|
74
|
+
let heading = '';
|
|
75
|
+
let page = 0;
|
|
76
|
+
|
|
77
|
+
const flush = (soft) => {
|
|
78
|
+
if (!buffer.length) return;
|
|
79
|
+
const body = buffer.join('\n').trim();
|
|
80
|
+
const charStart = start;
|
|
81
|
+
const charEnd = end;
|
|
82
|
+
buffer = [];
|
|
83
|
+
bufferChars = 0;
|
|
84
|
+
if (!body) return;
|
|
85
|
+
if (soft && body.length < MIN_CHUNK_CHARS) return;
|
|
86
|
+
chunks.push({
|
|
87
|
+
ord: chunks.length,
|
|
88
|
+
heading: heading.slice(0, 300),
|
|
89
|
+
page: page || null,
|
|
90
|
+
charStart,
|
|
91
|
+
charEnd,
|
|
92
|
+
text: body,
|
|
93
|
+
});
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
for (const line of lines) {
|
|
97
|
+
const lineText = line.text;
|
|
98
|
+
if (!lineText.trim() && bufferChars === 0) continue;
|
|
99
|
+
|
|
100
|
+
// 单行超长:硬切成多段,保证每块都有上限
|
|
101
|
+
const pieces = lineText.length > MAX_CHUNK_CHARS
|
|
102
|
+
? lineText.match(new RegExp(`[\\s\\S]{1,${TARGET_CHARS}}`, 'g')) ?? [lineText]
|
|
103
|
+
: [lineText];
|
|
104
|
+
|
|
105
|
+
for (const piece of pieces) {
|
|
106
|
+
if (bufferChars === 0) {
|
|
107
|
+
start = line.start;
|
|
108
|
+
heading = line.heading || '';
|
|
109
|
+
page = line.page || 0;
|
|
110
|
+
}
|
|
111
|
+
buffer.push(piece);
|
|
112
|
+
bufferChars += piece.length + 1;
|
|
113
|
+
end = line.end;
|
|
114
|
+
if (bufferChars >= TARGET_CHARS) flush(false);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
flush(false);
|
|
118
|
+
return chunks;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export { TARGET_CHARS, MAX_CHUNK_CHARS };
|
package/lib/extract.mjs
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// 文本抽取:把各种文件变成 [{kind, level?, page?, text}] 的块序列。
|
|
3
|
+
//
|
|
4
|
+
// 统一输出形状便于后面共用分块与阅读器:
|
|
5
|
+
// { blocks: [{ kind: 'heading'|'text'|'table'|'page', level?, page?, text }], pageCount }
|
|
6
|
+
// - heading:章节标题(level 1-6)
|
|
7
|
+
// - text :普通段落/行
|
|
8
|
+
// - table :表格行('| a | b |' 形式),给检索和阅读都能看
|
|
9
|
+
// - page :分页标记(PPT 幻灯片、PDF 页)
|
|
10
|
+
//
|
|
11
|
+
// 除 PDF 外都不依赖第三方库:docx/pptx/xlsx 都是 ZIP + XML,自己解。
|
|
12
|
+
// ============================================================================
|
|
13
|
+
import { inflateRawSync } from 'node:zlib';
|
|
14
|
+
|
|
15
|
+
// ── ZIP(只解 office 文档需要的那点:中央目录 + stored/deflate)─────────────
|
|
16
|
+
const EOCD_SIG = 0x06054b50;
|
|
17
|
+
const CD_SIG = 0x02014b50;
|
|
18
|
+
const LOCAL_SIG = 0x04034b50;
|
|
19
|
+
const MAX_ENTRY_BYTES = 96 * 1024 * 1024;
|
|
20
|
+
|
|
21
|
+
function findEocd(buffer) {
|
|
22
|
+
for (let i = buffer.length - 22; i >= Math.max(0, buffer.length - 65558); i -= 1) {
|
|
23
|
+
if (buffer.readUInt32LE(i) === EOCD_SIG) return i;
|
|
24
|
+
}
|
|
25
|
+
return -1;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function readZip(buffer) {
|
|
29
|
+
const eocd = findEocd(buffer);
|
|
30
|
+
if (eocd < 0) throw new Error('不是有效的 ZIP 容器');
|
|
31
|
+
const count = buffer.readUInt16LE(eocd + 10);
|
|
32
|
+
const cdOffset = buffer.readUInt32LE(eocd + 16);
|
|
33
|
+
if (cdOffset <= 0 || cdOffset >= buffer.length) throw new Error('ZIP 中央目录偏移异常');
|
|
34
|
+
const entries = new Map();
|
|
35
|
+
let pointer = cdOffset;
|
|
36
|
+
for (let index = 0; index < count && pointer + 46 <= buffer.length; index += 1) {
|
|
37
|
+
if (buffer.readUInt32LE(pointer) !== CD_SIG) break;
|
|
38
|
+
const method = buffer.readUInt16LE(pointer + 10);
|
|
39
|
+
const compSize = buffer.readUInt32LE(pointer + 20);
|
|
40
|
+
const uncompSize = buffer.readUInt32LE(pointer + 24);
|
|
41
|
+
const nameLen = buffer.readUInt16LE(pointer + 28);
|
|
42
|
+
const extraLen = buffer.readUInt16LE(pointer + 30);
|
|
43
|
+
const commentLen = buffer.readUInt16LE(pointer + 32);
|
|
44
|
+
const localOffset = buffer.readUInt32LE(pointer + 42);
|
|
45
|
+
const name = buffer.toString('utf8', pointer + 46, pointer + 46 + nameLen);
|
|
46
|
+
entries.set(name, { method, compSize, uncompSize, localOffset });
|
|
47
|
+
pointer += 46 + nameLen + extraLen + commentLen;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
names: [...entries.keys()],
|
|
51
|
+
has: (name) => entries.has(name),
|
|
52
|
+
get(name) {
|
|
53
|
+
const entry = entries.get(name);
|
|
54
|
+
if (!entry) return null;
|
|
55
|
+
const local = entry.localOffset;
|
|
56
|
+
if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== LOCAL_SIG) {
|
|
57
|
+
throw new Error('ZIP 局部头损坏');
|
|
58
|
+
}
|
|
59
|
+
const nameLen = buffer.readUInt16LE(local + 26);
|
|
60
|
+
const extraLen = buffer.readUInt16LE(local + 28);
|
|
61
|
+
const start = local + 30 + nameLen + extraLen;
|
|
62
|
+
const raw = buffer.subarray(start, start + entry.compSize);
|
|
63
|
+
if (entry.method === 0) return raw;
|
|
64
|
+
if (entry.method === 8) {
|
|
65
|
+
if (entry.uncompSize > MAX_ENTRY_BYTES) throw new Error('ZIP 条目过大');
|
|
66
|
+
return inflateRawSync(raw, { maxOutputLength: MAX_ENTRY_BYTES });
|
|
67
|
+
}
|
|
68
|
+
throw new Error(`不支持的 ZIP 压缩方式 ${entry.method}`);
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ── XML 小工具 ────────────────────────────────────────────────────────────
|
|
74
|
+
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
|
75
|
+
|
|
76
|
+
export function decodeEntities(text) {
|
|
77
|
+
return String(text).replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (whole, body) => {
|
|
78
|
+
if (body[0] === '#') {
|
|
79
|
+
const code = body[1] === 'x' || body[1] === 'X'
|
|
80
|
+
? Number.parseInt(body.slice(2), 16)
|
|
81
|
+
: Number.parseInt(body.slice(1), 10);
|
|
82
|
+
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
|
|
83
|
+
}
|
|
84
|
+
return ENTITIES[body] ?? whole;
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function stripTags(xml) {
|
|
89
|
+
return decodeEntities(String(xml).replace(/<[^>]*>/g, ''));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** 把一段 XML 里同名的兄弟元素按配对切出来(正确处理同名嵌套与自闭合)。 */
|
|
93
|
+
function sliceElements(xml, name) {
|
|
94
|
+
const out = [];
|
|
95
|
+
const tagRe = new RegExp(`<(/?)${name}(\\s[^>]*)?(/?)>`, 'g');
|
|
96
|
+
let depth = 0;
|
|
97
|
+
let start = -1;
|
|
98
|
+
let match;
|
|
99
|
+
while ((match = tagRe.exec(xml))) {
|
|
100
|
+
const closing = match[1] === '/';
|
|
101
|
+
const selfClosing = match[3] === '/';
|
|
102
|
+
if (closing) {
|
|
103
|
+
depth -= 1;
|
|
104
|
+
if (depth === 0 && start >= 0) {
|
|
105
|
+
out.push(xml.slice(start, tagRe.lastIndex));
|
|
106
|
+
start = -1;
|
|
107
|
+
}
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (depth === 0) start = match.index;
|
|
111
|
+
if (selfClosing) {
|
|
112
|
+
if (depth === 0) {
|
|
113
|
+
out.push(xml.slice(start, tagRe.lastIndex));
|
|
114
|
+
start = -1;
|
|
115
|
+
}
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
depth += 1;
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function paragraphText(xml) {
|
|
124
|
+
const withBreaks = String(xml)
|
|
125
|
+
.replace(/<w:tab\/>/g, ' ')
|
|
126
|
+
.replace(/<w:br\s*\/>/g, '\n')
|
|
127
|
+
.replace(/<w:cr\s*\/>/g, '\n')
|
|
128
|
+
.replace(/<\/w:p>/g, '\n</w:p>');
|
|
129
|
+
return decodeEntities(withBreaks.replace(/<[^>]*>/g, ''))
|
|
130
|
+
.replace(/\u000b/g, '\n')
|
|
131
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
132
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
133
|
+
.trim();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function headingLevelOfParagraph(xml) {
|
|
137
|
+
const style = /<w:pStyle\s+w:val="([^"]+)"/.exec(xml);
|
|
138
|
+
if (style) {
|
|
139
|
+
const value = style[1];
|
|
140
|
+
const heading = /^heading\s*(\d)$/i.exec(value);
|
|
141
|
+
if (heading) return Number(heading[1]);
|
|
142
|
+
const numeric = /^(\d)$/.exec(value);
|
|
143
|
+
if (numeric) return Number(numeric[1]);
|
|
144
|
+
const cn = /^标题\s*(\d)$/.exec(value);
|
|
145
|
+
if (cn) return Number(cn[1]);
|
|
146
|
+
}
|
|
147
|
+
const outline = /<w:outlineLvl\s+w:val="(\d)"/.exec(xml);
|
|
148
|
+
if (outline) return Math.min(6, Number(outline[1]) + 1);
|
|
149
|
+
return 0;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ── Markdown ──────────────────────────────────────────────────────────────
|
|
153
|
+
function extractMarkdown(text) {
|
|
154
|
+
const blocks = [];
|
|
155
|
+
let fence = false;
|
|
156
|
+
let buffer = [];
|
|
157
|
+
const flush = () => {
|
|
158
|
+
const value = buffer.join('\n').trim();
|
|
159
|
+
if (value) blocks.push({ kind: 'text', text: value });
|
|
160
|
+
buffer = [];
|
|
161
|
+
};
|
|
162
|
+
for (const line of text.split(/\r?\n/)) {
|
|
163
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
164
|
+
fence = !fence;
|
|
165
|
+
buffer.push(line);
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (!fence) {
|
|
169
|
+
const heading = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
|
|
170
|
+
if (heading) {
|
|
171
|
+
flush();
|
|
172
|
+
blocks.push({ kind: 'heading', level: heading[1].length, text: heading[2].trim() });
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
buffer.push(line);
|
|
177
|
+
}
|
|
178
|
+
flush();
|
|
179
|
+
return { blocks, pageCount: 0 };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ── HTML ──────────────────────────────────────────────────────────────────
|
|
183
|
+
function extractHtml(text) {
|
|
184
|
+
const cleaned = String(text)
|
|
185
|
+
.replace(/<!--[\s\S]*?-->/g, '')
|
|
186
|
+
.replace(/<(script|style|noscript)\b[\s\S]*?<\/\1>/gi, '');
|
|
187
|
+
const blocks = [];
|
|
188
|
+
const re = /<(h[1-6]|p|li|div|br|td|th|tr)\b[^>]*>([\s\S]*?)<\/\1>|<br\s*\/?>/gi;
|
|
189
|
+
let last = 0;
|
|
190
|
+
let match;
|
|
191
|
+
const pushText = (value) => {
|
|
192
|
+
const body = stripTags(value).replace(/[ \t]+/g, ' ').trim();
|
|
193
|
+
if (body) blocks.push({ kind: 'text', text: body });
|
|
194
|
+
};
|
|
195
|
+
while ((match = re.exec(cleaned))) {
|
|
196
|
+
if (match.index > last) pushText(cleaned.slice(last, match.index));
|
|
197
|
+
if (match[1] && /^h[1-6]$/i.test(match[1])) {
|
|
198
|
+
const body = stripTags(match[2]).replace(/\s+/g, ' ').trim();
|
|
199
|
+
if (body) blocks.push({ kind: 'heading', level: Number(match[1][1]), text: body });
|
|
200
|
+
} else if (match[1]) {
|
|
201
|
+
pushText(match[2]);
|
|
202
|
+
}
|
|
203
|
+
last = re.lastIndex;
|
|
204
|
+
}
|
|
205
|
+
if (last < cleaned.length) pushText(cleaned.slice(last));
|
|
206
|
+
const merged = [];
|
|
207
|
+
for (const block of blocks) {
|
|
208
|
+
if (block.kind === 'text' && block.text.length < 400) {
|
|
209
|
+
const previous = merged[merged.length - 1];
|
|
210
|
+
if (previous && previous.kind === 'text') {
|
|
211
|
+
previous.text = `${previous.text}\n${block.text}`;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
merged.push(block);
|
|
216
|
+
}
|
|
217
|
+
return { blocks: merged, pageCount: 0 };
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── 纯文本 / CSV ──────────────────────────────────────────────────────────
|
|
221
|
+
function extractPlain(text) {
|
|
222
|
+
const blocks = [];
|
|
223
|
+
const chunks = String(text).split(/\n{2,}/);
|
|
224
|
+
for (const chunk of chunks) {
|
|
225
|
+
const value = chunk.replace(/[ \t]+\n/g, '\n').trim();
|
|
226
|
+
if (value) blocks.push({ kind: 'text', text: value });
|
|
227
|
+
}
|
|
228
|
+
return { blocks: blocks.length ? blocks : [{ kind: 'text', text: '' }], pageCount: 0 };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function parseDelimited(text, delimiter) {
|
|
232
|
+
const rows = [];
|
|
233
|
+
let row = [];
|
|
234
|
+
let field = '';
|
|
235
|
+
let quoted = false;
|
|
236
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
237
|
+
const char = text[index];
|
|
238
|
+
if (quoted) {
|
|
239
|
+
if (char === '"') {
|
|
240
|
+
if (text[index + 1] === '"') { field += '"'; index += 1; } else { quoted = false; }
|
|
241
|
+
} else {
|
|
242
|
+
field += char;
|
|
243
|
+
}
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
if (char === '"') { quoted = true; continue; }
|
|
247
|
+
if (char === delimiter) { row.push(field); field = ''; continue; }
|
|
248
|
+
if (char === '\n') { row.push(field); rows.push(row); row = []; field = ''; continue; }
|
|
249
|
+
if (char === '\r') continue;
|
|
250
|
+
field += char;
|
|
251
|
+
}
|
|
252
|
+
if (field.length || row.length) { row.push(field); rows.push(row); }
|
|
253
|
+
return rows.filter((item) => item.some((cell) => String(cell).trim() !== ''));
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function extractCsv(text) {
|
|
257
|
+
const delimiter = (text.split('\n')[0] || '').includes('\t') ? '\t' : ',';
|
|
258
|
+
const rows = parseDelimited(text, delimiter);
|
|
259
|
+
const blocks = [{ kind: 'heading', level: 2, text: '表格' }];
|
|
260
|
+
for (const row of rows) blocks.push({ kind: 'table', text: `| ${row.join(' | ')} |` });
|
|
261
|
+
return { blocks, pageCount: 0, rows };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// ── DOCX ──────────────────────────────────────────────────────────────────
|
|
265
|
+
function extractDocx(buffer) {
|
|
266
|
+
const zip = readZip(buffer);
|
|
267
|
+
const main = zip.get('word/document.xml');
|
|
268
|
+
if (!main) throw new Error('DOCX 里找不到 word/document.xml');
|
|
269
|
+
const xml = main.toString('utf8');
|
|
270
|
+
const body = /<w:body[^>]*>([\s\S]*)<\/w:body>/.exec(xml);
|
|
271
|
+
const content = body ? body[1] : xml;
|
|
272
|
+
const blocks = [];
|
|
273
|
+
const walker = /<w:(p|tbl)\b[^>]*>/g;
|
|
274
|
+
let cursor = 0;
|
|
275
|
+
let match;
|
|
276
|
+
while ((match = walker.exec(content))) {
|
|
277
|
+
if (match.index < cursor) continue;
|
|
278
|
+
const tag = match[1];
|
|
279
|
+
const slice = sliceElements(content.slice(match.index), `w:${tag}`)[0];
|
|
280
|
+
if (!slice) continue;
|
|
281
|
+
cursor = match.index + slice.length;
|
|
282
|
+
walker.lastIndex = cursor;
|
|
283
|
+
if (tag === 'p') {
|
|
284
|
+
const body2 = paragraphText(slice);
|
|
285
|
+
if (!body2) continue;
|
|
286
|
+
const level = headingLevelOfParagraph(slice);
|
|
287
|
+
blocks.push(level ? { kind: 'heading', level, text: body2.replace(/\n+/g, ' ') } : { kind: 'text', text: body2 });
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
const rows = sliceElements(slice, 'w:tr');
|
|
291
|
+
for (const rowXml of rows) {
|
|
292
|
+
const cells = sliceElements(rowXml, 'w:tc').map((cell) => paragraphText(cell).replace(/\n+/g, ' ').trim());
|
|
293
|
+
if (cells.some(Boolean)) blocks.push({ kind: 'table', text: `| ${cells.join(' | ')} |` });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return { blocks, pageCount: 0 };
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ── PPTX ──────────────────────────────────────────────────────────────────
|
|
300
|
+
function extractPptx(buffer) {
|
|
301
|
+
const zip = readZip(buffer);
|
|
302
|
+
const slides = zip.names
|
|
303
|
+
.filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
|
|
304
|
+
.sort((left, right) => Number(/(\d+)\.xml$/.exec(left)[1]) - Number(/(\d+)\.xml$/.exec(right)[1]));
|
|
305
|
+
if (!slides.length) throw new Error('PPTX 里找不到幻灯片');
|
|
306
|
+
const blocks = [];
|
|
307
|
+
slides.forEach((name, index) => {
|
|
308
|
+
const page = index + 1;
|
|
309
|
+
blocks.push({ kind: 'page', page, text: `第 ${page} 页` });
|
|
310
|
+
const xml = zip.get(name)?.toString('utf8') ?? '';
|
|
311
|
+
const paragraphs = sliceElements(xml, 'a:p');
|
|
312
|
+
for (const paragraph of paragraphs) {
|
|
313
|
+
const body = paragraphText(paragraph).replace(/\n+/g, ' ').trim();
|
|
314
|
+
if (body) blocks.push({ kind: 'text', text: body, page });
|
|
315
|
+
}
|
|
316
|
+
const notesName = name.replace('slides/slide', 'notesSlides/notesSlide');
|
|
317
|
+
if (zip.has(notesName)) {
|
|
318
|
+
const notes = paragraphText(zip.get(notesName)?.toString('utf8') ?? '').trim();
|
|
319
|
+
if (notes) blocks.push({ kind: 'text', text: `备注:${notes}`, page });
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
return { blocks, pageCount: slides.length };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// ── XLSX ──────────────────────────────────────────────────────────────────
|
|
326
|
+
function extractXlsx(buffer) {
|
|
327
|
+
const zip = readZip(buffer);
|
|
328
|
+
const shared = [];
|
|
329
|
+
if (zip.has('xl/sharedStrings.xml')) {
|
|
330
|
+
const xml = zip.get('xl/sharedStrings.xml').toString('utf8');
|
|
331
|
+
for (const item of sliceElements(xml, 'si')) shared.push(paragraphText(item).replace(/\n+/g, ' ').trim());
|
|
332
|
+
}
|
|
333
|
+
const sheetNames = [];
|
|
334
|
+
if (zip.has('xl/workbook.xml')) {
|
|
335
|
+
const xml = zip.get('xl/workbook.xml').toString('utf8');
|
|
336
|
+
const re = /<sheet\b[^>]*name="([^"]*)"[^>]*\/?>/g;
|
|
337
|
+
let match;
|
|
338
|
+
while ((match = re.exec(xml))) sheetNames.push(decodeEntities(match[1]));
|
|
339
|
+
}
|
|
340
|
+
const sheetFiles = zip.names
|
|
341
|
+
.filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name))
|
|
342
|
+
.sort((left, right) => Number(/(\d+)\.xml$/.exec(left)[1]) - Number(/(\d+)\.xml$/.exec(right)[1]));
|
|
343
|
+
if (!sheetFiles.length) throw new Error('XLSX 里找不到工作表');
|
|
344
|
+
const blocks = [];
|
|
345
|
+
sheetFiles.forEach((name, index) => {
|
|
346
|
+
const title = sheetNames[index] || `Sheet${index + 1}`;
|
|
347
|
+
blocks.push({ kind: 'heading', level: 2, text: title });
|
|
348
|
+
const xml = zip.get(name)?.toString('utf8') ?? '';
|
|
349
|
+
const rows = sliceElements(xml, 'row');
|
|
350
|
+
let emitted = 0;
|
|
351
|
+
for (const rowXml of rows) {
|
|
352
|
+
const cells = sliceElements(rowXml, 'c').map((cellXml) => {
|
|
353
|
+
const type = /\st="([^"]+)"/.exec(cellXml)?.[1] ?? '';
|
|
354
|
+
if (type === 'inlineStr') {
|
|
355
|
+
const inline = /<t\b[^>]*>([\s\S]*?)<\/t>/.exec(cellXml);
|
|
356
|
+
return inline ? decodeEntities(inline[1]).trim() : '';
|
|
357
|
+
}
|
|
358
|
+
const value = /<v>([\s\S]*?)<\/v>/.exec(cellXml)?.[1] ?? '';
|
|
359
|
+
if (type === 's') return shared[Number(value)] ?? '';
|
|
360
|
+
return decodeEntities(value).trim();
|
|
361
|
+
});
|
|
362
|
+
if (!cells.some((cell) => cell !== '')) continue;
|
|
363
|
+
blocks.push({ kind: 'table', text: `| ${cells.join(' | ')} |` });
|
|
364
|
+
emitted += 1;
|
|
365
|
+
}
|
|
366
|
+
if (!emitted) blocks.push({ kind: 'text', text: '(空工作表)' });
|
|
367
|
+
});
|
|
368
|
+
return { blocks, pageCount: sheetFiles.length };
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// ── 入口 ──────────────────────────────────────────────────────────────────
|
|
372
|
+
const TEXT_EXTENSIONS = new Set([
|
|
373
|
+
'md', 'markdown', 'txt', 'text', 'log', 'csv', 'tsv', 'json', 'yaml', 'yml',
|
|
374
|
+
'js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'py', 'java', 'go', 'rs', 'c', 'h', 'cpp', 'hpp',
|
|
375
|
+
'cs', 'rb', 'php', 'sh', 'ps1', 'sql', 'css', 'scss', 'xml', 'toml', 'ini', 'conf',
|
|
376
|
+
]);
|
|
377
|
+
|
|
378
|
+
export const SUPPORTED_EXTENSIONS = new Set([
|
|
379
|
+
...TEXT_EXTENSIONS,
|
|
380
|
+
'html', 'htm', 'docx', 'pptx', 'xlsx', 'doc', 'ppt', 'xls', 'pdf',
|
|
381
|
+
'png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg',
|
|
382
|
+
]);
|
|
383
|
+
|
|
384
|
+
/** 这个扩展名是否需要正文索引(图片只做预览)。 */
|
|
385
|
+
export function isIndexable(ext) {
|
|
386
|
+
return ext !== 'png' && ext !== 'jpg' && ext !== 'jpeg' && ext !== 'gif'
|
|
387
|
+
&& ext !== 'webp' && ext !== 'bmp' && ext !== 'svg';
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
export const MIME_BY_EXT = {
|
|
391
|
+
pdf: 'application/pdf',
|
|
392
|
+
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
393
|
+
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
394
|
+
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
395
|
+
doc: 'application/msword',
|
|
396
|
+
ppt: 'application/vnd.ms-powerpoint',
|
|
397
|
+
xls: 'application/vnd.ms-excel',
|
|
398
|
+
png: 'image/png',
|
|
399
|
+
jpg: 'image/jpeg',
|
|
400
|
+
jpeg: 'image/jpeg',
|
|
401
|
+
gif: 'image/gif',
|
|
402
|
+
webp: 'image/webp',
|
|
403
|
+
bmp: 'image/bmp',
|
|
404
|
+
svg: 'image/svg+xml',
|
|
405
|
+
md: 'text/markdown',
|
|
406
|
+
markdown: 'text/markdown',
|
|
407
|
+
txt: 'text/plain',
|
|
408
|
+
csv: 'text/csv',
|
|
409
|
+
json: 'application/json',
|
|
410
|
+
html: 'text/html',
|
|
411
|
+
htm: 'text/html',
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
/** 抽取主入口。返回 { blocks, pageCount };不认识就抛错。 */
|
|
415
|
+
export async function extract({ ext, buffer, filePath }) {
|
|
416
|
+
if (ext === 'pdf') {
|
|
417
|
+
const { extractPdf } = await import('./pdf.mjs');
|
|
418
|
+
return extractPdf({ buffer, filePath });
|
|
419
|
+
}
|
|
420
|
+
if (ext === 'docx') return extractDocx(buffer);
|
|
421
|
+
if (ext === 'pptx') return extractPptx(buffer);
|
|
422
|
+
if (ext === 'xlsx') return extractXlsx(buffer);
|
|
423
|
+
if (ext === 'doc' || ext === 'ppt' || ext === 'xls') {
|
|
424
|
+
throw new Error('老版 Office 格式暂不支持正文索引(可以预览原文件)');
|
|
425
|
+
}
|
|
426
|
+
if (ext === 'html' || ext === 'htm') return extractHtml(buffer.toString('utf8'));
|
|
427
|
+
if (ext === 'md' || ext === 'markdown') return extractMarkdown(buffer.toString('utf8'));
|
|
428
|
+
if (ext === 'csv' || ext === 'tsv') {
|
|
429
|
+
const { blocks, pageCount } = extractCsv(buffer.toString('utf8'));
|
|
430
|
+
return { blocks, pageCount };
|
|
431
|
+
}
|
|
432
|
+
if (TEXT_EXTENSIONS.has(ext)) return extractPlain(buffer.toString('utf8'));
|
|
433
|
+
throw new Error(`暂不支持 ${ext || '该'} 格式的正文抽取`);
|
|
434
|
+
}
|