dsh-cooper-memory 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/LICENSE +21 -0
- package/README.md +107 -2
- package/cli.mjs +177 -0
- package/cordis.patch.yml +12 -0
- package/docs/DETAIL.md +83 -0
- package/icon.svg +10 -0
- package/lib/chunk.mjs +283 -0
- package/lib/core.mjs +439 -0
- package/lib/db.mjs +138 -0
- package/lib/extract.mjs +383 -0
- package/lib/import-lingshu.mjs +164 -0
- package/lib/llm.mjs +280 -0
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/mcp.mjs +298 -0
- package/package.json +37 -3
- package/web.mjs +230 -0
package/lib/chunk.mjs
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
// lib/chunk.mjs —— 三层分块器
|
|
2
|
+
//
|
|
3
|
+
// 移植自 WeKnora 的 chunker 设计(见 03-features/04-chunking.md):
|
|
4
|
+
// Tier1 heading → Tier2 heuristic → Tier3 legacy(递归兜底)
|
|
5
|
+
// 每层都要过 Validator(5 条拒绝规则),不合格降级到下一层
|
|
6
|
+
// overlap 取"语义后缀"(段落 > 换行 > 句末),不做定长硬切
|
|
7
|
+
//
|
|
8
|
+
// 简化:不解析 PDF/DOCX,只吃 markdown / 纯文本。
|
|
9
|
+
|
|
10
|
+
const SEPARATORS = ['\n\n', '\n', '。'];
|
|
11
|
+
const ABS_MAX = 7500;
|
|
12
|
+
|
|
13
|
+
const RE_HEADING = /^(#{1,6})\s+(.+?)\s*$/;
|
|
14
|
+
const RE_NUMBERED = /^\s*(?:\d+(?:\.\d+){0,3}[.、)]|[IVX]{1,5}\.)\s+\S/;
|
|
15
|
+
const RE_CHAPTER = /^\s*(?:第[一二三四五六七八九十百千0-9]+[章节篇部]|Chapter\s+\d+|Kapitel\s+\d+)/i;
|
|
16
|
+
const RE_ALLCAPS = /^\s*[A-Z][A-Z0-9 \-\u2013:]{6,60}\s*$/;
|
|
17
|
+
const RE_SEPARATOR = /^\s*(?:-{3,}|={3,}|\*{3,})\s*$/;
|
|
18
|
+
|
|
19
|
+
/** 文档画像:单遍扫描,决定用哪条 Tier 链 */
|
|
20
|
+
export function profile(text) {
|
|
21
|
+
let headings = 0;
|
|
22
|
+
let numbered = 0;
|
|
23
|
+
let chapters = 0;
|
|
24
|
+
let allcaps = 0;
|
|
25
|
+
let formfeeds = 0;
|
|
26
|
+
const levels = new Map();
|
|
27
|
+
for (const line of text.split('\n')) {
|
|
28
|
+
const m = RE_HEADING.exec(line);
|
|
29
|
+
if (m) {
|
|
30
|
+
headings++;
|
|
31
|
+
const lv = m[1].length;
|
|
32
|
+
levels.set(lv, (levels.get(lv) || 0) + 1);
|
|
33
|
+
}
|
|
34
|
+
if (RE_NUMBERED.test(line)) numbered++;
|
|
35
|
+
if (RE_CHAPTER.test(line)) chapters++;
|
|
36
|
+
if (RE_ALLCAPS.test(line)) allcaps++;
|
|
37
|
+
}
|
|
38
|
+
formfeeds = (text.match(/\f/g) || []).length;
|
|
39
|
+
const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
|
40
|
+
const latin = (text.match(/[A-Za-z]/g) || []).length;
|
|
41
|
+
const lang = cjk > latin ? 'zh' : latin > 0 ? 'en' : 'mixed';
|
|
42
|
+
const lines = text.split('\n').length || 1;
|
|
43
|
+
return {
|
|
44
|
+
chars: text.length,
|
|
45
|
+
lines,
|
|
46
|
+
headings,
|
|
47
|
+
headingDensity: headings / lines,
|
|
48
|
+
levels: [...levels.entries()],
|
|
49
|
+
numbered,
|
|
50
|
+
chapters,
|
|
51
|
+
allcaps,
|
|
52
|
+
formfeeds,
|
|
53
|
+
lang,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** 选主分割层级:出现 ≥3 次的最浅层级,否则最深 */
|
|
58
|
+
function dominantLevel(p) {
|
|
59
|
+
const stable = p.levels.filter(([, n]) => n >= 3).sort((a, b) => a[0] - b[0]);
|
|
60
|
+
if (stable.length) return stable[0][0];
|
|
61
|
+
const any = p.levels.sort((a, b) => b[0] - a[0]);
|
|
62
|
+
return any.length ? any[0][0] : 0;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function selectChain(p) {
|
|
66
|
+
const chain = [];
|
|
67
|
+
if (p.headings >= 3 && p.headingDensity > 0.005 && dominantLevel(p) > 0) chain.push('heading');
|
|
68
|
+
if (p.numbered >= 5 || p.chapters > 0 || p.formfeeds > 0 || p.allcaps >= 5) chain.push('heuristic');
|
|
69
|
+
chain.push('legacy');
|
|
70
|
+
return chain;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ---------- 受保护区间:代码块 / 公式不切开 ----------
|
|
74
|
+
function atomicize(text) {
|
|
75
|
+
const out = [];
|
|
76
|
+
const re = /(```[\s\S]*?```|\$\$[\s\S]*?\$\$)/g;
|
|
77
|
+
let last = 0;
|
|
78
|
+
let m;
|
|
79
|
+
while ((m = re.exec(text))) {
|
|
80
|
+
if (m.index > last) out.push({ atomic: false, text: text.slice(last, m.index) });
|
|
81
|
+
out.push({ atomic: true, text: m[0] });
|
|
82
|
+
last = m.index + m[0].length;
|
|
83
|
+
}
|
|
84
|
+
if (last < text.length) out.push({ atomic: false, text: text.slice(last) });
|
|
85
|
+
return out.length ? out : [{ atomic: false, text }];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** 递归按分隔符切分,返回片段数组 */
|
|
89
|
+
function splitRecursive(text, size, seps = SEPARATORS) {
|
|
90
|
+
if (text.length <= size) return [text];
|
|
91
|
+
for (let i = 0; i < seps.length; i++) {
|
|
92
|
+
const sep = seps[i];
|
|
93
|
+
if (!text.includes(sep)) continue;
|
|
94
|
+
const parts = text.split(sep).map((s, idx, arr) => (idx < arr.length - 1 ? s + sep : s)).filter(Boolean);
|
|
95
|
+
const res = [];
|
|
96
|
+
for (const part of parts) {
|
|
97
|
+
if (part.length <= size) res.push(part);
|
|
98
|
+
else res.push(...splitRecursive(part, size, seps.slice(i + 1)));
|
|
99
|
+
}
|
|
100
|
+
return res;
|
|
101
|
+
}
|
|
102
|
+
// 无分隔符可用:按字符硬切(兜底)
|
|
103
|
+
const res = [];
|
|
104
|
+
for (let i = 0; i < text.length; i += size) res.push(text.slice(i, i + size));
|
|
105
|
+
return res;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** 超长单元先按语义边界细化,保证不产出超过 budget 的块 */
|
|
109
|
+
function refine(unit, budget) {
|
|
110
|
+
if (unit.length <= budget) return [unit];
|
|
111
|
+
const pieces = splitRecursive(unit, budget);
|
|
112
|
+
const out = [];
|
|
113
|
+
for (const p of pieces) {
|
|
114
|
+
if (p.length <= budget) out.push(p);
|
|
115
|
+
else {
|
|
116
|
+
// 仍超长:按换行/空格/字符硬切
|
|
117
|
+
for (let i = 0; i < p.length; i += budget) out.push(p.slice(i, i + budget));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** 把片段装成块 */
|
|
124
|
+
function pack(units, size) {
|
|
125
|
+
const flat = [];
|
|
126
|
+
for (const u of units) {
|
|
127
|
+
if (u.length > size) flat.push(...refine(u, size));
|
|
128
|
+
else flat.push(u);
|
|
129
|
+
}
|
|
130
|
+
const chunks = [];
|
|
131
|
+
let cur = '';
|
|
132
|
+
for (const u of flat) {
|
|
133
|
+
if (cur && cur.length + u.length > size) { chunks.push(cur); cur = ''; }
|
|
134
|
+
cur += u;
|
|
135
|
+
}
|
|
136
|
+
if (cur.trim()) chunks.push(cur);
|
|
137
|
+
return chunks.filter((c) => c.trim());
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ---------- Tier 1:标题分块 ----------
|
|
141
|
+
function tierHeading(text, size) {
|
|
142
|
+
const p = profile(text);
|
|
143
|
+
const primary = dominantLevel(p);
|
|
144
|
+
if (!primary) return null;
|
|
145
|
+
const lines = text.split('\n');
|
|
146
|
+
const stack = [];
|
|
147
|
+
const sections = [];
|
|
148
|
+
let cur = { crumb: '', body: [] };
|
|
149
|
+
let inFence = false;
|
|
150
|
+
for (const line of lines) {
|
|
151
|
+
if (/^```/.test(line)) inFence = !inFence;
|
|
152
|
+
const m = !inFence && RE_HEADING.exec(line);
|
|
153
|
+
if (m && m[1].length <= primary) {
|
|
154
|
+
if (cur.body.length) sections.push(cur);
|
|
155
|
+
const lv = m[1].length;
|
|
156
|
+
while (stack.length && stack[stack.length - 1].lv >= lv) stack.pop();
|
|
157
|
+
stack.push({ lv, title: m[2] });
|
|
158
|
+
const crumb = stack.map((s) => '#'.repeat(s.lv) + ' ' + s.title).join('\n');
|
|
159
|
+
cur = { crumb, body: [line] };
|
|
160
|
+
} else {
|
|
161
|
+
cur.body.push(line);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (cur.body.length) sections.push(cur);
|
|
165
|
+
if (sections.length < 2) return null;
|
|
166
|
+
const chunks = [];
|
|
167
|
+
for (const s of sections) {
|
|
168
|
+
const body = s.body.join('\n').trim();
|
|
169
|
+
if (!body) continue;
|
|
170
|
+
if (body.length + s.crumb.length + 2 <= size) chunks.push({ text: body, crumb: s.crumb });
|
|
171
|
+
else for (const piece of splitRecursive(body, size)) chunks.push({ text: piece, crumb: s.crumb });
|
|
172
|
+
}
|
|
173
|
+
return chunks;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ---------- Tier 2:启发式边界 ----------
|
|
177
|
+
function tierHeuristic(text, size) {
|
|
178
|
+
const lines = text.split('\n');
|
|
179
|
+
const marks = [];
|
|
180
|
+
let inFence = false;
|
|
181
|
+
for (let i = 0; i < lines.length; i++) {
|
|
182
|
+
const line = lines[i];
|
|
183
|
+
if (/^```/.test(line)) { inFence = !inFence; continue; }
|
|
184
|
+
if (inFence) continue;
|
|
185
|
+
let prio = 0;
|
|
186
|
+
if (line.includes('\f')) prio = 100;
|
|
187
|
+
else if (RE_NUMBERED.test(line)) prio = 90;
|
|
188
|
+
else if (RE_CHAPTER.test(line)) prio = 85;
|
|
189
|
+
else if (RE_ALLCAPS.test(line)) prio = 70;
|
|
190
|
+
else if (RE_SEPARATOR.test(line)) prio = 60;
|
|
191
|
+
if (prio) marks.push({ line: i, prio });
|
|
192
|
+
}
|
|
193
|
+
if (marks.length < 3) return null;
|
|
194
|
+
marks.sort((a, b) => a.line - b.line || b.prio - a.prio);
|
|
195
|
+
const cuts = [0, ...marks.map((m) => m.line).filter((n) => n > 0), lines.length];
|
|
196
|
+
const uniq = [...new Set(cuts)].sort((a, b) => a - b);
|
|
197
|
+
const chunks = [];
|
|
198
|
+
for (let i = 0; i < uniq.length - 1; i++) {
|
|
199
|
+
const body = lines.slice(uniq[i], uniq[i + 1]).join('\n').trim();
|
|
200
|
+
if (!body) continue;
|
|
201
|
+
if (body.length <= size) chunks.push({ text: body, crumb: '' });
|
|
202
|
+
else for (const piece of splitRecursive(body, size)) chunks.push({ text: piece, crumb: '' });
|
|
203
|
+
}
|
|
204
|
+
return chunks.length >= 2 ? chunks : null;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ---------- Tier 3:递归兜底 ----------
|
|
208
|
+
function tierLegacy(text, size) {
|
|
209
|
+
const units = [];
|
|
210
|
+
for (const seg of atomicize(text)) {
|
|
211
|
+
if (seg.atomic) units.push(seg.text);
|
|
212
|
+
else units.push(...splitRecursive(seg.text, size));
|
|
213
|
+
}
|
|
214
|
+
return pack(units, size).map((t) => ({ text: t, crumb: '' }));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ---------- Validator:5 条拒绝规则 ----------
|
|
218
|
+
export function validate(chunks, size) {
|
|
219
|
+
const sizes = chunks.map((c) => c.text.length);
|
|
220
|
+
const total = sizes.reduce((a, b) => a + b, 0);
|
|
221
|
+
if (!chunks.length) return 'no chunks produced';
|
|
222
|
+
if (total > size * 2 && chunks.length === 1) return 'single chunk for large document';
|
|
223
|
+
const tiny = sizes.filter((s, i) => s < 50 && i < sizes.length - 1).length;
|
|
224
|
+
if (tiny > 2 && tiny > chunks.length / 4) return 'too many tiny chunks';
|
|
225
|
+
const max = Math.max(...sizes);
|
|
226
|
+
if (max < size / 4) return 'all chunks far below target size';
|
|
227
|
+
if (max > size * 2) return 'chunk exceeds 2x target size';
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** 语义重叠:从上一块尾部取一段,吸附到最近的语义边界 */
|
|
232
|
+
export function applyOverlap(chunks, overlap) {
|
|
233
|
+
if (!overlap || chunks.length < 2) return chunks;
|
|
234
|
+
const out = [chunks[0]];
|
|
235
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
236
|
+
const prev = chunks[i - 1].text;
|
|
237
|
+
const winStart = Math.max(0, prev.length - overlap - 4);
|
|
238
|
+
const win = prev.slice(winStart);
|
|
239
|
+
let cut = -1;
|
|
240
|
+
for (const sep of ['\n\n', '\n', '。', '?', '!', '. ']) {
|
|
241
|
+
const idx = win.lastIndexOf(sep);
|
|
242
|
+
if (idx >= 0) { cut = winStart + idx + sep.length; break; }
|
|
243
|
+
}
|
|
244
|
+
const tail = cut >= 0 ? prev.slice(cut) : '';
|
|
245
|
+
out.push({ ...chunks[i], text: tail + chunks[i].text });
|
|
246
|
+
}
|
|
247
|
+
return out;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* 主入口
|
|
252
|
+
* @returns {{chunks: Array<{ordinal,text,heading,context_header,tier}>, tier, rejected}}
|
|
253
|
+
*/
|
|
254
|
+
export function splitDoc(text, cfg = {}) {
|
|
255
|
+
const size = cfg.chunk_size || 512;
|
|
256
|
+
const overlap = Math.min(cfg.chunk_overlap ?? 80, Math.floor(size / 2));
|
|
257
|
+
const forced = cfg.strategy && cfg.strategy !== 'auto' ? cfg.strategy : null;
|
|
258
|
+
const p = profile(text);
|
|
259
|
+
const chain = forced ? [forced, 'legacy'] : selectChain(p);
|
|
260
|
+
const rejected = [];
|
|
261
|
+
for (const tier of chain) {
|
|
262
|
+
let raw = null;
|
|
263
|
+
if (tier === 'heading') raw = tierHeading(text, size);
|
|
264
|
+
else if (tier === 'heuristic') raw = tierHeuristic(text, size);
|
|
265
|
+
else raw = tierLegacy(text, size);
|
|
266
|
+
if (!raw) { rejected.push({ tier, reason: 'tier produced nothing' }); continue; }
|
|
267
|
+
const reason = validate(raw, size);
|
|
268
|
+
// 兜底层永不返回空(对齐 WeKnora:legacy 即使未通过校验也采纳)
|
|
269
|
+
const isLast = tier === 'legacy' || tier === chain[chain.length - 1];
|
|
270
|
+
if (reason && !isLast) { rejected.push({ tier, reason }); continue; }
|
|
271
|
+
if (reason && isLast) rejected.push({ tier, reason: reason + '(兜底层强制采纳)' });
|
|
272
|
+
const withOverlap = applyOverlap(raw, overlap);
|
|
273
|
+
const chunks = withOverlap.map((c, i) => ({
|
|
274
|
+
ordinal: i,
|
|
275
|
+
text: c.text.trim(),
|
|
276
|
+
heading: (c.crumb.split('\n').pop() || '').replace(/^#+\s*/, ''),
|
|
277
|
+
context_header: c.crumb || '',
|
|
278
|
+
tier,
|
|
279
|
+
})).filter((c) => c.text);
|
|
280
|
+
return { chunks, tier, rejected, profile: p };
|
|
281
|
+
}
|
|
282
|
+
return { chunks: [], tier: 'none', rejected, profile: p };
|
|
283
|
+
}
|