dsh-cooper-memory 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 dsh-cooper-memory contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,99 @@
1
- # Temporary Holding Version
1
+ # dsh-cooper-memory
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ 给 DeepSeek Harness 用的**本地记忆 + 文件夹知识库**。数据存在一个 SQLite 文件里,不依赖 Docker、不依赖向量服务、不出网(LLM 蒸馏走你已配置的模型通道)。
4
+
5
+ ## 解决什么
6
+
7
+ | 需求 | 实现 |
8
+ |---|---|
9
+ | 跨会话长期记忆,保存确认过的个人信息、偏好与事实 | 记忆表 + `pending → active` 状态机,未确认不注入 |
10
+ | 文件夹上传保留原始目录结构 | 递归入库,`rel_path` 保留层级,目录树由路径派生 |
11
+ | 检索分块可编辑、比对、回滚 | 只追加的版本表 + 行级 diff + 回滚生成新版本 |
12
+
13
+ ## 安装
14
+
15
+ 在 DSH 的 **Plugins** 页安装 `dsh-cooper-memory` 并启用;也可以用插件管理器从本地目录安装。
16
+
17
+ 安装后 bundle 会插入一个 MCP 客户端行,DSH 里出现 `mcp__memory__*` 工具(共 17 个)。
18
+
19
+ > 需要 Node.js ≥ 22.5(用到内置的 `node:sqlite`),且 `node` 需在 PATH 中。
20
+
21
+ ## 工具(17 个)
22
+
23
+ **记忆**
24
+
25
+ - `memory_recall` — 召回(常驻资料/偏好 + 按问匹配事实)
26
+ - `memory_remember` — 写入(默认 pending)
27
+ - `memory_confirm` / `memory_reject` — 确认 / 拒绝(拒绝留抑制记录)
28
+ - `memory_list` / `memory_edit`
29
+ - `memory_extract` — 规则提取(扫会话日志)
30
+ - `memory_distill` — LLM 蒸馏(分批,比规则准)
31
+ - `memory_probe_llm` — 探测 LLM 是否可用
32
+
33
+ **知识库**
34
+
35
+ - `kb_ingest` / `kb_tree` / `kb_search`
36
+ - `kb_get` / `kb_edit` / `kb_history` / `kb_diff` / `kb_rollback`
37
+
38
+ ## 命令行
39
+
40
+ ```bash
41
+ dsh-cooper-memory search "关键词"
42
+ dsh-cooper-memory extract --limit 20 # 规则提取
43
+ dsh-cooper-memory distill --turns 60 # LLM 蒸馏
44
+ dsh-cooper-memory memories --status pending
45
+ dsh-cooper-memory ingest D:/docs --size 512
46
+ dsh-cooper-memory tree
47
+ ```
48
+
49
+ ## 浏览器界面
50
+
51
+ ```bash
52
+ dsh-cooper-memory-web # http://127.0.0.1:17890
53
+ ```
54
+
55
+ 左边目录树、中间检索、右边分块编辑 + 版本历史 + 比对 + 回滚,另有「待确认记忆」队列。
56
+
57
+ ## 数据位置
58
+
59
+ 默认 `~/.dsh-memory/mem.db`,可用环境变量覆盖:
60
+
61
+ | 变量 | 作用 | 默认 |
62
+ |---|---|---|
63
+ | `MEM_ROOT` | 数据目录 | `~/.dsh-memory` |
64
+ | `MEM_DB` | 数据库文件完整路径 | `$MEM_ROOT/mem.db` |
65
+ | `MEM_WEB_PORT` | 界面端口 | `17890` |
66
+
67
+ ## 配置 LLM(可选)
68
+
69
+ 不配也能用:规则提取、知识库、检索、版本管理全部离线可用。配了才启用 `memory_distill`。
70
+
71
+ 解析顺序:
72
+
73
+ 1. `process.env[LLM_KEY_ENV]`(默认 `CT_API_KEY`)
74
+ 2. `~/.dsh/.credentials.yaml` 里同名 ref
75
+ 3. `DEEPSEEK_BASE_URL` / `LLM_MODEL` 覆盖地址与模型
76
+
77
+ 地址与模型默认从当前 profile 的 `cordis.patch.yml` 读取(`baseURL`、`apiKeyEnv`、`agent-default-model`)。
78
+
79
+ ## 导入工作流
80
+
81
+ 在灵枢(dsh-memory)里积累的记忆可以一次性导入:
82
+
83
+ ```bash
84
+ dsh-cooper-memory import-lingshu --dry # 先看统计
85
+ dsh-cooper-memory import-lingshu # 真正写入
86
+ ```
87
+
88
+ 只导入 `knowledge` 与 `contextual` 层;`structural`(审核裁决记录)会跳过。
89
+
90
+ ## 已知限制
91
+
92
+ - **不做向量检索**:只有 FTS5 trigram + 中文二元组,靠措辞重合召回
93
+ - **不解析 PDF/DOCX**:只吃 md / txt / csv / json / yaml
94
+ - **提取精度约 60~85%**:所以结果一律进 pending 队列,人工确认后才生效
95
+ - **单机单库**:没有多租户、没有同步
96
+
97
+ ## 许可
98
+
99
+ MIT
package/cli.mjs ADDED
@@ -0,0 +1,177 @@
1
+ #!/usr/bin/env node
2
+ // cli.mjs —— 命令行入口(人用 / 脚本用),所有输出为 JSON 或纯文本
3
+ import { openDb } from './lib/db.mjs';
4
+ import {
5
+ ingestFolder, searchChunks, getChunk, editChunk, chunkHistory,
6
+ chunkVersionText, rollbackChunk, diffTexts, tree,
7
+ remember, confirmMemory, rejectMemory, editMemory, listMemories, recall,
8
+ } from './lib/core.mjs';
9
+ import { runAutoExtract, recentTurns, llmDistill } from './lib/extract.mjs';
10
+ import { probe as llmProbe } from './lib/llm.mjs';
11
+ import { importLingshu } from './lib/import-lingshu.mjs';
12
+
13
+ const argv = process.argv.slice(2);
14
+ const cmd = argv[0];
15
+ const rest = argv.slice(1);
16
+
17
+ function flag(name, def = null) {
18
+ const i = rest.indexOf('--' + name);
19
+ if (i < 0) return def;
20
+ return rest[i + 1];
21
+ }
22
+ const has = (name) => rest.includes('--' + name);
23
+ const positional = rest.filter((a, i) => !a.startsWith('--') && !(i > 0 && rest[i - 1].startsWith('--')));
24
+
25
+ function out(obj) {
26
+ process.stdout.write(JSON.stringify(obj, null, 2) + '\n');
27
+ }
28
+
29
+ const db = openDb();
30
+
31
+ try {
32
+ switch (cmd) {
33
+ case 'ingest': {
34
+ const dir = positional[0];
35
+ if (!dir) throw new Error('用法: cli.mjs ingest <目录> [--size 512] [--overlap 80] [--strategy auto]');
36
+ const stats = ingestFolder(db, dir, {
37
+ chunk_size: Number(flag('size', 512)),
38
+ chunk_overlap: Number(flag('overlap', 80)),
39
+ strategy: flag('strategy', 'auto'),
40
+ max_files: Number(flag('max', 5000)),
41
+ });
42
+ out(stats);
43
+ break;
44
+ }
45
+ case 'tree':
46
+ process.stdout.write(tree(db) + '\n');
47
+ break;
48
+ case 'search': {
49
+ const q = positional[0];
50
+ if (!q) throw new Error('用法: cli.mjs search <query> [--variants "a|b"] [--limit 8]');
51
+ const variants = (flag('variants', '') || '').split('|').map((s) => s.trim()).filter(Boolean);
52
+ const r = searchChunks(db, { query: q, variants, limit: Number(flag('limit', 8)) });
53
+ out({
54
+ usedQueries: r.usedQueries, fused: r.fused,
55
+ results: r.results.map((x) => ({
56
+ id: x.id, path: x.rel_path, ordinal: x.ordinal, score: Number(x.score.toFixed(4)),
57
+ heading: x.heading, text: x.text.slice(0, 160),
58
+ })),
59
+ });
60
+ break;
61
+ }
62
+ case 'get': {
63
+ const id = Number(positional[0]);
64
+ const c = getChunk(db, id);
65
+ if (!c) throw new Error('未找到');
66
+ out(c);
67
+ break;
68
+ }
69
+ case 'edit': {
70
+ const id = Number(positional[0]);
71
+ const text = positional[1];
72
+ if (!text) throw new Error('用法: cli.mjs edit <chunkId> <新文本> [--note "..."] [--base N]');
73
+ out(editChunk(db, id, text, {
74
+ note: flag('note', ''),
75
+ baseVersion: flag('base') ? Number(flag('base')) : null,
76
+ }));
77
+ break;
78
+ }
79
+ case 'history': {
80
+ out(chunkHistory(db, Number(positional[0])));
81
+ break;
82
+ }
83
+ case 'diff': {
84
+ const id = Number(positional[0]);
85
+ const a = Number(flag('from', 1));
86
+ const b = Number(flag('to'));
87
+ const va = chunkVersionText(db, id, a);
88
+ const vb = b ? chunkVersionText(db, id, b) : getChunk(db, id);
89
+ if (!va || !vb) throw new Error('版本不存在');
90
+ out({ chunkId: id, from: a, to: b, diff: diffTexts(va.text, vb.text) });
91
+ break;
92
+ }
93
+ case 'rollback': {
94
+ out(rollbackChunk(db, Number(positional[0]), Number(positional[1])));
95
+ break;
96
+ }
97
+ case 'remember': {
98
+ out(remember(db, {
99
+ kind: flag('kind', 'fact'),
100
+ text: positional[0] || flag('text', ''),
101
+ subject: flag('subject', ''),
102
+ status: flag('status', 'pending'),
103
+ confidence: Number(flag('conf', 0.6)),
104
+ source: flag('source', 'cli'),
105
+ tags: flag('tags', ''),
106
+ supersedes: flag('supersedes', null),
107
+ }));
108
+ break;
109
+ }
110
+ case 'extract': {
111
+ const r = runAutoExtract(db, {
112
+ sessionsDir: flag('sessions', undefined),
113
+ limit: Number(flag('limit', 30)),
114
+ dryRun: has('dry'),
115
+ maxFiles: Number(flag('maxfiles', 50)),
116
+ });
117
+ out(r);
118
+ break;
119
+ }
120
+ case 'probe-llm': out(await llmProbe()); break;
121
+ case 'import-lingshu': {
122
+ out(importLingshu(db, {
123
+ dryRun: has('dry'),
124
+ limit: Number(flag('limit', 5000)),
125
+ dir: flag('dir', undefined) || undefined,
126
+ }));
127
+ break;
128
+ }
129
+ case 'distill': {
130
+ const turns = recentTurns(flag('sessions', undefined), Number(flag('turns', 60)));
131
+ const r = await llmDistill(db, {
132
+ turns,
133
+ limit: Number(flag('limit', 30)),
134
+ dryRun: has('dry'),
135
+ });
136
+ out(r);
137
+ break;
138
+ }
139
+ case 'confirm': out(confirmMemory(db, Number(positional[0]))); break;
140
+ case 'reject': out(rejectMemory(db, Number(positional[0]), { note: flag('note', '') })); break;
141
+ case 'mem-edit': out(editMemory(db, Number(positional[0]), positional[1], { note: flag('note', '') })); break;
142
+ case 'memories': out(listMemories(db, { status: flag('status'), kind: flag('kind'), limit: Number(flag('limit', 100)) })); break;
143
+ case 'recall': {
144
+ const r = recall(db, positional[0] || '', { budgetChars: Number(flag('budget', 1200)) });
145
+ out(r);
146
+ break;
147
+ }
148
+ case 'stats': {
149
+ const docs = db.prepare('SELECT COUNT(*) n FROM documents').get().n;
150
+ const chunks = db.prepare('SELECT COUNT(*) n FROM chunks').get().n;
151
+ const vers = db.prepare('SELECT COUNT(*) n FROM chunk_versions').get().n;
152
+ const mems = db.prepare('SELECT status, COUNT(*) n FROM memories GROUP BY status').all();
153
+ const tiers = db.prepare('SELECT tier, COUNT(*) n FROM chunks GROUP BY tier').all();
154
+ out({ docs, chunks, versions: vers, memories: mems, tiers });
155
+ break;
156
+ }
157
+ default:
158
+ process.stdout.write(`用法:
159
+ ingest <目录> 入库并分块
160
+ tree 目录树
161
+ search <query> 检索
162
+ get <chunkId> 读分块
163
+ edit <chunkId> <文本> 编辑分块(生成新版本)
164
+ history <chunkId> 版本历史
165
+ diff <chunkId> --from 1 --to 2 版本比对
166
+ rollback <chunkId> <版本> 回滚
167
+ remember --kind fact --text "..." 写记忆
168
+ confirm <id> / reject <id>
169
+ memories [--status pending]
170
+ recall <query>
171
+ stats
172
+ `);
173
+ }
174
+ } catch (e) {
175
+ process.stderr.write('ERROR: ' + (e.message || e) + '\n');
176
+ process.exit(1);
177
+ }
@@ -0,0 +1,12 @@
1
+ - insert:
2
+ - id: dsh-cooper-memory
3
+ name: '@deepseek-ai/dsh-mcp-client'
4
+ config:
5
+ serverName: memory
6
+ transport: stdio
7
+ command: node
8
+ args:
9
+ - 'node_modules/dsh-cooper-memory/mcp.mjs'
10
+ env:
11
+ NODE_NO_WARNINGS: '1'
12
+ failOnStartupError: true
package/docs/DETAIL.md ADDED
@@ -0,0 +1,83 @@
1
+ # 本地记忆 / Local memory
2
+
3
+ ## 它做什么
4
+
5
+ DSH 本身没有跨会话记忆:每个新会话从零开始。这个插件在本地维护一个 SQLite 文件,把三件事落在同一处:
6
+
7
+ 1. **长期记忆** —— 用户的偏好、身份、环境事实、待办事项
8
+ 2. **文件夹知识库** —— 任意目录递归入库,保留原始层级
9
+ 3. **可编辑的检索分块** —— 每次修改留版本,可逐行比对、可回滚
10
+
11
+ 全部离线,不需要 Docker,不需要向量数据库。
12
+
13
+ ## 记忆的生命周期
14
+
15
+ ```
16
+ 写入(规则 / LLM 蒸馏 / 手动)
17
+ │
18
+ ▼
19
+ status = pending ← 不参与召回
20
+ │
21
+ 人工确认 ──► status = active ← 参与召回
22
+ └──► status = rejected ← 保留做抑制,不再重复提取
23
+ ```
24
+
25
+ 三条硬规则:
26
+
27
+ - **pending 不注入**:自动提取必然有噪声,未确认的内容不进上下文
28
+ - **拒绝留痕**:被拒的内容参与后续去重,不会反复冒出来
29
+ - **新说法不立刻顶掉旧说法**:只挂 `supersedes` 链接,确认时才替换,旧条目转为 `superseded` 但保留在链条里
30
+
31
+ ## 提取的两种方式
32
+
33
+ | 方式 | 依赖 | 适用 |
34
+ |---|---|---|
35
+ | 规则提取 `memory_extract` | 无 | 离线环境、LLM 不可用时 |
36
+ | LLM 蒸馏 `memory_distill` | 可用的模型通道 | 需要理解隐含表达时,准确率更高 |
37
+
38
+ 两者都会扫描 `~/.dsh/sessions` 下的会话日志(追加式 zstd 多帧,按魔数切帧解压),用文件水位线去重,不会重复扫同一份日志。
39
+
40
+ LLM 蒸馏按 15 条一批切分——整批塞进去时模型容易判定"全是噪声"而返回空数组。
41
+
42
+ ## 分块与检索
43
+
44
+ 三层分块,逐级回退:
45
+
46
+ 1. **标题分块** —— 按 Markdown 标题层级切,面包屑存入 `context_header`(只影响索引,不污染正文)
47
+ 2. **启发式** —— 编号小节、章节标记、换页符、全大写行
48
+ 3. **递归兜底** —— 按 `\n\n` → `\n` → `。` 递归切分,重叠取语义后缀而非定长切片
49
+
50
+ 每层产出都要过校验(无输出、超大文档只有一块、碎片过多、块过小、超预算),不合格降级;**兜底层永不返回空**。
51
+
52
+ 检索是两路召回后融合:
53
+
54
+ - FTS5 trigram(精确子串)
55
+ - 中文二元组 LIKE(短查询、近义措辞)
56
+
57
+ 按加权 RRF 融合,再做 MMR 去冗余(λ=0.7)。
58
+
59
+ ## 版本模型
60
+
61
+ ```
62
+ chunks.version → 当前生效版本
63
+ chunk_versions → 只追加,永不删除
64
+ ```
65
+
66
+ 编辑生成新版本,回滚也是生成新版本(把旧正文写成新版本号),历史完整保留。编辑时可带 `base_version` 做乐观锁,版本对不上就拒绝。
67
+
68
+ ## 配置
69
+
70
+ | 变量 | 默认 | 说明 |
71
+ |---|---|---|
72
+ | `MEM_ROOT` | `~/.dsh-memory` | 数据目录 |
73
+ | `MEM_DB` | `$MEM_ROOT/mem.db` | 数据库路径 |
74
+ | `MEM_WEB_PORT` | `17890` | 浏览器界面端口 |
75
+ | `LLM_KEY_ENV` | `CT_API_KEY` | 从哪个环境变量 / 凭据 ref 取 key |
76
+ | `LLM_MODEL` | 取 profile 配置 | 覆盖模型 |
77
+ | `DEEPSEEK_BASE_URL` | 取 profile 配置 | 覆盖推理网关地址 |
78
+
79
+ ## 边界
80
+
81
+ - 只做词法检索,没有 embedding;语义相近但用词不同的内容可能召回不到
82
+ - 只解析纯文本类文件(md / txt / csv / json / yaml),PDF 与 DOCX 需要先自行转换
83
+ - 自动提取是**辅助**,pending 队列才是质量闸门
package/icon.svg ADDED
@@ -0,0 +1,10 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" width="64" height="64" role="img" aria-label="Local memory">
2
+ <rect x="4" y="4" width="56" height="56" rx="14" fill="#1b2433"/>
3
+ <g fill="none" stroke="#7fb2ff" stroke-width="3" stroke-linecap="round" stroke-linejoin="round">
4
+ <ellipse cx="32" cy="24" rx="14" ry="7"/>
5
+ <path d="M18 24v16c0 3.9 6.3 7 14 7s14-3.1 14-7V24"/>
6
+ <path d="M18 32c0 3.9 6.3 7 14 7s14-3.1 14-7"/>
7
+ </g>
8
+ <circle cx="45" cy="45" r="7" fill="#1b2433" stroke="#8ee6b0" stroke-width="3"/>
9
+ <path d="M42 45l2.4 2.4L48.5 43" fill="none" stroke="#8ee6b0" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round"/>
10
+ </svg>
package/lib/chunk.mjs ADDED
@@ -0,0 +1,283 @@
1
+ // lib/chunk.mjs —— 三层分块器
2
+ //
3
+ // 移植自 WeKnora 的 chunker 设计(见 03-features/04-chunking.md):
4
+ // Tier1 heading → Tier2 heuristic → Tier3 legacy(递归兜底)
5
+ // 每层都要过 Validator(5 条拒绝规则),不合格降级到下一层
6
+ // overlap 取"语义后缀"(段落 > 换行 > 句末),不做定长硬切
7
+ //
8
+ // 简化:不解析 PDF/DOCX,只吃 markdown / 纯文本。
9
+
10
+ const SEPARATORS = ['\n\n', '\n', '。'];
11
+ const ABS_MAX = 7500;
12
+
13
+ const RE_HEADING = /^(#{1,6})\s+(.+?)\s*$/;
14
+ const RE_NUMBERED = /^\s*(?:\d+(?:\.\d+){0,3}[.、)]|[IVX]{1,5}\.)\s+\S/;
15
+ const RE_CHAPTER = /^\s*(?:第[一二三四五六七八九十百千0-9]+[章节篇部]|Chapter\s+\d+|Kapitel\s+\d+)/i;
16
+ const RE_ALLCAPS = /^\s*[A-Z][A-Z0-9 \-\u2013:]{6,60}\s*$/;
17
+ const RE_SEPARATOR = /^\s*(?:-{3,}|={3,}|\*{3,})\s*$/;
18
+
19
+ /** 文档画像:单遍扫描,决定用哪条 Tier 链 */
20
+ export function profile(text) {
21
+ let headings = 0;
22
+ let numbered = 0;
23
+ let chapters = 0;
24
+ let allcaps = 0;
25
+ let formfeeds = 0;
26
+ const levels = new Map();
27
+ for (const line of text.split('\n')) {
28
+ const m = RE_HEADING.exec(line);
29
+ if (m) {
30
+ headings++;
31
+ const lv = m[1].length;
32
+ levels.set(lv, (levels.get(lv) || 0) + 1);
33
+ }
34
+ if (RE_NUMBERED.test(line)) numbered++;
35
+ if (RE_CHAPTER.test(line)) chapters++;
36
+ if (RE_ALLCAPS.test(line)) allcaps++;
37
+ }
38
+ formfeeds = (text.match(/\f/g) || []).length;
39
+ const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
40
+ const latin = (text.match(/[A-Za-z]/g) || []).length;
41
+ const lang = cjk > latin ? 'zh' : latin > 0 ? 'en' : 'mixed';
42
+ const lines = text.split('\n').length || 1;
43
+ return {
44
+ chars: text.length,
45
+ lines,
46
+ headings,
47
+ headingDensity: headings / lines,
48
+ levels: [...levels.entries()],
49
+ numbered,
50
+ chapters,
51
+ allcaps,
52
+ formfeeds,
53
+ lang,
54
+ };
55
+ }
56
+
57
+ /** 选主分割层级:出现 ≥3 次的最浅层级,否则最深 */
58
+ function dominantLevel(p) {
59
+ const stable = p.levels.filter(([, n]) => n >= 3).sort((a, b) => a[0] - b[0]);
60
+ if (stable.length) return stable[0][0];
61
+ const any = p.levels.sort((a, b) => b[0] - a[0]);
62
+ return any.length ? any[0][0] : 0;
63
+ }
64
+
65
+ function selectChain(p) {
66
+ const chain = [];
67
+ if (p.headings >= 3 && p.headingDensity > 0.005 && dominantLevel(p) > 0) chain.push('heading');
68
+ if (p.numbered >= 5 || p.chapters > 0 || p.formfeeds > 0 || p.allcaps >= 5) chain.push('heuristic');
69
+ chain.push('legacy');
70
+ return chain;
71
+ }
72
+
73
+ // ---------- 受保护区间:代码块 / 公式不切开 ----------
74
+ function atomicize(text) {
75
+ const out = [];
76
+ const re = /(```[\s\S]*?```|\$\$[\s\S]*?\$\$)/g;
77
+ let last = 0;
78
+ let m;
79
+ while ((m = re.exec(text))) {
80
+ if (m.index > last) out.push({ atomic: false, text: text.slice(last, m.index) });
81
+ out.push({ atomic: true, text: m[0] });
82
+ last = m.index + m[0].length;
83
+ }
84
+ if (last < text.length) out.push({ atomic: false, text: text.slice(last) });
85
+ return out.length ? out : [{ atomic: false, text }];
86
+ }
87
+
88
+ /** 递归按分隔符切分,返回片段数组 */
89
+ function splitRecursive(text, size, seps = SEPARATORS) {
90
+ if (text.length <= size) return [text];
91
+ for (let i = 0; i < seps.length; i++) {
92
+ const sep = seps[i];
93
+ if (!text.includes(sep)) continue;
94
+ const parts = text.split(sep).map((s, idx, arr) => (idx < arr.length - 1 ? s + sep : s)).filter(Boolean);
95
+ const res = [];
96
+ for (const part of parts) {
97
+ if (part.length <= size) res.push(part);
98
+ else res.push(...splitRecursive(part, size, seps.slice(i + 1)));
99
+ }
100
+ return res;
101
+ }
102
+ // 无分隔符可用:按字符硬切(兜底)
103
+ const res = [];
104
+ for (let i = 0; i < text.length; i += size) res.push(text.slice(i, i + size));
105
+ return res;
106
+ }
107
+
108
+ /** 超长单元先按语义边界细化,保证不产出超过 budget 的块 */
109
+ function refine(unit, budget) {
110
+ if (unit.length <= budget) return [unit];
111
+ const pieces = splitRecursive(unit, budget);
112
+ const out = [];
113
+ for (const p of pieces) {
114
+ if (p.length <= budget) out.push(p);
115
+ else {
116
+ // 仍超长:按换行/空格/字符硬切
117
+ for (let i = 0; i < p.length; i += budget) out.push(p.slice(i, i + budget));
118
+ }
119
+ }
120
+ return out;
121
+ }
122
+
123
+ /** 把片段装成块 */
124
+ function pack(units, size) {
125
+ const flat = [];
126
+ for (const u of units) {
127
+ if (u.length > size) flat.push(...refine(u, size));
128
+ else flat.push(u);
129
+ }
130
+ const chunks = [];
131
+ let cur = '';
132
+ for (const u of flat) {
133
+ if (cur && cur.length + u.length > size) { chunks.push(cur); cur = ''; }
134
+ cur += u;
135
+ }
136
+ if (cur.trim()) chunks.push(cur);
137
+ return chunks.filter((c) => c.trim());
138
+ }
139
+
140
+ // ---------- Tier 1:标题分块 ----------
141
+ function tierHeading(text, size) {
142
+ const p = profile(text);
143
+ const primary = dominantLevel(p);
144
+ if (!primary) return null;
145
+ const lines = text.split('\n');
146
+ const stack = [];
147
+ const sections = [];
148
+ let cur = { crumb: '', body: [] };
149
+ let inFence = false;
150
+ for (const line of lines) {
151
+ if (/^```/.test(line)) inFence = !inFence;
152
+ const m = !inFence && RE_HEADING.exec(line);
153
+ if (m && m[1].length <= primary) {
154
+ if (cur.body.length) sections.push(cur);
155
+ const lv = m[1].length;
156
+ while (stack.length && stack[stack.length - 1].lv >= lv) stack.pop();
157
+ stack.push({ lv, title: m[2] });
158
+ const crumb = stack.map((s) => '#'.repeat(s.lv) + ' ' + s.title).join('\n');
159
+ cur = { crumb, body: [line] };
160
+ } else {
161
+ cur.body.push(line);
162
+ }
163
+ }
164
+ if (cur.body.length) sections.push(cur);
165
+ if (sections.length < 2) return null;
166
+ const chunks = [];
167
+ for (const s of sections) {
168
+ const body = s.body.join('\n').trim();
169
+ if (!body) continue;
170
+ if (body.length + s.crumb.length + 2 <= size) chunks.push({ text: body, crumb: s.crumb });
171
+ else for (const piece of splitRecursive(body, size)) chunks.push({ text: piece, crumb: s.crumb });
172
+ }
173
+ return chunks;
174
+ }
175
+
176
+ // ---------- Tier 2:启发式边界 ----------
177
+ function tierHeuristic(text, size) {
178
+ const lines = text.split('\n');
179
+ const marks = [];
180
+ let inFence = false;
181
+ for (let i = 0; i < lines.length; i++) {
182
+ const line = lines[i];
183
+ if (/^```/.test(line)) { inFence = !inFence; continue; }
184
+ if (inFence) continue;
185
+ let prio = 0;
186
+ if (line.includes('\f')) prio = 100;
187
+ else if (RE_NUMBERED.test(line)) prio = 90;
188
+ else if (RE_CHAPTER.test(line)) prio = 85;
189
+ else if (RE_ALLCAPS.test(line)) prio = 70;
190
+ else if (RE_SEPARATOR.test(line)) prio = 60;
191
+ if (prio) marks.push({ line: i, prio });
192
+ }
193
+ if (marks.length < 3) return null;
194
+ marks.sort((a, b) => a.line - b.line || b.prio - a.prio);
195
+ const cuts = [0, ...marks.map((m) => m.line).filter((n) => n > 0), lines.length];
196
+ const uniq = [...new Set(cuts)].sort((a, b) => a - b);
197
+ const chunks = [];
198
+ for (let i = 0; i < uniq.length - 1; i++) {
199
+ const body = lines.slice(uniq[i], uniq[i + 1]).join('\n').trim();
200
+ if (!body) continue;
201
+ if (body.length <= size) chunks.push({ text: body, crumb: '' });
202
+ else for (const piece of splitRecursive(body, size)) chunks.push({ text: piece, crumb: '' });
203
+ }
204
+ return chunks.length >= 2 ? chunks : null;
205
+ }
206
+
207
+ // ---------- Tier 3:递归兜底 ----------
208
+ function tierLegacy(text, size) {
209
+ const units = [];
210
+ for (const seg of atomicize(text)) {
211
+ if (seg.atomic) units.push(seg.text);
212
+ else units.push(...splitRecursive(seg.text, size));
213
+ }
214
+ return pack(units, size).map((t) => ({ text: t, crumb: '' }));
215
+ }
216
+
217
+ // ---------- Validator:5 条拒绝规则 ----------
218
+ export function validate(chunks, size) {
219
+ const sizes = chunks.map((c) => c.text.length);
220
+ const total = sizes.reduce((a, b) => a + b, 0);
221
+ if (!chunks.length) return 'no chunks produced';
222
+ if (total > size * 2 && chunks.length === 1) return 'single chunk for large document';
223
+ const tiny = sizes.filter((s, i) => s < 50 && i < sizes.length - 1).length;
224
+ if (tiny > 2 && tiny > chunks.length / 4) return 'too many tiny chunks';
225
+ const max = Math.max(...sizes);
226
+ if (max < size / 4) return 'all chunks far below target size';
227
+ if (max > size * 2) return 'chunk exceeds 2x target size';
228
+ return null;
229
+ }
230
+
231
+ /** 语义重叠:从上一块尾部取一段,吸附到最近的语义边界 */
232
+ export function applyOverlap(chunks, overlap) {
233
+ if (!overlap || chunks.length < 2) return chunks;
234
+ const out = [chunks[0]];
235
+ for (let i = 1; i < chunks.length; i++) {
236
+ const prev = chunks[i - 1].text;
237
+ const winStart = Math.max(0, prev.length - overlap - 4);
238
+ const win = prev.slice(winStart);
239
+ let cut = -1;
240
+ for (const sep of ['\n\n', '\n', '。', '?', '!', '. ']) {
241
+ const idx = win.lastIndexOf(sep);
242
+ if (idx >= 0) { cut = winStart + idx + sep.length; break; }
243
+ }
244
+ const tail = cut >= 0 ? prev.slice(cut) : '';
245
+ out.push({ ...chunks[i], text: tail + chunks[i].text });
246
+ }
247
+ return out;
248
+ }
249
+
250
+ /**
251
+ * 主入口
252
+ * @returns {{chunks: Array<{ordinal,text,heading,context_header,tier}>, tier, rejected}}
253
+ */
254
+ export function splitDoc(text, cfg = {}) {
255
+ const size = cfg.chunk_size || 512;
256
+ const overlap = Math.min(cfg.chunk_overlap ?? 80, Math.floor(size / 2));
257
+ const forced = cfg.strategy && cfg.strategy !== 'auto' ? cfg.strategy : null;
258
+ const p = profile(text);
259
+ const chain = forced ? [forced, 'legacy'] : selectChain(p);
260
+ const rejected = [];
261
+ for (const tier of chain) {
262
+ let raw = null;
263
+ if (tier === 'heading') raw = tierHeading(text, size);
264
+ else if (tier === 'heuristic') raw = tierHeuristic(text, size);
265
+ else raw = tierLegacy(text, size);
266
+ if (!raw) { rejected.push({ tier, reason: 'tier produced nothing' }); continue; }
267
+ const reason = validate(raw, size);
268
+ // 兜底层永不返回空(对齐 WeKnora:legacy 即使未通过校验也采纳)
269
+ const isLast = tier === 'legacy' || tier === chain[chain.length - 1];
270
+ if (reason && !isLast) { rejected.push({ tier, reason }); continue; }
271
+ if (reason && isLast) rejected.push({ tier, reason: reason + '(兜底层强制采纳)' });
272
+ const withOverlap = applyOverlap(raw, overlap);
273
+ const chunks = withOverlap.map((c, i) => ({
274
+ ordinal: i,
275
+ text: c.text.trim(),
276
+ heading: (c.crumb.split('\n').pop() || '').replace(/^#+\s*/, ''),
277
+ context_header: c.crumb || '',
278
+ tier,
279
+ })).filter((c) => c.text);
280
+ return { chunks, tier, rejected, profile: p };
281
+ }
282
+ return { chunks: [], tier: 'none', rejected, profile: p };
283
+ }