@modusensus/dsh-mneme 0.12.0 → 0.14.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +51 -20
  2. package/bin/dsh-mneme-choro.mjs +9 -1
  3. package/docs/CONFIGURATION.md +47 -6
  4. package/docs/DIRECTORIES.md +114 -0
  5. package/docs/DISTILL.md +64 -0
  6. package/docs/DSH-GO.md +97 -0
  7. package/docs/ENTITIES.md +5 -3
  8. package/docs/FORGET.md +77 -0
  9. package/docs/KB.md +110 -0
  10. package/docs/SLEEP.md +28 -3
  11. package/docs/STORAGE.md +23 -0
  12. package/lib/api-standalone.js +4 -1
  13. package/lib/api.js +788 -30
  14. package/lib/attachment-text.js +122 -0
  15. package/lib/choro-command.js +9 -2
  16. package/lib/client.js +4627 -1409
  17. package/lib/config.js +166 -1
  18. package/lib/continuity.js +7 -6
  19. package/lib/diary/auto.js +190 -0
  20. package/lib/diary/compose.js +440 -0
  21. package/lib/diary/dir.js +59 -0
  22. package/lib/diary/export.js +97 -0
  23. package/lib/diary/index.js +26 -0
  24. package/lib/diary/library.js +189 -0
  25. package/lib/diary/qq-inbox.js +142 -0
  26. package/lib/diary/rollover.js +61 -0
  27. package/lib/diary/session-provider.js +122 -0
  28. package/lib/dream/decisions.js +95 -6
  29. package/lib/dream/sleep.js +760 -19
  30. package/lib/dream/triage.js +139 -0
  31. package/lib/dream.js +168 -21
  32. package/lib/dsh-go/host.js +162 -0
  33. package/lib/dsh-go/panel.js +2091 -0
  34. package/lib/dsh-go/text.js +342 -0
  35. package/lib/dsh-go/views/common.js +68 -0
  36. package/lib/dsh-go/views/entity.js +48 -0
  37. package/lib/dsh-go/views/memory.js +198 -0
  38. package/lib/dsh-go/views/root.js +30 -0
  39. package/lib/dsh-go/views/settings.js +398 -0
  40. package/lib/dsh-go/views/status.js +51 -0
  41. package/lib/embedding.js +42 -8
  42. package/lib/entities/extractor.js +36 -7
  43. package/lib/hot-memory.js +22 -11
  44. package/lib/index.js +123 -3
  45. package/lib/inject.js +203 -20
  46. package/lib/kb/index.js +309 -0
  47. package/lib/kb/routes.js +220 -0
  48. package/lib/kb/tools.js +148 -0
  49. package/lib/lang.js +98 -4
  50. package/lib/peak-hours.js +57 -2
  51. package/lib/semantic.js +347 -60
  52. package/lib/service.js +268 -69
  53. package/lib/settings.js +128 -15
  54. package/lib/store.js +585 -26
  55. package/lib/summarize.js +38 -11
  56. package/lib/tools.js +35 -4
  57. package/lib/user-settings.js +128 -32
  58. package/lib/vector-index.js +76 -6
  59. package/lib/wiki/claims.js +267 -0
  60. package/lib/wiki/cli-hint.js +90 -0
  61. package/lib/wiki/queries.js +282 -2
  62. package/lib/wiki/render.js +74 -0
  63. package/lib/wiki/t0.js +43 -6
  64. package/lib/wiki/t1-jobs.js +232 -0
  65. package/lib/wiki/t1.js +145 -32
  66. package/package.json +39 -1
  67. package/scripts/release-docs.mjs +7 -1
  68. package/scripts/release-notes.mjs +4 -1
  69. package/src/api-standalone.js +4 -1
  70. package/src/api.js +788 -30
  71. package/src/attachment-text.js +122 -0
  72. package/src/choro-command.js +9 -2
  73. package/src/config.js +166 -1
  74. package/src/continuity.js +7 -6
  75. package/src/diary/auto.js +190 -0
  76. package/src/diary/compose.js +440 -0
  77. package/src/diary/dir.js +59 -0
  78. package/src/diary/export.js +97 -0
  79. package/src/diary/index.js +26 -0
  80. package/src/diary/library.js +189 -0
  81. package/src/diary/qq-inbox.js +142 -0
  82. package/src/diary/rollover.js +61 -0
  83. package/src/diary/session-provider.js +122 -0
  84. package/src/dream/decisions.js +95 -6
  85. package/src/dream/sleep.js +760 -19
  86. package/src/dream/triage.js +139 -0
  87. package/src/dream.js +168 -21
  88. package/src/dsh-go/host.js +162 -0
  89. package/src/dsh-go/panel.js +2091 -0
  90. package/src/dsh-go/text.js +342 -0
  91. package/src/dsh-go/views/common.js +68 -0
  92. package/src/dsh-go/views/entity.js +48 -0
  93. package/src/dsh-go/views/memory.js +198 -0
  94. package/src/dsh-go/views/root.js +30 -0
  95. package/src/dsh-go/views/settings.js +398 -0
  96. package/src/dsh-go/views/status.js +51 -0
  97. package/src/embedding.js +42 -8
  98. package/src/entities/extractor.js +36 -7
  99. package/src/hot-memory.js +22 -11
  100. package/src/index.js +123 -3
  101. package/src/inject.js +203 -20
  102. package/src/kb/index.js +309 -0
  103. package/src/kb/routes.js +220 -0
  104. package/src/kb/tools.js +148 -0
  105. package/src/lang.js +98 -4
  106. package/src/peak-hours.js +57 -2
  107. package/src/semantic.js +347 -60
  108. package/src/service.js +268 -69
  109. package/src/settings.js +128 -15
  110. package/src/store.js +585 -26
  111. package/src/summarize.js +38 -11
  112. package/src/tools.js +35 -4
  113. package/src/user-settings.js +128 -32
  114. package/src/vector-index.js +76 -6
  115. package/src/wiki/claims.js +267 -0
  116. package/src/wiki/cli-hint.js +90 -0
  117. package/src/wiki/queries.js +282 -2
  118. package/src/wiki/render.js +74 -0
  119. package/src/wiki/t0.js +43 -6
  120. package/src/wiki/t1-jobs.js +232 -0
  121. package/src/wiki/t1.js +145 -32
  122. package/test/api-routes-gaps.test.js +63 -0
  123. package/test/api.test.js +385 -4
  124. package/test/attachment-text.test.js +140 -0
  125. package/test/bm25-corpus-cache.test.js +145 -0
  126. package/test/capacity-warning.test.js +262 -0
  127. package/test/client.test.js +1051 -185
  128. package/test/conflict-freeze.test.js +45 -1
  129. package/test/conflict-ledger-enforce.test.js +452 -0
  130. package/test/conflict-ledger.test.js +242 -0
  131. package/test/continuity.test.js +8 -1
  132. package/test/contrast.test.js +124 -12
  133. package/test/diary-library.test.js +187 -0
  134. package/test/diary.test.js +576 -0
  135. package/test/dream-baseline-clamp.test.js +174 -0
  136. package/test/dream-maxtokens-fallback.test.js +3 -3
  137. package/test/dream.test.js +62 -1
  138. package/test/dsh-go-host.test.js +134 -0
  139. package/test/dsh-go-panel-actions.test.js +854 -0
  140. package/test/dsh-go-panel.test.js +251 -0
  141. package/test/embedding-ready.test.js +33 -3
  142. package/test/entities.test.js +73 -0
  143. package/test/forget-corrections.test.js +70 -0
  144. package/test/graph-api.test.js +124 -0
  145. package/test/hot-memory.test.js +48 -0
  146. package/test/image-memory.test.js +171 -0
  147. package/test/inject-parent-gate.test.js +6 -3
  148. package/test/inject.test.js +59 -0
  149. package/test/issue-617-step1-query-and-fusion.test.js +361 -0
  150. package/test/kb-api-wiring.test.js +128 -0
  151. package/test/kb-collections.test.js +272 -0
  152. package/test/kb-routes.test.js +379 -0
  153. package/test/mint-source.test.js +26 -0
  154. package/test/panel-chart-scale.test.js +90 -0
  155. package/test/panel-tokens.test.js +245 -0
  156. package/test/peak-hours-observability.test.js +70 -0
  157. package/test/persona-api.test.js +3 -3
  158. package/test/persona-by-session.test.js +568 -0
  159. package/test/persona-settings.test.js +7 -7
  160. package/test/reflection-update-gate.test.js +129 -0
  161. package/test/reindex-backfill.test.js +334 -0
  162. package/test/reindex-scheduler.test.js +369 -0
  163. package/test/release-docs.test.js +20 -0
  164. package/test/release-notes.test.js +25 -0
  165. package/test/scope-candidates.test.js +249 -0
  166. package/test/security-doc.test.js +10 -0
  167. package/test/sleep-action-set.test.js +14 -0
  168. package/test/sleep-heat.test.js +49 -0
  169. package/test/sleep.test.js +635 -0
  170. package/test/standalone-api.test.js +24 -0
  171. package/test/store.test.js +20 -0
  172. package/test/summarize.test.js +26 -0
  173. package/test/user-settings-segments.test.js +528 -0
  174. package/test/vector-index.test.js +120 -6
  175. package/test/wiki-api.test.js +463 -1
  176. package/test/wiki-claims.test.js +194 -0
  177. package/test/wiki-cli-hint.test.js +118 -0
  178. package/test/wiki-provenance.test.js +202 -0
  179. package/test/wiki-queries.test.js +128 -0
  180. package/test/wiki-t0.test.js +77 -1
  181. package/test/wiki-t1-jobs.test.js +125 -0
  182. package/test/wiki-t1.test.js +226 -1
@@ -0,0 +1,309 @@
1
+ // dsh-mneme/src/kb/index.js
2
+ // 知识库第二层(§3.3/§3.4):集合 + 文档入库(本地文件 / 网页 / 仓库地志页)+
3
+ // 分块 + BM25 检索。真源在 kb_* 三张表里(不像 chorography 那样以磁盘 MD 为真源
4
+ // ——知识库收的就是「外来原文的副本」,用户随时可能把原文件挪走/删掉,检索面不能
5
+ // 跟着消失),所以读侧零 IO。
6
+ //
7
+ // 物理隔离:本模块只碰 kb_* 三张表,绝不写 memories,也不进巩固/质量过滤任何池子
8
+ // (与 wiki 行同一条纪律,也是 §3.3 的红线)。
9
+ import { readFileSync, statSync } from "node:fs";
10
+ import { basename } from "node:path";
11
+ import { createHash } from "node:crypto";
12
+ import { createBM25Index } from "../search/bm25.js";
13
+
14
+ // 单文档上限:知识库是检索面不是文件仓,几十兆的语料进来只会把 BM25 全量重算
15
+ // 变成卡顿源。超限给明确 error 回执(面板要能显示「这份为什么不进来」),不静默截断。
16
+ const DEFAULT_MAX_BYTES = 2 * 1024 * 1024;
17
+ // 单块目标长度:BM25 的 length normalization 对超长块惩罚很重,而太碎的块丢上下文。
18
+ // 1200 字符约等于中文 600 字,是「一段完整论述」的常见尺度。
19
+ const CHUNK_TARGET_CHARS = 1200;
20
+
21
+ const KINDS = new Set(["file", "url", "choro_page"]);
22
+
23
+ function sha256(text) {
24
+ return createHash("sha256").update(text, "utf8").digest("hex");
25
+ }
26
+
27
+ /**
28
+ * 按行切块,**严格连续、不跳行**——这是 readDoc 能逐字节还原原文、以及检索命中
29
+ * 能给出准确行号(起始行 = 1 + 前面所有块的行数)的前提。破坏它这两件事同时坏。
30
+ * `## ` 标题开启新节并重置 target(一节尽量整块进来,除非它自己就超长);
31
+ * 超长单行单独成块不拆(行号与原文还原都靠这个假设)。
32
+ */
33
+ function chunkContent(content) {
34
+ const lines = String(content ?? "").replace(/\r\n/g, "\n").split("\n");
35
+ const chunks = [];
36
+ let heading = "";
37
+ let buf = [];
38
+ let bufChars = 0;
39
+
40
+ const flush = () => {
41
+ if (!buf.length) return;
42
+ // 一个空行都不剥:块间用 \n 直接拼接就是原文(readDoc 靠这条),而且空白行
43
+ // 不产生 token,对 BM25 的 len 分母零成本——剥它反而是纯损。
44
+ chunks.push({ heading, text: buf.join("\n") });
45
+ buf = [];
46
+ bufChars = 0;
47
+ };
48
+
49
+ for (const line of lines) {
50
+ const isH2 = /^##\s+/.test(line);
51
+ const startsSection = isH2 && buf.length > 0;
52
+ if (startsSection) flush();
53
+ if (isH2) heading = line.replace(/^##\s+/, "").trim();
54
+ buf.push(line);
55
+ bufChars += line.length + 1;
56
+ if (bufChars >= CHUNK_TARGET_CHARS) flush();
57
+ }
58
+ flush();
59
+ return chunks;
60
+ }
61
+
62
+ function makeExcerpt(text, max = 200) {
63
+ return String(text ?? "").replace(/\s+/g, " ").trim().slice(0, max);
64
+ }
65
+
66
+ /**
67
+ * @param {object} deps
68
+ * @param {object} deps.store createStore() 的返回值(用到 kb* 那组方法)
69
+ * @param {object} [deps.embedder] v1 不接向量(BM25 打底);保留位,配了也不影响
70
+ * 检索结果,接向量是下一档的事
71
+ * @param {object} [deps.cfg]
72
+ */
73
+ export function createKb({ store, embedder = null, cfg = {} } = {}) {
74
+ if (!store || !store.db) throw new TypeError("createKb: store is required");
75
+ const maxBytes = Number(cfg?.kbMaxBytes) > 0 ? Number(cfg.kbMaxBytes) : DEFAULT_MAX_BYTES;
76
+ // 检索索引按集合懒物化:kb 是 opt-in 子系统,一次 ingest 之后反复检索的收益远大于
77
+ // 「每次重建一遍 corpus」。写路径(ingest / delete / upsert)令其失效。
78
+ const indexCache = new Map();
79
+
80
+ function resolveCollection(key) {
81
+ return store.kbCollectionByKey(String(key ?? "").trim());
82
+ }
83
+
84
+ function invalidate(collectionKey) {
85
+ if (collectionKey == null) indexCache.clear();
86
+ else indexCache.delete(String(collectionKey));
87
+ }
88
+
89
+ function listCollections() {
90
+ return store.listKbCollections().map((r) => ({
91
+ key: r.key,
92
+ title: r.title,
93
+ description: r.description ?? "",
94
+ docs: Number(r.doc_count ?? 0),
95
+ chunks: Number(r.chunk_count ?? 0),
96
+ updatedAt: r.updated_at,
97
+ }));
98
+ }
99
+
100
+ function upsertCollection({ key, title, description = "" } = {}) {
101
+ const k = String(key ?? "").trim();
102
+ if (k === "") throw new TypeError("upsertCollection: key is required");
103
+ const row = store.upsertKbCollection({
104
+ key: k,
105
+ title: String(title ?? "").trim() || k,
106
+ description: String(description ?? ""),
107
+ at: new Date().toISOString(),
108
+ });
109
+ invalidate(k);
110
+ return { key: row.key, title: row.title, description: row.description ?? "" };
111
+ }
112
+
113
+ function deleteCollection(key) {
114
+ const k = String(key ?? "").trim();
115
+ if (k === "") return false;
116
+ const gone = store.deleteKbCollection(k);
117
+ invalidate(k);
118
+ return gone;
119
+ }
120
+
121
+ function listDocs({ collection } = {}) {
122
+ const col = resolveCollection(collection);
123
+ if (!col) return [];
124
+ return store.listKbDocs(col.id).map((d) => ({
125
+ relKey: d.rel_key,
126
+ kind: d.kind,
127
+ title: d.title,
128
+ bytes: Number(d.bytes ?? 0),
129
+ indexedAt: d.indexed_at,
130
+ status: d.status,
131
+ error: d.error ?? null,
132
+ }));
133
+ }
134
+
135
+ /** 读来源正文。三种 kind 各自的最短路径,出错一律抛,由 ingest 收成 error 回执。 */
136
+ async function readSource({ kind, source }) {
137
+ const src = String(source ?? "").trim();
138
+ if (src === "") throw new Error("empty-source");
139
+ if (kind === "url") {
140
+ if (typeof fetch !== "function") throw new Error("fetch-unavailable");
141
+ const res = await fetch(src, { redirect: "follow" });
142
+ if (!res.ok) throw new Error(`http-${res.status}`);
143
+ const raw = await res.text();
144
+ return { text: raw, relKey: src, title: src };
145
+ }
146
+ // file / choro_page:choro_page 的 source 就是该地志页 MD 的绝对路径(面板从
147
+ // 地志页「导入」时拿到的正是它)。kind 只作分类与展示,读取路径与普通文件一致
148
+ // ——不为它去跨域读 choro_* 表,那会把两条线的写作用域搅在一起。
149
+ let size = 0;
150
+ try {
151
+ size = statSync(src).size;
152
+ } catch (cause) {
153
+ throw new Error(cause.code === "ENOENT" ? "file-not-found" : "file-unreadable");
154
+ }
155
+ if (size > maxBytes) throw new Error("too-large");
156
+ const text = readFileSync(src, "utf8");
157
+ return { text, relKey: src, title: basename(src) };
158
+ }
159
+
160
+ /**
161
+ * 入库。同 rel_key + 同 sha256 是 no-op(回执 skipped=true,chunks 是库里现有的
162
+ * 块数不是重新切出来的块数)——这条是整个幂等契约的落点,编辑器保存一次不能
163
+ * 触发全量重切块。
164
+ */
165
+ async function ingest({ collection, kind, source, title } = {}) {
166
+ const colKey = String(collection ?? "").trim();
167
+ const k = String(kind ?? "").trim();
168
+ const now = new Date().toISOString();
169
+ if (!KINDS.has(k)) {
170
+ return { docId: null, chunks: 0, skipped: false, error: "bad-kind" };
171
+ }
172
+ const col = resolveCollection(colKey);
173
+ if (!col) {
174
+ // 不静默建集合:拼错 key 的入库会把资料塞进一个凭空出现的集合里,用户永远
175
+ // 找不到。面板的流程是先建集合再入库,工具侧也是先 list 再 ingest。
176
+ return { docId: null, chunks: 0, skipped: false, error: "collection-not-found" };
177
+ }
178
+
179
+ let body;
180
+ try {
181
+ body = await readSource({ kind: k, source });
182
+ } catch (cause) {
183
+ const message = String(cause?.message ?? cause);
184
+ const docId = store.markKbDocError({
185
+ collectionId: col.id,
186
+ relKey: String(source ?? "").trim() || "(empty)",
187
+ kind: k,
188
+ sourcePath: String(source ?? "").trim() || null,
189
+ title: String(title ?? "").trim() || String(source ?? ""),
190
+ error: message,
191
+ at: now,
192
+ });
193
+ invalidate(colKey);
194
+ return { docId, chunks: 0, skipped: false, error: message };
195
+ }
196
+
197
+ const chunks = chunkContent(body.text);
198
+ const result = store.replaceKbDoc({
199
+ collectionId: col.id,
200
+ relKey: body.relKey,
201
+ kind: k,
202
+ sourcePath: body.relKey,
203
+ title: String(title ?? "").trim() || body.title,
204
+ bytes: Buffer.byteLength(body.text, "utf8"),
205
+ sha256: sha256(body.text),
206
+ at: now,
207
+ chunks: chunks.map((c, i) => ({ ordinal: i, heading: c.heading, text: c.text })),
208
+ });
209
+ invalidate(colKey);
210
+ return { docId: result.docId, chunks: result.chunks, skipped: result.skipped };
211
+ }
212
+
213
+ /** 集合级语料的 BM25 索引(懒物化,写路径失效)。 */
214
+ function indexFor(col) {
215
+ const cacheKey = col ? String(col.key) : "*";
216
+ const cached = indexCache.get(cacheKey);
217
+ if (cached) return cached;
218
+ const rows = store.kbChunksForSearch(col ? col.id : null);
219
+ const docs = rows.map((r) => ({
220
+ id: r.id,
221
+ // 标题与本节标题都进索引文本:用户搜「安装」时,命中「## 安装」比命中正文里
222
+ // 偶然出现的「安装」更该排前面,而 BM25 只看词频,标题必须显式入语料。
223
+ title: `${r.title} ${r.heading ?? ""}`.trim(),
224
+ content: r.text ?? "",
225
+ row: r,
226
+ }));
227
+ const built = { index: createBM25Index(docs), byId: new Map(docs.map((d) => [d.id, d])) };
228
+ indexCache.set(cacheKey, built);
229
+ return built;
230
+ }
231
+
232
+ function search({ query, collection = null, limit = 20 } = {}) {
233
+ const q = String(query ?? "").trim();
234
+ if (q === "") return { hits: [], unreadable: [] };
235
+ let col = null;
236
+ if (collection != null && String(collection).trim() !== "") {
237
+ col = resolveCollection(collection);
238
+ if (!col) return { hits: [], unreadable: [] };
239
+ }
240
+ const cap = Math.max(1, Math.min(100, Number(limit) || 20));
241
+ const { index, byId } = indexFor(col);
242
+ const ranked = index.search(q, { limit: cap });
243
+ // 行号:块是**严格连续按行切**的,所以某块起始行 = 1 + 它前面所有块的行数。
244
+ // 这里按需回算(只算命中的那一篇),避免给 kb_chunks 加一列只为省这点算术。
245
+ const lineCache = new Map();
246
+ const lineOf = (row) => {
247
+ if (!lineCache.has(row.doc_id)) {
248
+ const all = store.kbChunksByDoc(row.doc_id);
249
+ let cursor = 1;
250
+ const map = new Map();
251
+ for (const c of all) {
252
+ map.set(c.id, cursor);
253
+ // 块间用 \n 拼接,所以上一块的行数就是游标增量(不 +1)。
254
+ cursor += String(c.text ?? "").split("\n").length;
255
+ }
256
+ lineCache.set(row.doc_id, map);
257
+ }
258
+ return lineCache.get(row.doc_id).get(row.id) ?? 1;
259
+ };
260
+ const hits = ranked.map((hit) => {
261
+ const row = byId.get(hit.id).row;
262
+ return {
263
+ collection: row.collection_key,
264
+ relKey: row.rel_key,
265
+ title: row.title,
266
+ heading: row.heading ?? "",
267
+ line: lineOf(row),
268
+ excerpt: makeExcerpt(row.text),
269
+ score: hit.score,
270
+ };
271
+ });
272
+ return { hits, unreadable: [] };
273
+ }
274
+
275
+ /** 读整篇:块按行连续切,用 \n 拼回即是原文(见 chunkContent 的注释)。 */
276
+ function readDoc({ collection, relKey } = {}) {
277
+ const col = resolveCollection(collection);
278
+ if (!col) return null;
279
+ const doc = store.kbDocByRelKey(col.id, String(relKey ?? ""));
280
+ if (!doc) return null;
281
+ const chunks = store.kbChunksByDoc(doc.id);
282
+ return {
283
+ title: doc.title,
284
+ content: chunks.map((c) => c.text).join("\n"),
285
+ kind: doc.kind,
286
+ relKey: doc.rel_key,
287
+ status: doc.status,
288
+ error: doc.error ?? null,
289
+ };
290
+ }
291
+
292
+ function dispose() {
293
+ indexCache.clear();
294
+ try {
295
+ embedder?.dispose?.();
296
+ } catch { /* 释放期的失败不该盖住调用方的正常退出 */ }
297
+ }
298
+
299
+ return {
300
+ listCollections,
301
+ upsertCollection,
302
+ deleteCollection,
303
+ listDocs,
304
+ ingest,
305
+ search,
306
+ readDoc,
307
+ dispose,
308
+ };
309
+ }
@@ -0,0 +1,220 @@
1
+ // dsh-mneme/src/kb/routes.js
2
+ // 知识库第二层的 HTTP 面(契约 §3.6 的路径表)。**这张表是 C 线面板的合同**:
3
+ // 路径、方法、请求体字段、回执形状都照表写死,两边不许各自发挥。
4
+ //
5
+ // 本文件不判 cfg.kbEnabled——那是 A 线 api.js 里 kbGate 的职责(一处判定,不两处
6
+ // 分叉)。这里只吃传进来的 gate;万一没传(单测直接装路由),退化成「kb 未装配 →
7
+ // 503」这一条最小围栏,语义与 #497 先例一致:503 = 环境缺件、409 = 用户没开。
8
+ //
9
+ // kb 允许为 null(未装配),此时一律 503 而不是抛:面板要靠状态码渲染空态。
10
+ function defaultGate(kb, sendJson, res) {
11
+ if (!kb) {
12
+ sendJson(res, 503, { error: "unavailable", reason: "kb-unavailable" });
13
+ return false;
14
+ }
15
+ return true;
16
+ }
17
+
18
+ function queryOf(req) {
19
+ return new URL(req.url, "http://localhost").searchParams;
20
+ }
21
+
22
+ /**
23
+ * @param {(route: object) => void} register createRouteRegistrar 的注册函数
24
+ * @param {object} deps
25
+ * @param {object|null} deps.kb createKb() 的返回值,未装配时 null
26
+ * @param {object} deps.cfg
27
+ * @param {(res, status, body) => void} deps.sendJson
28
+ * @param {(req, res) => Promise<string|null>} deps.readBody 超限时自己回过 413 并返回 null
29
+ * @param {(text: string) => object} deps.parseBody 坏 JSON 回 {}
30
+ * @param {(req, res) => boolean} [deps.gate] 关 409 / 缺件 503,false = 已回执
31
+ */
32
+ export function registerKbRoutes(register, { kb = null, cfg = {}, sendJson, readBody, parseBody, gate } = {}) {
33
+ if (typeof register !== "function") throw new TypeError("registerKbRoutes: register is required");
34
+ if (typeof sendJson !== "function") throw new TypeError("registerKbRoutes: sendJson is required");
35
+
36
+ const allowed = (req, res) => (typeof gate === "function" ? gate(req, res) : defaultGate(kb, sendJson, res));
37
+
38
+ // 读 JSON body 的唯一入口(三条写路由共用)。返回 null = readBody 已经回过 413,
39
+ // 调用点**必须**直接 return:否则会拿着空对象继续往 store 写,413 之后又落一次库。
40
+ // 坏 JSON 退化成 {},由各 handler 的必填校验收成 400。
41
+ const readJson = async (req, res) => {
42
+ const text = await readBody(req, res);
43
+ if (text === null) return null;
44
+ if (typeof parseBody === "function") return parseBody(text);
45
+ try {
46
+ return JSON.parse(text);
47
+ } catch {
48
+ return {};
49
+ }
50
+ };
51
+
52
+ // --- 集合 ---------------------------------------------------------------
53
+ // 一条路径上读(GET)与写(POST/DELETE)混在一起,只能声明一个 policy,取更严的
54
+ // `token-write`——与 /profile 同款处理(src/api.js:495-514)。registrar 对同一
55
+ // kind+path 重复登记直接抛错,所以不能拆成两条。
56
+ //
57
+ // 失败闭环:kb.collections 里出现过的文档之外一切都不动。不存在「删了一半」——
58
+ // deleteCollection 是整个事务(store.deleteKbCollection 里有注释)。
59
+ register({
60
+ kind: "exact",
61
+ path: "/api/dsh-mneme/kb/collections",
62
+ policy: "token-write",
63
+ handler(req, res) {
64
+ if (!allowed(req, res)) return;
65
+ if (req.method === "GET") {
66
+ sendJson(res, 200, { collections: kb.listCollections() });
67
+ return;
68
+ }
69
+ if (req.method === "DELETE") {
70
+ // key 走 query:集合 key 允许含斜杠,塞进路径会把 router 的 exact 匹配搅坏。
71
+ const key = String(queryOf(req).get("key") ?? "").trim();
72
+ if (key === "") {
73
+ sendJson(res, 400, { error: "key-required" });
74
+ return;
75
+ }
76
+ sendJson(res, 200, { ok: true, deleted: kb.deleteCollection(key) === true });
77
+ return;
78
+ }
79
+ if (req.method !== "POST") {
80
+ sendJson(res, 405, { error: "method-not-allowed" });
81
+ return;
82
+ }
83
+ // 必须 return:handler 是 async 时吞掉 Promise,调用点(测试里的 await、
84
+ // 宿主里的 invoke)就看不到回执已经发出——与 choro/search 踩过的坑同源。
85
+ return readJson(req, res).then((body) => {
86
+ if (body === null) return;
87
+ const key = String(body?.key ?? "").trim();
88
+ if (key === "") {
89
+ sendJson(res, 400, { error: "key-required" });
90
+ return;
91
+ }
92
+ const saved = kb.upsertCollection({
93
+ key,
94
+ title: body?.title,
95
+ description: body?.description
96
+ });
97
+ sendJson(res, 200, { ok: true, key: saved.key });
98
+ });
99
+ }
100
+ });
101
+
102
+ // --- 文档列表 -----------------------------------------------------------
103
+ register({
104
+ kind: "exact",
105
+ path: "/api/dsh-mneme/kb/docs",
106
+ policy: "token",
107
+ handler(req, res) {
108
+ if (!allowed(req, res)) return;
109
+ const collection = String(queryOf(req).get("collection") ?? "").trim();
110
+ if (collection === "") {
111
+ sendJson(res, 400, { error: "collection-required" });
112
+ return;
113
+ }
114
+ sendJson(res, 200, { docs: kb.listDocs({ collection }) });
115
+ }
116
+ });
117
+
118
+ // --- 读整篇 -------------------------------------------------------------
119
+ register({
120
+ kind: "exact",
121
+ path: "/api/dsh-mneme/kb/doc",
122
+ policy: "token",
123
+ handler(req, res) {
124
+ if (!allowed(req, res)) return;
125
+ const url = queryOf(req);
126
+ const collection = String(url.get("collection") ?? "").trim();
127
+ const relKey = String(url.get("relKey") ?? "");
128
+ if (collection === "" || relKey === "") {
129
+ sendJson(res, 400, { error: "collection-and-relKey-required" });
130
+ return;
131
+ }
132
+ const doc = kb.readDoc({ collection, relKey });
133
+ if (!doc) {
134
+ sendJson(res, 404, { error: "doc-not-found", collection, relKey });
135
+ return;
136
+ }
137
+ sendJson(res, 200, {
138
+ title: doc.title,
139
+ content: doc.content,
140
+ kind: doc.kind,
141
+ relKey: doc.relKey,
142
+ status: doc.status
143
+ });
144
+ }
145
+ });
146
+
147
+ // --- 入库 ---------------------------------------------------------------
148
+ // 入库是异步的(url 要 fetch),handler 必须把整条 Promise 链 return 出去——
149
+ // 不 return 会在响应发出之前就返回,body 恒空(choro/search 那条踩过同一个坑)。
150
+ register({
151
+ kind: "exact",
152
+ path: "/api/dsh-mneme/kb/ingest",
153
+ policy: "token-write",
154
+ handler(req, res) {
155
+ if (!allowed(req, res)) return;
156
+ if (req.method !== "POST") {
157
+ sendJson(res, 405, { error: "method-not-allowed" });
158
+ return;
159
+ }
160
+ return readJson(req, res).then(async (body) => {
161
+ if (body === null) return;
162
+ const collection = String(body?.collection ?? "").trim();
163
+ const kind = String(body?.kind ?? "").trim();
164
+ const source = String(body?.source ?? "").trim();
165
+ if (collection === "" || kind === "" || source === "") {
166
+ sendJson(res, 400, { error: "collection-kind-source-required" });
167
+ return;
168
+ }
169
+ const r = await kb.ingest({ collection, kind, source, title: body?.title });
170
+ if (r.error === "bad-kind") {
171
+ sendJson(res, 400, { ok: false, error: r.error });
172
+ return;
173
+ }
174
+ if (r.error === "collection-not-found") {
175
+ // 拼错的集合 key 是调用方的错,不是文档的错——不能悄悄建集合(见 index.js)。
176
+ sendJson(res, 404, { ok: false, error: r.error, collection });
177
+ return;
178
+ }
179
+ // 来源级失败(找不到文件 / 太大 / http-404)回 200 + ok:false:kb_docs 里已经
180
+ // 落了 status='error' 的行,面板要能显示「这份为什么没进来」,而 HTTP 4xx
181
+ // 会让面板把它当成「请求本身错了」丢掉那条记录。
182
+ sendJson(res, 200, {
183
+ ok: !r.error,
184
+ docId: r.docId,
185
+ chunks: r.chunks,
186
+ skipped: r.skipped,
187
+ ...(r.error ? { error: r.error } : {})
188
+ });
189
+ });
190
+ }
191
+ });
192
+
193
+ // --- 检索 ---------------------------------------------------------------
194
+ register({
195
+ kind: "exact",
196
+ path: "/api/dsh-mneme/kb/search",
197
+ policy: "token-write",
198
+ handler(req, res) {
199
+ if (!allowed(req, res)) return;
200
+ if (req.method !== "POST") {
201
+ sendJson(res, 405, { error: "method-not-allowed" });
202
+ return;
203
+ }
204
+ return readJson(req, res).then((body) => {
205
+ if (body === null) return;
206
+ const query = String(body?.query ?? "").trim();
207
+ if (query === "") {
208
+ sendJson(res, 400, { error: "query-required" });
209
+ return;
210
+ }
211
+ const r = kb.search({
212
+ query,
213
+ collection: body?.collection ?? null,
214
+ limit: body?.limit
215
+ });
216
+ sendJson(res, 200, { hits: r.hits, unreadable: r.unreadable ?? [] });
217
+ });
218
+ }
219
+ });
220
+ }
@@ -0,0 +1,148 @@
1
+ // dsh-mneme/src/kb/tools.js
2
+ // 知识库的模型侧出口:三个**只读**工具(kb_list_collections / kb_search /
3
+ // kb_read_doc)。写操作一个都不给模型——建集合、入库都是用户在面板上按的按钮
4
+ // (与 chorography「生成只走 CLI」同一条红线:知识库收的是用户自己的资料,模型
5
+ // 不该有权决定往里塞什么)。
6
+ //
7
+ // 返回 defineTool() 兼容的描述对象数组,由 src/tools.js 在 kbEnabled 时展开进
8
+ // 它自己的数组——本文件不 import defineTool,装配顺序与两个开关(kbEnabled 与
9
+ // kb 是否装配到位)由调用方掌握,一处判定不两处分叉。
10
+ const TEXT_OUTPUT = (text) => [{ type: "text", text }];
11
+
12
+ /**
13
+ * @param {object} deps
14
+ * @param {object|null} deps.kb createKb() 的返回值;null = 未装配,仍返回数组但
15
+ * execute 会明确抛错(调用方本不该在此情形注册它们,
16
+ * 真注册了也要给人话而不是 undefined 崩)
17
+ * @param {object} [deps.cfg]
18
+ */
19
+ export function kbToolDefinitions({ kb = null, cfg = {} } = {}) {
20
+ const limitCap = Number(cfg?.kbSearchHitCap) > 0 ? Number(cfg.kbSearchHitCap) : 20;
21
+
22
+ const requireKb = () => {
23
+ if (!kb) throw new Error("Knowledge base is not available in this session (kb-unavailable).");
24
+ return kb;
25
+ };
26
+
27
+ return [
28
+ {
29
+ name: "kb_list_collections",
30
+ description: "List knowledge-base collections with document and chunk counts. The knowledge base holds the user's own reference material (local files, web pages, chorography pages) and is separate from memory. Use kb_search to search inside them. Read-only.",
31
+ parameters: {},
32
+ output: {
33
+ schema: {
34
+ type: "object",
35
+ additionalProperties: false,
36
+ properties: {
37
+ collections: {
38
+ type: "array",
39
+ required: true,
40
+ items: {
41
+ type: "object",
42
+ additionalProperties: false,
43
+ properties: {
44
+ key: { type: "string", required: true },
45
+ title: { type: "string", required: true },
46
+ description: { type: "string" },
47
+ docs: { type: "integer", required: true },
48
+ chunks: { type: "integer", required: true },
49
+ updatedAt: { type: "string" }
50
+ }
51
+ }
52
+ }
53
+ }
54
+ },
55
+ render: (_args, value) => {
56
+ const list = value.collections ?? [];
57
+ if (list.length === 0) {
58
+ return TEXT_OUTPUT("0 collections. The user can add one from the mneme panel (Knowledge base → new collection).");
59
+ }
60
+ const lines = list.map((c) => `- ${c.key} — ${c.title}(${c.docs} docs, ${c.chunks} chunks)${c.description ? ` · ${c.description}` : ""}`);
61
+ return TEXT_OUTPUT(`${list.length} collections:\n${lines.join("\n")}`);
62
+ }
63
+ },
64
+ async execute() {
65
+ return { collections: requireKb().listCollections() };
66
+ }
67
+ },
68
+
69
+ {
70
+ name: "kb_search",
71
+ description: "Keyword (BM25) search over the knowledge base. Returns matching chunks with collection key, document relative key, section heading, line number and excerpt. Read-only — the knowledge base is not memory and nothing here is written back to it.",
72
+ parameters: {
73
+ query: { type: "string", description: "Search text (CJK and ASCII both work)" },
74
+ collection: { type: "string", description: "Restrict to one collection key; omit to search all collections" },
75
+ limit: { type: "integer", description: `Max hits (default 20, cap ${limitCap})` }
76
+ },
77
+ output: {
78
+ schema: {
79
+ type: "object",
80
+ additionalProperties: false,
81
+ properties: {
82
+ hits: { type: "array", required: true, items: { type: "object", additionalProperties: true } },
83
+ unreadable: { type: "array", items: { type: "string" }, description: "Documents that could not be read during the search" }
84
+ }
85
+ },
86
+ render: (_args, value) => {
87
+ const hits = value.hits ?? [];
88
+ if (hits.length === 0) return TEXT_OUTPUT("0 hits.");
89
+ const lines = hits.map(
90
+ (h) => `${h.collection}/${h.relKey} · ${h.heading || "(no section)"} · L${h.line}: ${h.excerpt}`
91
+ );
92
+ return TEXT_OUTPUT(`${hits.length} hits:\n${lines.join("\n")}`);
93
+ }
94
+ },
95
+ async execute(args) {
96
+ const instance = requireKb();
97
+ const query = String(args?.query ?? "").trim();
98
+ if (query === "") throw new Error("kb_search: query is required.");
99
+ const limit = Math.max(1, Math.min(limitCap, Number(args?.limit) || 20));
100
+ const r = instance.search({ query, collection: args?.collection ?? null, limit });
101
+ return { hits: r.hits, unreadable: r.unreadable ?? [] };
102
+ }
103
+ },
104
+
105
+ {
106
+ name: "kb_read_doc",
107
+ description: "Read one knowledge-base document in full by collection key and relative key (as returned by kb_search). Read-only.",
108
+ parameters: {
109
+ collection: { type: "string", description: "Collection key" },
110
+ relKey: { type: "string", description: "Document relative key, e.g. the absolute source path shown in kb_search hits" }
111
+ },
112
+ output: {
113
+ schema: {
114
+ type: "object",
115
+ additionalProperties: false,
116
+ properties: {
117
+ collection: { type: "string", required: true },
118
+ relKey: { type: "string", required: true },
119
+ title: { type: "string", required: true },
120
+ content: { type: "string", required: true }
121
+ }
122
+ },
123
+ render: (_args, value) => TEXT_OUTPUT(value.content)
124
+ },
125
+ async execute(args) {
126
+ const instance = requireKb();
127
+ const collection = String(args?.collection ?? "").trim();
128
+ const relKey = String(args?.relKey ?? "");
129
+ if (collection === "" || relKey === "") {
130
+ throw new Error("kb_read_doc: collection and relKey are both required.");
131
+ }
132
+ const doc = instance.readDoc({ collection, relKey });
133
+ if (!doc) {
134
+ // 明确说「查不到」而不是回空串:空串会被模型当成「文件是空的」继续瞎编。
135
+ throw new Error(
136
+ `Knowledge-base document ${JSON.stringify(relKey)} was not found in collection ${JSON.stringify(collection)}. Use kb_list_collections / kb_search to find the right key.`
137
+ );
138
+ }
139
+ if (doc.status === "error") {
140
+ throw new Error(
141
+ `Document ${JSON.stringify(relKey)} was not indexed successfully (last error: ${doc.error ?? "unknown"}). The user can re-ingest it from the mneme panel.`
142
+ );
143
+ }
144
+ return { collection, relKey, title: doc.title, content: doc.content };
145
+ }
146
+ }
147
+ ];
148
+ }