@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/LICENSE +661 -0
  3. package/README.en.md +110 -0
  4. package/README.md +102 -2
  5. package/dsh-memory-local/FORK-BASE.json +24 -0
  6. package/dsh-memory-local/README.md +113 -0
  7. package/dsh-memory-local/build.mjs +101 -0
  8. package/dsh-memory-local/cordis.patch.yml +20 -0
  9. package/dsh-memory-local/deploy.mjs +294 -0
  10. package/dsh-memory-local/entity-status.mjs +75 -0
  11. package/dsh-memory-local/lib/bridge.mjs +108 -0
  12. package/dsh-memory-local/lib/host-adapter.js +1171 -0
  13. package/dsh-memory-local/lib/index.d.ts +128 -0
  14. package/dsh-memory-local/lib/index.js +944 -0
  15. package/dsh-memory-local/lib/injector.mjs +400 -0
  16. package/dsh-memory-local/lib/memory-read.mjs +116 -0
  17. package/dsh-memory-local/lib/module-gate.mjs +68 -0
  18. package/dsh-memory-local/lib/pointer.mjs +152 -0
  19. package/dsh-memory-local/lib/prod-gate.mjs +121 -0
  20. package/dsh-memory-local/lib/recall-log.mjs +118 -0
  21. package/dsh-memory-local/lib/recall.mjs +192 -0
  22. package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
  23. package/dsh-memory-local/lib/user-message.mjs +32 -0
  24. package/dsh-memory-local/lib/vectors-read.mjs +110 -0
  25. package/dsh-memory-local/package.json +51 -0
  26. package/dsh-memory-local/restore.mjs +259 -0
  27. package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
  28. package/dsh-memory-local/src/bridge.mjs +108 -0
  29. package/dsh-memory-local/src/host-adapter.js +1171 -0
  30. package/dsh-memory-local/src/index.d.ts +128 -0
  31. package/dsh-memory-local/src/index.js +944 -0
  32. package/dsh-memory-local/src/injector.mjs +400 -0
  33. package/dsh-memory-local/src/memory-read.mjs +116 -0
  34. package/dsh-memory-local/src/module-gate.mjs +68 -0
  35. package/dsh-memory-local/src/pointer.mjs +152 -0
  36. package/dsh-memory-local/src/recall-log.mjs +118 -0
  37. package/dsh-memory-local/src/recall.mjs +192 -0
  38. package/dsh-memory-local/src/schema-migrate.mjs +365 -0
  39. package/dsh-memory-local/src/user-message.mjs +32 -0
  40. package/dsh-memory-local/src/vectors-read.mjs +110 -0
  41. package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
  42. package/dsh-memory-local/test/probe-entity.mjs +170 -0
  43. package/dsh-memory-local/test/regression.mjs +270 -0
  44. package/dsh-memory-local/test/s2b.test.mjs +568 -0
  45. package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
  46. package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
  47. package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
  48. package/dsh-memory-local/verify-after-restart.mjs +181 -0
  49. package/lib/deploy-guard.mjs +391 -0
  50. package/lib/guard-http.mjs +93 -0
  51. package/lib/pitfall-form-map.mjs +130 -0
  52. package/lib/pitfall-score.mjs +185 -0
  53. package/lib/session-file-name.mjs +160 -0
  54. package/lib/zstd-frames.mjs +78 -0
  55. package/memory-db-tools/README.md +68 -0
  56. package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
  57. package/memory-db-tools/audit-analyze.mjs +352 -0
  58. package/memory-db-tools/audit-dump-ids.mjs +245 -0
  59. package/memory-db-tools/audit-export.mjs +270 -0
  60. package/memory-db-tools/audit-hit-probe.mjs +227 -0
  61. package/memory-db-tools/audit-ledger-probe.mjs +236 -0
  62. package/memory-db-tools/audit-persona-overlap.mjs +277 -0
  63. package/memory-db-tools/audit-struct-probe.mjs +213 -0
  64. package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
  65. package/memory-db-tools/check-memory-db.mjs +8 -0
  66. package/memory-db-tools/dump-memory-db.mjs +12 -0
  67. package/memory-db-tools/lib/memory-scope.mjs +446 -0
  68. package/memory-db-tools/list-memory-summary.mjs +477 -0
  69. package/memory-db-tools/memory-identity-check.mjs +101 -0
  70. package/memory-db-tools/memory-schema-migrate.mjs +290 -0
  71. package/memory-db-tools/memory-scope-fixture.mjs +197 -0
  72. package/memory-db-tools/read-memories.mjs +26 -0
  73. package/memory-db-tools/read-memory-db.mjs +20 -0
  74. package/memory-db-tools/refine-memories.mjs +247 -0
  75. package/memory-db-tools/show-mem-times.mjs +22 -0
  76. package/memory-engine/README.md +70 -0
  77. package/memory-engine/bench/bench.mjs +221 -0
  78. package/memory-engine/cli/distill-rollback.mjs +86 -0
  79. package/memory-engine/cli/distill.mjs +156 -0
  80. package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
  81. package/memory-engine/cli/doc-index.mjs +48 -0
  82. package/memory-engine/cli/domain-tags.mjs +251 -0
  83. package/memory-engine/cli/pitfall-hook.mjs +91 -0
  84. package/memory-engine/cli/recall.mjs +132 -0
  85. package/memory-engine/cli/selfcheck.mjs +57 -0
  86. package/memory-engine/cli/serve-start.mjs +247 -0
  87. package/memory-engine/cli/sweep.mjs +58 -0
  88. package/memory-engine/cli/sync.mjs +69 -0
  89. package/memory-engine/lib/cli-util.mjs +197 -0
  90. package/memory-engine/lib/config.mjs +135 -0
  91. package/memory-engine/lib/dirty-ledger.mjs +174 -0
  92. package/memory-engine/lib/distill-rollback.mjs +178 -0
  93. package/memory-engine/lib/distill.mjs +898 -0
  94. package/memory-engine/lib/doc-index-store.mjs +325 -0
  95. package/memory-engine/lib/doc-index.mjs +529 -0
  96. package/memory-engine/lib/doc-scan.mjs +271 -0
  97. package/memory-engine/lib/doc-summarize.mjs +305 -0
  98. package/memory-engine/lib/domain-tags.mjs +429 -0
  99. package/memory-engine/lib/embedder.mjs +242 -0
  100. package/memory-engine/lib/hashes.mjs +12 -0
  101. package/memory-engine/lib/memory-source.mjs +127 -0
  102. package/memory-engine/lib/pitfall-source.mjs +170 -0
  103. package/memory-engine/lib/policy-source.mjs +371 -0
  104. package/memory-engine/lib/recall.mjs +331 -0
  105. package/memory-engine/lib/selfcheck.mjs +131 -0
  106. package/memory-engine/lib/sweep.mjs +159 -0
  107. package/memory-engine/lib/sync.mjs +180 -0
  108. package/memory-engine/lib/tick-hook.mjs +218 -0
  109. package/memory-engine/lib/vectors-store.mjs +292 -0
  110. package/memory-engine/package-lock.json +1086 -0
  111. package/memory-engine/package.json +22 -0
  112. package/memory-engine/serve.mjs +171 -0
  113. package/memory-engine/test/01-store.test.mjs +127 -0
  114. package/memory-engine/test/02-sync.test.mjs +196 -0
  115. package/memory-engine/test/03-sweep.test.mjs +145 -0
  116. package/memory-engine/test/04-recall.test.mjs +214 -0
  117. package/memory-engine/test/05-selfcheck.test.mjs +175 -0
  118. package/memory-engine/test/06-doc-index.test.mjs +928 -0
  119. package/memory-engine/test/07-domain-tags.test.mjs +298 -0
  120. package/memory-engine/test/08-distill.test.mjs +686 -0
  121. package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
  122. package/memory-engine/test/10-tick-hook.test.mjs +252 -0
  123. package/memory-engine/test/11-policy-source.test.mjs +224 -0
  124. package/memory-engine/test/_tickhook-worker.mjs +56 -0
  125. package/memory-engine/test/fixtures.mjs +205 -0
  126. package/memory-engine/test/harness.mjs +73 -0
  127. package/memory-engine/test/run.mjs +35 -0
  128. package/memory-engine/tools/bridge-smoke.mjs +80 -0
  129. package/memory-engine/tools/fetch-model.mjs +128 -0
  130. package/memory-engine/tools/smoke-recall.mjs +76 -0
  131. package/package.json +53 -4
  132. package/persona-memory-audit.mjs +217 -0
@@ -0,0 +1,127 @@
1
+ // 记忆引擎 v2 · 离线层 —— 记忆库只读数据源
2
+ //
3
+ // 铁律(方案 §3.1 / §7):对 memory.db 只读,绝不写。只读连接在 DELETE journal 模式下
4
+ // 不阻塞写者,也不被写者阻塞(A6 实测:未提交写不阻塞读)。
5
+ import fs from 'node:fs';
6
+ import { DatabaseSync } from 'node:sqlite';
7
+
8
+ export function openMemorySource(dbPath) {
9
+ if (!fs.existsSync(dbPath)) {
10
+ throw new Error(`记忆库不存在: ${dbPath}`);
11
+ }
12
+ const db = new DatabaseSync(dbPath, { readOnly: true });
13
+ db.exec('PRAGMA busy_timeout = 5000;');
14
+
15
+ const S = {
16
+ count: db.prepare('SELECT COUNT(*) AS n FROM memories'),
17
+ maxUpdated: db.prepare('SELECT MAX(updated_at) AS m FROM memories'),
18
+ idRange: db.prepare('SELECT MIN(id) AS lo, MAX(id) AS hi, COUNT(*) AS n FROM memories'),
19
+ pinned: db.prepare('SELECT id FROM memories WHERE pinned = 1 ORDER BY id'),
20
+ page: db.prepare(
21
+ `SELECT id, text, tags, pinned, updated_at FROM memories
22
+ WHERE updated_at >= ? AND id > ? ORDER BY id LIMIT ?`,
23
+ ),
24
+ segment: db.prepare(
25
+ `SELECT id, text, tags, pinned, updated_at FROM memories
26
+ WHERE id >= ? AND id <= ? ORDER BY id`,
27
+ ),
28
+ byId: db.prepare(
29
+ 'SELECT id, text, tags, pinned, created_at, updated_at FROM memories WHERE id = ?',
30
+ ),
31
+ allTexts: db.prepare('SELECT id, text FROM memories ORDER BY id'),
32
+ tagsOf: db.prepare('SELECT id, tags, updated_at FROM memories'),
33
+ };
34
+
35
+ return {
36
+ dbPath,
37
+ readOnly: true,
38
+
39
+ count() {
40
+ return Number(S.count.get().n);
41
+ },
42
+
43
+ maxUpdatedAt() {
44
+ const v = S.maxUpdated.get().m;
45
+ return v == null ? 0 : Number(v);
46
+ },
47
+
48
+ idRange() {
49
+ const r = S.idRange.get();
50
+ return { lo: r.lo == null ? null : Number(r.lo), hi: r.hi == null ? null : Number(r.hi), n: Number(r.n) };
51
+ },
52
+
53
+ /** 召回用:pinned 集合(方案 §3.3:召回排除 pinned,交给 dsh-memory 注入) */
54
+ pinnedIds() {
55
+ return S.pinned.all().map((r) => Number(r.id));
56
+ },
57
+
58
+ /** 增量页:updated_at >= sinceTs 且 id > afterId,按 id 升序 */
59
+ incrementalPage(sinceTs, afterId, limit) {
60
+ return S.page.all(Number(sinceTs), Number(afterId), Number(limit)).map(mapRow);
61
+ },
62
+
63
+ /** id 段(兜底 hash 轮转用) */
64
+ segment(lo, hi) {
65
+ return S.segment.all(Number(lo), Number(hi)).map(mapRow);
66
+ },
67
+
68
+ getById(id) {
69
+ const r = S.byId.get(Number(id));
70
+ return r ? { ...r, id: Number(r.id) } : null;
71
+ },
72
+
73
+ /** 全表 id + 正文(兜底校验用;570 条量级纯内存可承受) */
74
+ allTexts() {
75
+ return S.allTexts.all().map((r) => ({ id: Number(r.id), text: r.text }));
76
+ },
77
+
78
+ allTags() {
79
+ return S.tagsOf.all().map((r) => ({ id: Number(r.id), tags: r.tags }));
80
+ },
81
+
82
+ /**
83
+ * FTS5 补位检索(方案 §3.4 A4:中文连续串几近不可用,只补标签/英文/数字)。
84
+ * @param {string} matchExpr 已构造好的 FTS5 MATCH 表达式
85
+ */
86
+ ftsSearch(matchExpr, limit) {
87
+ try {
88
+ return db
89
+ .prepare(
90
+ `SELECT m.id AS id, m.pinned AS pinned, bm25(memories_fts) AS score
91
+ FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid
92
+ WHERE memories_fts MATCH ? AND m.pinned = 0
93
+ ORDER BY score LIMIT ?`,
94
+ )
95
+ .all(String(matchExpr), Number(limit))
96
+ .map((r) => ({ id: Number(r.id), score: Number(r.score) }));
97
+ } catch (e) {
98
+ // 表达式非法/表缺失 → 补位路径空(方案 §3.3:FTS 缺失时不参与,不做零分占位)
99
+ return { error: String(e && e.message ? e.message : e) };
100
+ }
101
+ },
102
+
103
+ close() {
104
+ // 幂等:重复 close 不得抛错(调用方常在 finally 里再关一次)
105
+ if (this._closed) return;
106
+ this._closed = true;
107
+ try {
108
+ db.close();
109
+ } catch {
110
+ /* ignore */
111
+ }
112
+ },
113
+ get closed() {
114
+ return Boolean(this._closed);
115
+ },
116
+ };
117
+ }
118
+
119
+ function mapRow(r) {
120
+ return {
121
+ id: Number(r.id),
122
+ text: r.text,
123
+ tags: r.tags,
124
+ pinned: Number(r.pinned),
125
+ updated_at: Number(r.updated_at),
126
+ };
127
+ }
@@ -0,0 +1,170 @@
1
+ // 记忆引擎 v2 · 离线层 —— **坑库只读数据源**(方案 §3 A 点 · 第二向量来源 source='pitfall')
2
+ //
3
+ // 为什么需要它:开工简报原来无差别取 `topN(pitfalls,3)`(与本次任务语义无关,229 条里永远只看最高 3 条)。
4
+ // A 点要把「坑库」变成向量库里的第二个来源,让开工简报按任务语义召回相关坑。
5
+ //
6
+ // 铁律(方案 §A.2 / §A.4):
7
+ // 1. **只读** `butler-pitfalls.json`(A 点不写它);`readOnly: true` 是自证字段。
8
+ // 2. 接口必须与 `memory-source.mjs` 的 `openMemorySource` **同形状**(硬契约)——
9
+ // 下游三个消费面按方法名硬调用:`sync.mjs:81 incrementalPage` / `sweep.mjs:36 idRange`+`:69 segment`
10
+ // / `recall.mjs:120 pinnedIds`+`:141 ftsSearch`。少一个方法 = 第一次跑就 `is not a function`。
11
+ // ⚠️ 方案旧文写的 `{ list(), count(), maxSeen() }` 是**错的**(`maxSeen` 在对照物里不存在,实际名 `maxUpdatedAt`)。
12
+ // 3. `segment()` / `incrementalPage()` 返回行**必须含 `id / text / updated_at`**:
13
+ // `sweep.mjs:77` 读 `r.text`、`:110` 取 `r.updated_at`,缺 `updated_at` 不报错、只会把 `undefined` 写进向量行。
14
+ // 4. 轮转语义**不是 id 段**:坑无数字 id 段 ⇒ `idRange()` 给 `{lo:0,hi:n-1,n}`(稳定等价量,= 排序后下标),
15
+ // `segment(lo,hi)` 按 `lastSeen` 升序(平局按 `id` 字典序)切片;`incrementalPage` 用同一排序做**稳定游标**。
16
+ import fs from 'node:fs';
17
+
18
+ /** 坑条目的向量文本(方案 §A.3 定稿;**不含**建议文本 —— 候选池的 suggested/why 不进向量,避免建议变化导致向量漂移) */
19
+ export function pitfallText(row) {
20
+ const cat = row.category ? `[${row.category}] ` : '';
21
+ const seed = String(row.source || '') === 'seed' ? '[seed]' : '';
22
+ const tools = row.tools && typeof row.tools === 'object' ? Object.keys(row.tools).join(',') : '';
23
+ return `${seed}${cat}${row.normText ?? ''}${tools ? ` | tools=${tools}` : ''}`;
24
+ }
25
+
26
+ function sortRows(rows) {
27
+ return rows
28
+ .slice()
29
+ .sort((a, b) => a.updated_at - b.updated_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
30
+ }
31
+
32
+ /** 稳定游标键:`<lastSeen 补零>|<id>`(同 `sortRows` 的序即字符串序,平局按 id 字典序) */
33
+ function cursorKey(lastSeen, id) {
34
+ return `${String(lastSeen).padStart(16, '0')}|${id}`;
35
+ }
36
+
37
+ function mapRow(r) {
38
+ return {
39
+ id: r.id,
40
+ text: r.text,
41
+ tags: r.tags,
42
+ pinned: 0,
43
+ updated_at: r.updated_at,
44
+ };
45
+ }
46
+
47
+ /**
48
+ * 打开坑库(只读)。
49
+ * @param {string} jsonPath `butler-pitfalls.json`
50
+ */
51
+ export function openPitfallSource(jsonPath) {
52
+ if (!fs.existsSync(jsonPath)) {
53
+ throw new Error(`坑库不存在: ${jsonPath}`);
54
+ }
55
+ let raw;
56
+ try {
57
+ raw = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
58
+ } catch (e) {
59
+ throw new Error(`坑库解析失败: ${jsonPath}(${String(e && e.message ? e.message : e)})`);
60
+ }
61
+ const list = Array.isArray(raw) ? raw : Array.isArray(raw.pitfalls) ? raw.pitfalls : null;
62
+ if (!list) throw new Error(`坑库结构不认识(应含 pitfalls 数组): ${jsonPath}`);
63
+
64
+ const rows = sortRows(
65
+ list
66
+ .filter((x) => x && x.id != null)
67
+ .map((x) => ({
68
+ id: String(x.id),
69
+ text: pitfallText(x),
70
+ tags: x.tools && typeof x.tools === 'object' ? Object.keys(x.tools).join(',') : '',
71
+ updated_at: Number(x.lastSeen || 0),
72
+ })),
73
+ );
74
+ const byId = new Map(rows.map((r) => [r.id, r]));
75
+
76
+ let closed = false;
77
+ return {
78
+ dbPath: jsonPath,
79
+ readOnly: true,
80
+ /** 自证:来源名(下游日志/自检按它报"逐 source") */
81
+ sourceName: 'pitfall',
82
+ /** 原始条目数(= rows.length;观测用) */
83
+ n: rows.length,
84
+
85
+ count() {
86
+ return rows.length;
87
+ },
88
+
89
+ maxUpdatedAt() {
90
+ let m = 0;
91
+ for (const r of rows) if (r.updated_at > m) m = r.updated_at;
92
+ return m;
93
+ },
94
+
95
+ /** 坑无数字 id 段 ⇒ 给稳定等价量(排序后下标区间) */
96
+ idRange() {
97
+ if (!rows.length) return { lo: null, hi: null, n: 0 };
98
+ return { lo: 0, hi: rows.length - 1, n: rows.length };
99
+ },
100
+
101
+ /** 坑库无 pinned 概念 ⇒ 空集(召回侧据此不排除任何条目) */
102
+ pinnedIds() {
103
+ return [];
104
+ },
105
+
106
+ /**
107
+ * 增量页:`updated_at >= sinceTs`,按 (lastSeen, id) 升序,游标 = 上一页末条 `id`。
108
+ * 游标按「键比较」而非仅按下标:翻页期间即使集合变化也不会重复/跳过(键比较天然幂等)。
109
+ */
110
+ incrementalPage(sinceTs, afterId, limit) {
111
+ const since = Number(sinceTs) || 0;
112
+ const lim = Number(limit) || 0;
113
+ if (!rows.length || lim <= 0) return [];
114
+ const cur = afterId == null || afterId === '' ? null : String(afterId);
115
+ let floor = '';
116
+ if (cur != null) {
117
+ const hit = byId.get(cur);
118
+ if (hit) floor = cursorKey(hit.updated_at, hit.id);
119
+ else floor = cursorKey(0, cur); // 游标条已被删:退化成"按 id 起步"(仍单调、不重复)
120
+ }
121
+ const out = [];
122
+ for (const r of rows) {
123
+ if (r.updated_at < since) continue;
124
+ if (floor && cursorKey(r.updated_at, r.id) <= floor) continue;
125
+ out.push(mapRow(r));
126
+ if (out.length >= lim) break;
127
+ }
128
+ return out;
129
+ },
130
+
131
+ /** 段(下标区间,排序 = lastSeen 升序):返回行含 id/text/updated_at(sweep 硬契约) */
132
+ segment(lo, hi) {
133
+ const a = Number(lo);
134
+ const b = Number(hi);
135
+ if (!Number.isFinite(a) || !Number.isFinite(b)) return [];
136
+ return rows.slice(Math.max(0, a), Math.max(0, b) + 1).map(mapRow);
137
+ },
138
+
139
+ getById(id) {
140
+ const r = byId.get(String(id));
141
+ if (!r) return null;
142
+ return { id: r.id, text: r.text, tags: r.tags, pinned: 0, created_at: r.updated_at, updated_at: r.updated_at };
143
+ },
144
+
145
+ allTexts() {
146
+ return rows.map((r) => ({ id: r.id, text: r.text }));
147
+ },
148
+
149
+ allTags() {
150
+ return rows.map((r) => ({ id: r.id, tags: r.tags }));
151
+ },
152
+
153
+ /**
154
+ * ⚠️ 坑库**没有 FTS5** ⇒ 必须返回对象而不是抛错(`recall.mjs:141` 期望拿到对象、拿数组才走补位路)。
155
+ * 照 `memory-source.mjs:86-101` 的兜底形状。
156
+ */
157
+ ftsSearch() {
158
+ return { error: '坑库无 FTS5(pitfall source 无 FTS 补位路)' };
159
+ },
160
+
161
+ close() {
162
+ // 幂等:重复 close 不得抛错(调用方常在 finally 里再关一次)
163
+ if (closed) return;
164
+ closed = true;
165
+ },
166
+ get closed() {
167
+ return closed;
168
+ },
169
+ };
170
+ }
@@ -0,0 +1,371 @@
1
+ // 记忆引擎 v2 · 离线层 —— **policy 只读数据源**(方案 §4 B 点 · 落地判据向量化)
2
+ //
3
+ // 为什么需要它:候选池里「这条坑的防复发措施到底落地没有」原来靠**一次性人工核对** 59 条
4
+ // (`docs\坑库\P4候选分类与落地核对-20260920.md`)。B 点把「落地处」编成 `source='policy'`
5
+ // 的向量库(persona 铁律段 + 技能段 + 值得当判据的文档段),用候选的 `normText` 去查相似度
6
+ // ⇒ 自动产出「疑似已落地 / 确属新根 / 中性(需人看)」的**建议**。
7
+ //
8
+ // 铁的边界(方案 §4 B.1 写死,本文件照此实现):
9
+ // **向量判据只产出建议,不自动改候选池 status、不自动采纳、永不自动改 persona/技能。**
10
+ //
11
+ // 接口契约:与 `memory-source.mjs` / `pitfall-source.mjs` **同形状**(sync/sweep/recall 按方法名硬调用)。
12
+ // ⚠️ `segment()` / `incrementalPage()` 返回行**必须含 `id / text / updated_at`**
13
+ // (`sweep.mjs:77` 读 `.text`、`:110` 取 `.updated_at`;缺了不报错,只会把 undefined 写进向量)。
14
+ //
15
+ // 段化规则(方案 §4 B.2 第 1 行):按 markdown 标题 / 【…】 标记切段,**每段 ≤400 字**,超长段落再按句切。
16
+ import fs from 'node:fs';
17
+ import path from 'node:path';
18
+ import crypto from 'node:crypto';
19
+
20
+ /** 每段硬切上限(方案 §4 B.2:「每段 ≤400 字,超长段落再按句切」) */
21
+ export const POLICY_MAX_CHARS = 400;
22
+ /** 段文本哈希(与 `hashes.mjs#textHash` 同算法:sha256 hex ⇒ 增量同步的跳过判据可复用) */
23
+ export function policyTextHash(text) {
24
+ return crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
25
+ }
26
+ /** 段身份锚(16 位 hex,与坑库 id 同形):**只由「文件 + 段锚 + 分片号」决定**,与正文无关
27
+ * ⇒ 正文改了是**同一条 ref_id 换 text_hash**(增量重编码),不会产生孤儿行。 */
28
+ export function policyRefId(relPath, anchor, chunkIndex = 1) {
29
+ return crypto
30
+ .createHash('sha1')
31
+ .update(`${relPath}\u0000${anchor}\u0000${chunkIndex}`, 'utf8')
32
+ .digest('hex')
33
+ .slice(0, 16);
34
+ }
35
+
36
+ /** 路径统一为「正斜杠 + 小写盘符」的逻辑形式(跨机/跨平台可复现;policy 库不同步,本机自用) */
37
+ export function normalizeDocPath(p) {
38
+ const s = String(p).replace(/\\/g, '/');
39
+ return s.length > 1 && s[1] === ':' ? s[0].toLowerCase() + s.slice(1) : s;
40
+ }
41
+
42
+ /**
43
+ * 从 `agent.cordis.yml` 里抽出 persona 段(YAML 块标量 `prefix: |` / `|-` / `>` 的内容)。
44
+ * 抽不到 ⇒ 返回整份文件文本(**不抛错**:宁可编粗一点,也不要让整条 policy 链断在这里)。
45
+ */
46
+ export function extractPersonaText(yamlText) {
47
+ const lines = String(yamlText ?? '').split(/\r?\n/);
48
+ let start = -1;
49
+ let markerIndent = 0;
50
+ for (let i = 0; i < lines.length; i += 1) {
51
+ const m = lines[i].match(/^(\s*)(?:prefix|persona|text)\s*:\s*[|>][-+]?\s*$/);
52
+ if (m) {
53
+ start = i + 1;
54
+ markerIndent = m[1].length;
55
+ break;
56
+ }
57
+ }
58
+ if (start < 0) return String(yamlText ?? '');
59
+ const body = [];
60
+ for (let i = start; i < lines.length; i += 1) {
61
+ const line = lines[i];
62
+ if (!line.trim()) {
63
+ body.push('');
64
+ continue;
65
+ }
66
+ const indent = line.length - line.replace(/^[ \t]*/, '').length;
67
+ if (indent <= markerIndent) break; // 块结束
68
+ body.push(line);
69
+ }
70
+ // 去掉块标量的公共缩进(YAML 默认 = 首行缩进)
71
+ const firstNonEmpty = body.find((l) => l.trim());
72
+ const base = firstNonEmpty ? firstNonEmpty.length - firstNonEmpty.replace(/^[ \t]*/, '').length : 0;
73
+ return body.map((l) => (l.trim() ? l.slice(Math.min(base, l.length)) : '')).join('\n').trim();
74
+ }
75
+
76
+ /** 按「行首标记」切块:标记 = 正则;返回 [{ title, body }] */
77
+ function splitByLineMarker(text, markerRe) {
78
+ const lines = String(text ?? '').split(/\r?\n/);
79
+ const blocks = [];
80
+ let cur = null;
81
+ let preamble = [];
82
+ for (const line of lines) {
83
+ const m = line.match(markerRe);
84
+ if (m) {
85
+ if (cur) blocks.push(cur);
86
+ else if (preamble.join('').trim()) blocks.push({ title: '(前言)', body: preamble.join('\n').trim() });
87
+ preamble = [];
88
+ cur = { title: m[0].trim(), body: [line] };
89
+ } else if (cur) {
90
+ cur.body.push(line);
91
+ } else {
92
+ preamble.push(line);
93
+ }
94
+ }
95
+ if (cur) blocks.push(cur);
96
+ else if (preamble.join('').trim()) blocks.push({ title: '(前言)', body: preamble.join('\n').trim() });
97
+ return blocks.map((b) => ({ title: b.title, body: Array.isArray(b.body) ? b.body.join('\n').trim() : String(b.body).trim() }));
98
+ }
99
+
100
+ /** 超长段按句切(句末标点 + 换行),仍超长则硬切 */
101
+ function chunkText(text, maxChars) {
102
+ const s = String(text ?? '').trim();
103
+ if (!s) return [];
104
+ if (s.length <= maxChars) return [s];
105
+ const parts = s.split(/(?<=[。!?;\n])/u).filter((x) => x.trim());
106
+ const out = [];
107
+ let buf = '';
108
+ for (const p of parts.length ? parts : [s]) {
109
+ if ((buf + p).length > maxChars && buf) {
110
+ out.push(buf.trim());
111
+ buf = '';
112
+ }
113
+ if (p.length > maxChars) {
114
+ if (buf) {
115
+ out.push(buf.trim());
116
+ buf = '';
117
+ }
118
+ for (let i = 0; i < p.length; i += maxChars) out.push(p.slice(i, i + maxChars).trim());
119
+ } else {
120
+ buf += p;
121
+ }
122
+ }
123
+ if (buf.trim()) out.push(buf.trim());
124
+ return out.filter(Boolean);
125
+ }
126
+
127
+ /** 单份文本 → 段(按 kind 选切法)。返回 [{ anchor, section, chunkIndex, text }](顺序稳定:文件出现序) */
128
+ export function segmentText(text, { kind = 'markdown', maxChars = POLICY_MAX_CHARS } = {}) {
129
+ // persona 段:**行首 【…】 或 `- 【…】` 都算新段起点**。
130
+ // 为什么必须带 `- ` 前缀这一支:persona 的协作偏好是「列表式」的
131
+ // (每条写作 `- 【规矩名】…`)—— 只认行首 `【` 会把整节当一块、再硬切成几十个无意义碎片
132
+ // (实测粗切 = 12 个"小节"、124 段;细切后与方案 §B.5 的 123 个标记口径对齐)。
133
+ const markerRe = kind === 'persona' ? /^\s*-?\s*【[^】]*】.*$/u : /^#{2,3}\s+.*$/u;
134
+ const blocks = splitByLineMarker(text, markerRe);
135
+
136
+ // 无标记大块(例:另一个 preset 的 persona 完全没有 【】)先按**空行段**拆,
137
+ // 再对仍超长的段落做句级硬切 —— 避免把一整份 19 KB 文本切成一串半句碎片。
138
+ const pre = [];
139
+ for (const b of blocks) {
140
+ const title = b.title || '(无标题)';
141
+ const body = String(b.body ?? '');
142
+ if (body.length > maxChars * 4) {
143
+ const paras = body.split(/\n\s*\n/).map((s) => s.trim()).filter(Boolean);
144
+ if (paras.length > 1) {
145
+ paras.forEach((p, i) => pre.push({ base: title, sub: i + 1, body: p }));
146
+ continue;
147
+ }
148
+ }
149
+ pre.push({ base: title, sub: 0, body });
150
+ }
151
+
152
+ const seen = new Map();
153
+ const out = [];
154
+ for (const b of pre) {
155
+ const k = (seen.get(b.base) ?? 0) + 1;
156
+ seen.set(b.base, k);
157
+ const anchor = b.sub ? `${b.base}#${k}.${b.sub}` : `${b.base}#${k}`;
158
+ const chunks = chunkText(b.body, maxChars);
159
+ chunks.forEach((c, i) => {
160
+ // 段展示名:`<标题>` / `<标题> (part k)`;段内换行折成 ' · ',让 `docPath#section` 可读
161
+ const head = b.body.trim().split('\n')[0].slice(0, 40).replace(/\s+/g, ' ').trim();
162
+ const label = b.base.startsWith('(') ? `${b.base} ${head}` : b.base;
163
+ out.push({
164
+ anchor,
165
+ section: chunks.length > 1 ? `${label} (part ${i + 1})` : label,
166
+ chunkIndex: i + 1,
167
+ text: c,
168
+ });
169
+ });
170
+ }
171
+ return out;
172
+ }
173
+
174
+ /** 递归收集文件(跳过 node_modules / 隐藏目录;只取指定后缀) */
175
+ function walkFiles(dir, { ext = null, out = [], maxFiles = 2000 } = {}) {
176
+ let entries = [];
177
+ try {
178
+ entries = fs.readdirSync(dir, { withFileTypes: true });
179
+ } catch {
180
+ return out;
181
+ }
182
+ for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
183
+ if (out.length >= maxFiles) break;
184
+ if (e.name.startsWith('.') || e.name === 'node_modules') continue;
185
+ const full = path.join(dir, e.name);
186
+ if (e.isDirectory()) walkFiles(full, { ext, out, maxFiles });
187
+ else if (!ext || e.name.toLowerCase().endsWith(ext)) out.push(full);
188
+ }
189
+ return out;
190
+ }
191
+
192
+ /**
193
+ * 扫描 policy 语料 ⇒ 段清单(只读,零写入)。
194
+ *
195
+ * 语料三份(方案 §4 B.3-1,**按判据强度排序**;MVP 只做前两份,第三份默认空):
196
+ * ① persona 铁律段:`.agent-presets\**\agent.cordis.yml` 的 persona 文本 → 按 `【…】` 切
197
+ * ② 技能段:`skills\*\SKILL.md` → 按 `##` 小节切
198
+ * ③ 文档段(可选/后置):`docsRoots` 里被点名的少数几篇 —— **不做全量 docs**(那是 doc-index 的活)
199
+ *
200
+ * @param {object} p
201
+ * @param {string[]} [p.personaPaths] 显式 persona 文件(缺省 = 扫 personaDir 下的 agent.cordis.yml)
202
+ * @param {string} [p.personaDir]
203
+ * @param {string} [p.skillsRoot]
204
+ * @param {string[]} [p.docsPaths]
205
+ * @param {number} [p.maxChars]
206
+ * @returns {Array<{refId:string, docPath:string, relPath:string, section:string, text:string, mtime:number, textHash:string}>}
207
+ */
208
+ export function scanPolicySources({
209
+ personaPaths = null,
210
+ personaDir = null,
211
+ skillsRoot = null,
212
+ docsPaths = null,
213
+ maxChars = POLICY_MAX_CHARS,
214
+ maxFiles = 2000,
215
+ } = {}) {
216
+ const files = [];
217
+ if (Array.isArray(personaPaths) && personaPaths.length) {
218
+ for (const p of personaPaths) files.push({ file: path.resolve(String(p)), kind: 'persona' });
219
+ } else if (personaDir && fs.existsSync(personaDir)) {
220
+ for (const f of walkFiles(personaDir, { ext: '.yml', maxFiles })) {
221
+ if (path.basename(f).toLowerCase() === 'agent.cordis.yml') files.push({ file: f, kind: 'persona' });
222
+ }
223
+ }
224
+ if (skillsRoot && fs.existsSync(skillsRoot)) {
225
+ for (const f of walkFiles(skillsRoot, { ext: null, maxFiles })) {
226
+ if (path.basename(f).toLowerCase() === 'skill.md') files.push({ file: f, kind: 'markdown' });
227
+ }
228
+ }
229
+ for (const d of Array.isArray(docsPaths) ? docsPaths : []) {
230
+ if (!d) continue;
231
+ const resolved = path.resolve(String(d));
232
+ if (!fs.existsSync(resolved)) continue;
233
+ if (fs.statSync(resolved).isDirectory()) {
234
+ for (const f of walkFiles(resolved, { ext: '.md', maxFiles })) files.push({ file: f, kind: 'markdown' });
235
+ } else {
236
+ files.push({ file: resolved, kind: 'markdown' });
237
+ }
238
+ }
239
+
240
+ const segments = [];
241
+ for (const { file, kind } of files) {
242
+ let raw;
243
+ let mtime = 0;
244
+ try {
245
+ raw = fs.readFileSync(file, 'utf8');
246
+ mtime = Math.round(fs.statSync(file).mtimeMs);
247
+ } catch {
248
+ continue; // 读不到就跳过(不阻断整条链)
249
+ }
250
+ const relPath = normalizeDocPath(file);
251
+ const text = kind === 'persona' ? extractPersonaText(raw) : raw;
252
+ if (!String(text).trim()) continue;
253
+ for (const seg of segmentText(text, { kind, maxChars })) {
254
+ segments.push({
255
+ refId: policyRefId(relPath, seg.anchor, seg.chunkIndex),
256
+ docPath: normalizeDocPath(file),
257
+ relPath,
258
+ section: seg.section,
259
+ text: seg.text,
260
+ mtime,
261
+ textHash: policyTextHash(seg.text),
262
+ });
263
+ }
264
+ }
265
+ // 稳定排序(段序 = 文件名字典序 × 文件内出现序;refId 单调游标依赖它)
266
+ segments.sort((a, b) => (a.docPath < b.docPath ? -1 : a.docPath > b.docPath ? 1 : 0)
267
+ || (a.mtime - b.mtime)
268
+ || (a.refId < b.refId ? -1 : a.refId > b.refId ? 1 : 0));
269
+ return segments;
270
+ }
271
+
272
+ function cursorKey(updatedAt, id) {
273
+ return `${String(updatedAt).padStart(16, '0')}|${id}`;
274
+ }
275
+
276
+ /**
277
+ * 把段清单包成**与 memory-source 同形状**的只读数据源(供 `sync --source policy` / `sweep` / `recall` 直接消费)。
278
+ * @param {{segments:Array, dbPath?:string}} p
279
+ */
280
+ export function openPolicySource({ segments = [], dbPath = null } = {}) {
281
+ const rows = segments
282
+ .map((s) => ({
283
+ id: String(s.refId),
284
+ text: s.text,
285
+ tags: s.section || '',
286
+ pinned: 0,
287
+ updated_at: Number(s.mtime) || 0,
288
+ docPath: s.docPath,
289
+ section: s.section,
290
+ textHash: s.textHash,
291
+ }))
292
+ .sort((a, b) => a.updated_at - b.updated_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
293
+ const byId = new Map(rows.map((r) => [r.id, r]));
294
+ let closed = false;
295
+
296
+ return {
297
+ dbPath: dbPath || '(scanned:policy)',
298
+ readOnly: true,
299
+ sourceName: 'policy',
300
+ n: rows.length,
301
+ /** 原始段对象(triage 侧要 docPath/section 做「出处可追溯」) */
302
+ segments,
303
+
304
+ count() {
305
+ return rows.length;
306
+ },
307
+ maxUpdatedAt() {
308
+ let m = 0;
309
+ for (const r of rows) if (r.updated_at > m) m = r.updated_at;
310
+ return m;
311
+ },
312
+ idRange() {
313
+ if (!rows.length) return { lo: null, hi: null, n: 0 };
314
+ return { lo: 0, hi: rows.length - 1, n: rows.length };
315
+ },
316
+ pinnedIds() {
317
+ return [];
318
+ },
319
+ incrementalPage(sinceTs, afterId, limit) {
320
+ const since = Number(sinceTs) || 0;
321
+ const lim = Number(limit) || 0;
322
+ if (!rows.length || lim <= 0) return [];
323
+ const cur = afterId == null || afterId === '' ? null : String(afterId);
324
+ let floor = '';
325
+ if (cur != null) {
326
+ const hit = byId.get(cur);
327
+ floor = hit ? cursorKey(hit.updated_at, hit.id) : cursorKey(0, cur);
328
+ }
329
+ const out = [];
330
+ for (const r of rows) {
331
+ if (r.updated_at < since) continue;
332
+ if (floor && cursorKey(r.updated_at, r.id) <= floor) continue;
333
+ out.push({ ...r });
334
+ if (out.length >= lim) break;
335
+ }
336
+ return out;
337
+ },
338
+ segment(lo, hi) {
339
+ const a = Number(lo);
340
+ const b = Number(hi);
341
+ if (!Number.isFinite(a) || !Number.isFinite(b)) return [];
342
+ return rows.slice(Math.max(0, a), Math.max(0, b) + 1).map((r) => ({ ...r }));
343
+ },
344
+ getById(id) {
345
+ const r = byId.get(String(id));
346
+ return r ? { ...r } : null;
347
+ },
348
+ allTexts() {
349
+ return rows.map((r) => ({ id: r.id, text: r.text }));
350
+ },
351
+ allTags() {
352
+ return rows.map((r) => ({ id: r.id, tags: r.tags }));
353
+ },
354
+ /** policy 段无 FTS5 ⇒ 必须返回对象(`recall.mjs:167` 期望对象;返回数组才会走补位路) */
355
+ ftsSearch() {
356
+ return { error: 'policy 段源无 FTS5(policy source 无 FTS 补位路)' };
357
+ },
358
+ close() {
359
+ if (closed) return;
360
+ closed = true;
361
+ },
362
+ get closed() {
363
+ return closed;
364
+ },
365
+ };
366
+ }
367
+
368
+ /** 便捷:扫描 + 打包一步到位 */
369
+ export function openPolicySourceFromDisk(opts = {}) {
370
+ return openPolicySource({ segments: scanPolicySources(opts), dbPath: opts.dbPath || null });
371
+ }