@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +92 -0
  2. package/README.en.md +225 -2
  3. package/README.md +180 -2
  4. package/lib/deploy-guard.mjs +463 -275
  5. package/lib/guard-http.mjs +59 -62
  6. package/lib/pitfall-form-map.mjs +94 -84
  7. package/lib/pitfall-score.mjs +262 -185
  8. package/lib/session-file-name.mjs +116 -100
  9. package/lib/zstd-frames.mjs +88 -67
  10. package/memory-db-tools/README.md +29 -1
  11. package/memory-db-tools/audit-analyze.mjs +1 -1
  12. package/memory-db-tools/audit-export.mjs +60 -16
  13. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  14. package/memory-db-tools/lib/known-devices.mjs +137 -0
  15. package/memory-engine/README.md +58 -4
  16. package/memory-engine/bench/bench.mjs +1 -1
  17. package/memory-engine/cli/band-sentinel.mjs +164 -0
  18. package/memory-engine/cli/distill-rollback.mjs +112 -44
  19. package/memory-engine/cli/distill-sessions.mjs +237 -0
  20. package/memory-engine/cli/distill.mjs +196 -95
  21. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  22. package/memory-engine/cli/doc-index.mjs +47 -38
  23. package/memory-engine/cli/domain-tags.mjs +246 -162
  24. package/memory-engine/cli/layer-migrate.mjs +176 -0
  25. package/memory-engine/cli/layer-move.mjs +189 -0
  26. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  27. package/memory-engine/cli/recall.mjs +243 -110
  28. package/memory-engine/cli/selfcheck.mjs +93 -44
  29. package/memory-engine/cli/serve-start.mjs +221 -140
  30. package/memory-engine/cli/sweep.mjs +83 -44
  31. package/memory-engine/cli/sync.mjs +93 -57
  32. package/memory-engine/config/audit-devices.json +6 -0
  33. package/memory-engine/config/bands.json +7 -0
  34. package/memory-engine/config/layers.json +7 -0
  35. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  36. package/memory-engine/dict/lexicon-config.json +11 -0
  37. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  38. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  39. package/memory-engine/lib/band-sentinel.mjs +417 -0
  40. package/memory-engine/lib/cli-strict.mjs +128 -0
  41. package/memory-engine/lib/cli-util.mjs +147 -125
  42. package/memory-engine/lib/config.mjs +83 -96
  43. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  44. package/memory-engine/lib/distill-rollback.mjs +173 -133
  45. package/memory-engine/lib/distill.mjs +878 -585
  46. package/memory-engine/lib/doc-index-store.mjs +208 -235
  47. package/memory-engine/lib/doc-index.mjs +351 -350
  48. package/memory-engine/lib/doc-scan.mjs +178 -156
  49. package/memory-engine/lib/doc-summarize.mjs +220 -205
  50. package/memory-engine/lib/domain-tags.mjs +358 -277
  51. package/memory-engine/lib/embedder.mjs +175 -194
  52. package/memory-engine/lib/hashes.mjs +22 -7
  53. package/memory-engine/lib/layer-migrate.mjs +330 -0
  54. package/memory-engine/lib/layer-move.mjs +307 -0
  55. package/memory-engine/lib/layers.mjs +191 -0
  56. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  57. package/memory-engine/lib/memory-source.mjs +78 -80
  58. package/memory-engine/lib/pitfall-source.mjs +115 -114
  59. package/memory-engine/lib/policy-source.mjs +279 -261
  60. package/memory-engine/lib/recall-log.mjs +285 -0
  61. package/memory-engine/lib/recall.mjs +565 -240
  62. package/memory-engine/lib/selfcheck.mjs +142 -106
  63. package/memory-engine/lib/session-distill.mjs +603 -0
  64. package/memory-engine/lib/session-source.mjs +421 -0
  65. package/memory-engine/lib/sweep.mjs +107 -130
  66. package/memory-engine/lib/sync.mjs +110 -140
  67. package/memory-engine/lib/tick-hook.mjs +123 -148
  68. package/memory-engine/lib/vectors-store.mjs +213 -188
  69. package/memory-engine/test/12-layering.test.mjs +539 -0
  70. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  71. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  72. package/memory-engine/tools/fetch-model.mjs +167 -87
  73. package/memory-engine/tools/smoke-recall.mjs +94 -38
  74. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  75. package/package.json +1 -1
@@ -1,241 +1,253 @@
1
- // 记忆引擎 v2 · 离线层 —— **policy 只读数据源**(方案 §4 B 点 · 落地判据向量化)
1
+ // 离线层 —— policy 只读段源(把「判据落地处」编成可检索的段)
2
2
  //
3
- // 为什么需要它:候选池里「这条坑的防复发措施到底落地没有」原来靠**一次性人工核对** 59 条
4
- // (`docs\坑库\P4候选分类与落地核对-20260920.md`)。B 点把「落地处」编成 `source='policy'`
5
- // 的向量库(persona 铁律段 + 技能段 + 值得当判据的文档段),用候选的 `normText` 去查相似度
6
- // ⇒ 自动产出「疑似已落地 / 确属新根 / 中性(需人看)」的**建议**。
3
+ // 用途:候选池里那条坑「防复发措施到底落地没」原本只能人工翻文档。这里把落地处
4
+ // (persona 铁律段 + 技能段 + 少数指定文档段)切成段,候选正文拿去比相似度,
5
+ // 产出「疑似已落地 / 确实是新根 / 中性待看」的**建议**。
7
6
  //
8
- // 铁的边界(方案 §4 B.1 写死,本文件照此实现):
9
- // **向量判据只产出建议,不自动改候选池 status、不自动采纳、永不自动改 persona/技能。**
7
+ // 边界(写死,别越):只出建议,不改候选状态、不自动采纳、永不改 persona 或技能文件。
10
8
  //
11
- // 接口契约:与 `memory-source.mjs` / `pitfall-source.mjs` **同形状**(sync/sweep/recall 按方法名硬调用)。
12
- // ⚠️ `segment()` / `incrementalPage()` 返回行**必须含 `id / text / updated_at`**
13
- // (`sweep.mjs:77` 读 `.text`、`:110` 取 `.updated_at`;缺了不报错,只会把 undefined 写进向量)。
9
+ // 接口形状与记忆源 / 坑源保持一致(同步、清扫、召回都是按方法名直接调的):
10
+ // segment() / incrementalPage() 返回的每一行都必须带 id / text / updated_at 三个字段,
11
+ // 缺了不会报错,只会把 undefined 悄悄写进向量库。
14
12
  //
15
- // 段化规则(方案 §4 B.2 第 1 行):按 markdown 标题 / 【…】 标记切段,**每段 ≤400 字**,超长段落再按句切。
13
+ // 段化口径:按 markdown 标题或 【…】 标记切段,每段 ≤400 字,超长段落再按句子切。
16
14
  import fs from 'node:fs';
17
15
  import path from 'node:path';
18
16
  import crypto from 'node:crypto';
19
17
 
20
- /** 每段硬切上限(方案 §4 B.2:「每段 ≤400 字,超长段落再按句切」) */
18
+ // 每段硬上限(超长先按句切,单句还超长就硬截)
21
19
  export const POLICY_MAX_CHARS = 400;
22
- /** 段文本哈希(与 `hashes.mjs#textHash` 同算法:sha256 hex ⇒ 增量同步的跳过判据可复用) */
23
- export function policyTextHash(text) {
24
- return crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
25
- }
26
- /** 段身份锚(16 位 hex,与坑库 id 同形):**只由「文件 + 段锚 + 分片号」决定**,与正文无关
27
- * ⇒ 正文改了是**同一条 ref_id 换 text_hash**(增量重编码),不会产生孤儿行。 */
28
- export function policyRefId(relPath, anchor, chunkIndex = 1) {
29
- return crypto
30
- .createHash('sha1')
31
- .update(`${relPath}\u0000${anchor}\u0000${chunkIndex}`, 'utf8')
32
- .digest('hex')
33
- .slice(0, 16);
34
- }
35
-
36
- /** 路径统一为「正斜杠 + 小写盘符」的逻辑形式(跨机/跨平台可复现;policy 库不同步,本机自用) */
37
- export function normalizeDocPath(p) {
38
- const s = String(p).replace(/\\/g, '/');
39
- return s.length > 1 && s[1] === ':' ? s[0].toLowerCase() + s.slice(1) : s;
40
- }
20
+
21
+ // 段身份锚里的分隔符:用 NUL,保证不会跟正文内容撞
22
+ const ANCHOR_SEP = '\u0000';
23
+
24
+ // 段文本哈希(sha256 hex,与增量同步的跳过判据同一套算法)
25
+ export const policyTextHash = (text) =>
26
+ crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
27
+
28
+ // 段身份锚(16 位 hex,与坑库 id 同形):只由「文件 + 段锚 + 分片号」决定,与正文无关。
29
+ // ⇒ 正文改了是同一条 ref_id 换 text_hash(增量重编码),不会留孤儿行。
30
+ export const policyRefId = (relPath, anchor, chunkIndex = 1) =>
31
+ crypto.createHash('sha1').update([relPath, anchor, chunkIndex].join(ANCHOR_SEP), 'utf8').digest('hex').slice(0, 16);
32
+
33
+ // 路径归一:反斜杠转正斜杠 + 盘符小写(跨平台可复现;policy 库本机自用、不参与同步)
34
+ export const normalizeDocPath = (p) => {
35
+ const slashed = String(p).replace(/\\/g, '/');
36
+ if (slashed.length < 2 || slashed[1] !== ':') return slashed;
37
+ return slashed[0].toLowerCase() + slashed.slice(1);
38
+ };
39
+
40
+ // ── persona 块标量抽取 ──────────────────────────────────────────────────────
41
+
42
+ // 块起始行:`prefix: |` / `persona: >-` 这类写法都认
43
+ const BLOCK_MARK = /^(\s*)(?:prefix|persona|text)\s*:\s*[|>][-+]?\s*$/;
44
+
45
+ const indentWidth = (line) => line.length - line.replace(/^[ \t]*/, '').length;
41
46
 
42
47
  /**
43
- * 从 `agent.cordis.yml` 里抽出 persona 段(YAML 块标量 `prefix: |` / `|-` / `>` 的内容)。
44
- * 抽不到 ⇒ 返回整份文件文本(**不抛错**:宁可编粗一点,也不要让整条 policy 链断在这里)。
48
+ * 从 agent.cordis.yml 里抽出 persona 段(YAML 块标量的内容)。
49
+ * 抽不到就返回整份文本 —— 宁可编粗一点,也不让整条链断在这。
45
50
  */
46
- export function extractPersonaText(yamlText) {
47
- const lines = String(yamlText ?? '').split(/\r?\n/);
48
- let start = -1;
49
- let markerIndent = 0;
50
- for (let i = 0; i < lines.length; i += 1) {
51
- const m = lines[i].match(/^(\s*)(?:prefix|persona|text)\s*:\s*[|>][-+]?\s*$/);
52
- if (m) {
53
- start = i + 1;
54
- markerIndent = m[1].length;
55
- break;
56
- }
57
- }
58
- if (start < 0) return String(yamlText ?? '');
59
- const body = [];
60
- for (let i = start; i < lines.length; i += 1) {
51
+ export const extractPersonaText = (yamlText) => {
52
+ const source = String(yamlText ?? '');
53
+ const lines = source.split(/\r?\n/);
54
+ const at = lines.findIndex((line) => BLOCK_MARK.test(line));
55
+ if (at < 0) return source;
56
+
57
+ const markerIndent = indentWidth(lines[at]);
58
+ const kept = [];
59
+ for (let i = at + 1; i < lines.length; i += 1) {
61
60
  const line = lines[i];
62
61
  if (!line.trim()) {
63
- body.push('');
62
+ kept.push('');
64
63
  continue;
65
64
  }
66
- const indent = line.length - line.replace(/^[ \t]*/, '').length;
67
- if (indent <= markerIndent) break; // 块结束
68
- body.push(line);
65
+ if (indentWidth(line) <= markerIndent) break; // 缩进退回标记行 ⇒ 块结束
66
+ kept.push(line);
69
67
  }
70
- // 去掉块标量的公共缩进(YAML 默认 = 首行缩进)
71
- const firstNonEmpty = body.find((l) => l.trim());
72
- const base = firstNonEmpty ? firstNonEmpty.length - firstNonEmpty.replace(/^[ \t]*/, '').length : 0;
73
- return body.map((l) => (l.trim() ? l.slice(Math.min(base, l.length)) : '')).join('\n').trim();
74
- }
75
-
76
- /** 按「行首标记」切块:标记 = 正则;返回 [{ title, body }] */
77
- function splitByLineMarker(text, markerRe) {
68
+
69
+ // 去掉块标量的公共缩进(YAML 默认取首行缩进)
70
+ const firstKept = kept.find((line) => line.trim());
71
+ const base = firstKept ? indentWidth(firstKept) : 0;
72
+ return kept.map((line) => (line.trim() ? line.slice(Math.min(base, line.length)) : '')).join('\n').trim();
73
+ };
74
+
75
+ // ── 切块 ────────────────────────────────────────────────────────────────────
76
+
77
+ // 按「行首标记」切块,返回 [{title, body}];第一个标记之前的非空内容归到 (前言)
78
+ const splitBlocks = (text, markRe) => {
78
79
  const lines = String(text ?? '').split(/\r?\n/);
79
- const blocks = [];
80
- let cur = null;
81
- let preamble = [];
80
+ const groups = [];
81
+ let current = null;
82
+ let loose = [];
83
+
82
84
  for (const line of lines) {
83
- const m = line.match(markerRe);
84
- if (m) {
85
- if (cur) blocks.push(cur);
86
- else if (preamble.join('').trim()) blocks.push({ title: '(前言)', body: preamble.join('\n').trim() });
87
- preamble = [];
88
- cur = { title: m[0].trim(), body: [line] };
89
- } else if (cur) {
90
- cur.body.push(line);
91
- } else {
92
- preamble.push(line);
85
+ if (markRe.test(line)) {
86
+ if (current) groups.push(current);
87
+ else if (loose.join('').trim()) groups.push({ title: '(前言)', body: loose });
88
+ loose = [];
89
+ current = { title: line.trim(), body: [line] };
90
+ continue;
93
91
  }
92
+ (current ? current.body : loose).push(line);
94
93
  }
95
- if (cur) blocks.push(cur);
96
- else if (preamble.join('').trim()) blocks.push({ title: '(前言)', body: preamble.join('\n').trim() });
97
- return blocks.map((b) => ({ title: b.title, body: Array.isArray(b.body) ? b.body.join('\n').trim() : String(b.body).trim() }));
98
- }
99
-
100
- /** 超长段按句切(句末标点 + 换行),仍超长则硬切 */
101
- function chunkText(text, maxChars) {
102
- const s = String(text ?? '').trim();
103
- if (!s) return [];
104
- if (s.length <= maxChars) return [s];
105
- const parts = s.split(/(?<=[。!?;\n])/u).filter((x) => x.trim());
106
- const out = [];
107
- let buf = '';
108
- for (const p of parts.length ? parts : [s]) {
109
- if ((buf + p).length > maxChars && buf) {
110
- out.push(buf.trim());
111
- buf = '';
94
+ if (current) groups.push(current);
95
+ else if (loose.join('').trim()) groups.push({ title: '(前言)', body: loose });
96
+
97
+ return groups.map((group) => ({ title: group.title, body: group.body.join('\n').trim() }));
98
+ };
99
+
100
+ // 超长文本按句切(句末标点与换行都算断点);单句仍超长就按上限硬截
101
+ const sliceLong = (text, maxChars) => {
102
+ const whole = String(text ?? '').trim();
103
+ if (!whole) return [];
104
+ if (whole.length <= maxChars) return [whole];
105
+
106
+ const sentences = whole.split(/(?<=[。!?;\n])/u).filter((piece) => piece.trim());
107
+ const slices = [];
108
+ let buffer = '';
109
+
110
+ for (const sentence of sentences.length ? sentences : [whole]) {
111
+ if (buffer && (buffer + sentence).length > maxChars) {
112
+ slices.push(buffer.trim());
113
+ buffer = '';
112
114
  }
113
- if (p.length > maxChars) {
114
- if (buf) {
115
- out.push(buf.trim());
116
- buf = '';
117
- }
118
- for (let i = 0; i < p.length; i += maxChars) out.push(p.slice(i, i + maxChars).trim());
119
- } else {
120
- buf += p;
115
+ if (sentence.length > maxChars) {
116
+ for (let at = 0; at < sentence.length; at += maxChars) slices.push(sentence.slice(at, at + maxChars).trim());
117
+ continue;
121
118
  }
119
+ buffer += sentence;
122
120
  }
123
- if (buf.trim()) out.push(buf.trim());
124
- return out.filter(Boolean);
125
- }
126
-
127
- /** 单份文本 → 段(按 kind 选切法)。返回 [{ anchor, section, chunkIndex, text }](顺序稳定:文件出现序) */
128
- export function segmentText(text, { kind = 'markdown', maxChars = POLICY_MAX_CHARS } = {}) {
129
- // persona 段:**行首 【…】 或 `- 【…】` 都算新段起点**。
130
- // 为什么必须带 `- ` 前缀这一支:persona 的协作偏好是「列表式」的
131
- // (每条写作 `- 【规矩名】…`)—— 只认行首 `【` 会把整节当一块、再硬切成几十个无意义碎片
132
- // (实测粗切 = 12 个"小节"、124 段;细切后与方案 §B.5 的 123 个标记口径对齐)。
133
- const markerRe = kind === 'persona' ? /^\s*-?\s*【[^】]*】.*$/u : /^#{2,3}\s+.*$/u;
134
- const blocks = splitByLineMarker(text, markerRe);
135
-
136
- // 无标记大块(例:另一个 preset 的 persona 完全没有 【】)先按**空行段**拆,
137
- // 再对仍超长的段落做句级硬切 —— 避免把一整份 19 KB 文本切成一串半句碎片。
138
- const pre = [];
139
- for (const b of blocks) {
140
- const title = b.title || '(无标题)';
141
- const body = String(b.body ?? '');
121
+ if (buffer.trim()) slices.push(buffer.trim());
122
+ return slices.filter(Boolean);
123
+ };
124
+
125
+ // 块太大(超过上限 4 倍)且内部有空行分段 ⇒ 按空行先拆开,免得切出一串半句碎片
126
+ const splitOversized = (blocks, maxChars) => {
127
+ const pieces = [];
128
+ for (const block of blocks) {
129
+ const base = block.title || '(无标题)';
130
+ const body = String(block.body ?? '');
142
131
  if (body.length > maxChars * 4) {
143
132
  const paras = body.split(/\n\s*\n/).map((s) => s.trim()).filter(Boolean);
144
133
  if (paras.length > 1) {
145
- paras.forEach((p, i) => pre.push({ base: title, sub: i + 1, body: p }));
134
+ paras.forEach((para, i) => pieces.push({ base, sub: i + 1, body: para }));
146
135
  continue;
147
136
  }
148
137
  }
149
- pre.push({ base: title, sub: 0, body });
138
+ pieces.push({ base, sub: 0, body });
150
139
  }
140
+ return pieces;
141
+ };
142
+
143
+ // 段展示名:`<标题>`;前言块后面补首行摘要,段内换行折成空格,让 docPath#section 可读
144
+ const sectionLabel = (piece) => {
145
+ const firstLine = piece.body.trim().split('\n')[0];
146
+ const head = firstLine.slice(0, 40).replace(/\s+/g, ' ').trim();
147
+ return piece.base.startsWith('(') ? `${piece.base} ${head}` : piece.base;
148
+ };
151
149
 
152
- const seen = new Map();
153
- const out = [];
154
- for (const b of pre) {
155
- const k = (seen.get(b.base) ?? 0) + 1;
156
- seen.set(b.base, k);
157
- const anchor = b.sub ? `${b.base}#${k}.${b.sub}` : `${b.base}#${k}`;
158
- const chunks = chunkText(b.body, maxChars);
159
- chunks.forEach((c, i) => {
160
- // 段展示名:`<标题>` / `<标题> (part k)`;段内换行折成 ' · ',让 `docPath#section` 可读
161
- const head = b.body.trim().split('\n')[0].slice(0, 40).replace(/\s+/g, ' ').trim();
162
- const label = b.base.startsWith('(') ? `${b.base} ${head}` : b.base;
163
- out.push({
150
+ // persona 段:行首 【…】 与列表式 `- 【…】` 都算新段起点(协作偏好写成列表,只认行首 【 会切不动)
151
+ const PERSONA_MARK = /^\s*-?\s*【[^】]*】.*$/u;
152
+ const HEADING_MARK = /^#{2,3}\s+.*$/u;
153
+
154
+ /** 单份文本 → 段清单(顺序稳定:文件出现序;同名小节用 #N 区分) */
155
+ export const segmentText = (text, { kind = 'markdown', maxChars = POLICY_MAX_CHARS } = {}) => {
156
+ const markRe = kind === 'persona' ? PERSONA_MARK : HEADING_MARK;
157
+ const pieces = splitOversized(splitBlocks(text, markRe), maxChars);
158
+ const occurrences = new Map();
159
+ const segments = [];
160
+
161
+ for (const piece of pieces) {
162
+ const nth = (occurrences.get(piece.base) ?? 0) + 1;
163
+ occurrences.set(piece.base, nth);
164
+ const anchor = piece.sub ? `${piece.base}#${nth}.${piece.sub}` : `${piece.base}#${nth}`;
165
+ const slices = sliceLong(piece.body, maxChars);
166
+ const label = sectionLabel(piece);
167
+ slices.forEach((chunk, i) => {
168
+ segments.push({
164
169
  anchor,
165
- section: chunks.length > 1 ? `${label} (part ${i + 1})` : label,
170
+ section: slices.length > 1 ? `${label} (part ${i + 1})` : label,
166
171
  chunkIndex: i + 1,
167
- text: c,
172
+ text: chunk,
168
173
  });
169
174
  });
170
175
  }
171
- return out;
172
- }
176
+ return segments;
177
+ };
178
+
179
+ // ── 文件遍历与扫描 ──────────────────────────────────────────────────────────
173
180
 
174
- /** 递归收集文件(跳过 node_modules / 隐藏目录;只取指定后缀) */
175
- function walkFiles(dir, { ext = null, out = [], maxFiles = 2000 } = {}) {
176
- let entries = [];
181
+ // 递归收文件(跳过隐藏目录与 node_modules;ext 为空表示不过滤后缀)
182
+ const collectFiles = (dir, { ext = null, out = [], maxFiles = 2000 } = {}) => {
183
+ let entries;
177
184
  try {
178
185
  entries = fs.readdirSync(dir, { withFileTypes: true });
179
186
  } catch {
180
- return out;
187
+ return out; // 目录读不动就跳过,不阻断整条链
181
188
  }
182
- for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
189
+ for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
183
190
  if (out.length >= maxFiles) break;
184
- if (e.name.startsWith('.') || e.name === 'node_modules') continue;
185
- const full = path.join(dir, e.name);
186
- if (e.isDirectory()) walkFiles(full, { ext, out, maxFiles });
187
- else if (!ext || e.name.toLowerCase().endsWith(ext)) out.push(full);
191
+ if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
192
+ const full = path.join(dir, entry.name);
193
+ if (entry.isDirectory()) collectFiles(full, { ext, out, maxFiles });
194
+ else if (!ext || entry.name.toLowerCase().endsWith(ext)) out.push(full);
188
195
  }
189
196
  return out;
190
- }
197
+ };
198
+
199
+ // ① persona 铁律段:显式给的文件优先,否则扫目录里的 agent.cordis.yml
200
+ const personaEntries = ({ personaPaths, personaDir, maxFiles }) => {
201
+ if (Array.isArray(personaPaths) && personaPaths.length) {
202
+ return personaPaths.map((p) => ({ file: path.resolve(String(p)), kind: 'persona' }));
203
+ }
204
+ if (!personaDir || !fs.existsSync(personaDir)) return [];
205
+ return collectFiles(personaDir, { ext: '.yml', maxFiles })
206
+ .filter((file) => path.basename(file).toLowerCase() === 'agent.cordis.yml')
207
+ .map((file) => ({ file, kind: 'persona' }));
208
+ };
209
+
210
+ // ② 技能段:每个技能的 SKILL.md,按 ## 小节切
211
+ const skillEntries = ({ skillsRoot, maxFiles }) => {
212
+ if (!skillsRoot || !fs.existsSync(skillsRoot)) return [];
213
+ return collectFiles(skillsRoot, { ext: null, maxFiles })
214
+ .filter((file) => path.basename(file).toLowerCase() === 'skill.md')
215
+ .map((file) => ({ file, kind: 'markdown' }));
216
+ };
217
+
218
+ // ③ 文档段:只收点名的少数几篇(不做全量 docs —— 那是文档索引的活)
219
+ const docEntries = ({ docsPaths, maxFiles }) => {
220
+ const picked = [];
221
+ for (const item of Array.isArray(docsPaths) ? docsPaths : []) {
222
+ if (!item) continue;
223
+ const resolved = path.resolve(String(item));
224
+ if (!fs.existsSync(resolved)) continue;
225
+ if (fs.statSync(resolved).isDirectory()) {
226
+ for (const file of collectFiles(resolved, { ext: '.md', maxFiles })) picked.push({ file, kind: 'markdown' });
227
+ } else {
228
+ picked.push({ file: resolved, kind: 'markdown' });
229
+ }
230
+ }
231
+ return picked;
232
+ };
191
233
 
192
234
  /**
193
- * 扫描 policy 语料 ⇒ 段清单(只读,零写入)。
194
- *
195
- * 语料三份(方案 §4 B.3-1,**按判据强度排序**;MVP 只做前两份,第三份默认空):
196
- * ① persona 铁律段:`.agent-presets\**\agent.cordis.yml` 的 persona 文本 → 按 `【…】` 切
197
- * ② 技能段:`skills\*\SKILL.md` → 按 `##` 小节切
198
- * ③ 文档段(可选/后置):`docsRoots` 里被点名的少数几篇 —— **不做全量 docs**(那是 doc-index 的活)
199
- *
200
- * @param {object} p
201
- * @param {string[]} [p.personaPaths] 显式 persona 文件(缺省 = 扫 personaDir 下的 agent.cordis.yml)
202
- * @param {string} [p.personaDir]
203
- * @param {string} [p.skillsRoot]
204
- * @param {string[]} [p.docsPaths]
205
- * @param {number} [p.maxChars]
235
+ * 扫描 policy 语料 ⇒ 段清单(只读、零写入)。
206
236
  * @returns {Array<{refId:string, docPath:string, relPath:string, section:string, text:string, mtime:number, textHash:string}>}
207
237
  */
208
- export function scanPolicySources({
238
+ export const scanPolicySources = ({
209
239
  personaPaths = null,
210
240
  personaDir = null,
211
241
  skillsRoot = null,
212
242
  docsPaths = null,
213
243
  maxChars = POLICY_MAX_CHARS,
214
244
  maxFiles = 2000,
215
- } = {}) {
216
- const files = [];
217
- if (Array.isArray(personaPaths) && personaPaths.length) {
218
- for (const p of personaPaths) files.push({ file: path.resolve(String(p)), kind: 'persona' });
219
- } else if (personaDir && fs.existsSync(personaDir)) {
220
- for (const f of walkFiles(personaDir, { ext: '.yml', maxFiles })) {
221
- if (path.basename(f).toLowerCase() === 'agent.cordis.yml') files.push({ file: f, kind: 'persona' });
222
- }
223
- }
224
- if (skillsRoot && fs.existsSync(skillsRoot)) {
225
- for (const f of walkFiles(skillsRoot, { ext: null, maxFiles })) {
226
- if (path.basename(f).toLowerCase() === 'skill.md') files.push({ file: f, kind: 'markdown' });
227
- }
228
- }
229
- for (const d of Array.isArray(docsPaths) ? docsPaths : []) {
230
- if (!d) continue;
231
- const resolved = path.resolve(String(d));
232
- if (!fs.existsSync(resolved)) continue;
233
- if (fs.statSync(resolved).isDirectory()) {
234
- for (const f of walkFiles(resolved, { ext: '.md', maxFiles })) files.push({ file: f, kind: 'markdown' });
235
- } else {
236
- files.push({ file: resolved, kind: 'markdown' });
237
- }
238
- }
245
+ } = {}) => {
246
+ const files = [
247
+ ...personaEntries({ personaPaths, personaDir, maxFiles }),
248
+ ...skillEntries({ skillsRoot, maxFiles }),
249
+ ...docEntries({ docsPaths, maxFiles }),
250
+ ];
239
251
 
240
252
  const segments = [];
241
253
  for (const { file, kind } of files) {
@@ -245,15 +257,17 @@ export function scanPolicySources({
245
257
  raw = fs.readFileSync(file, 'utf8');
246
258
  mtime = Math.round(fs.statSync(file).mtimeMs);
247
259
  } catch {
248
- continue; // 读不到就跳过(不阻断整条链)
260
+ continue; // 读不到就跳过 —— 一份文件坏掉不该拖死整条链
249
261
  }
250
- const relPath = normalizeDocPath(file);
262
+
251
263
  const text = kind === 'persona' ? extractPersonaText(raw) : raw;
252
264
  if (!String(text).trim()) continue;
265
+
266
+ const relPath = normalizeDocPath(file);
253
267
  for (const seg of segmentText(text, { kind, maxChars })) {
254
268
  segments.push({
255
269
  refId: policyRefId(relPath, seg.anchor, seg.chunkIndex),
256
- docPath: normalizeDocPath(file),
270
+ docPath: relPath,
257
271
  relPath,
258
272
  section: seg.section,
259
273
  text: seg.text,
@@ -262,110 +276,114 @@ export function scanPolicySources({
262
276
  });
263
277
  }
264
278
  }
265
- // 稳定排序(段序 = 文件名字典序 × 文件内出现序;refId 单调游标依赖它)
266
- segments.sort((a, b) => (a.docPath < b.docPath ? -1 : a.docPath > b.docPath ? 1 : 0)
267
- || (a.mtime - b.mtime)
268
- || (a.refId < b.refId ? -1 : a.refId > b.refId ? 1 : 0));
279
+
280
+ // 稳定排序:文件名(字典序)× 文件内出现序 —— 增量游标依赖这个顺序
281
+ segments.sort(
282
+ (a, b) =>
283
+ (a.docPath < b.docPath ? -1 : a.docPath > b.docPath ? 1 : 0) ||
284
+ a.mtime - b.mtime ||
285
+ (a.refId < b.refId ? -1 : a.refId > b.refId ? 1 : 0),
286
+ );
269
287
  return segments;
270
- }
288
+ };
271
289
 
272
- function cursorKey(updatedAt, id) {
273
- return `${String(updatedAt).padStart(16, '0')}|${id}`;
274
- }
290
+ // 游标键:updated_at 左补零后拼 id,字符串比较即等价于 (时间, id) 的字典序
291
+ const cursorOf = (updatedAt, id) => `${String(updatedAt).padStart(16, '0')}|${id}`;
275
292
 
276
293
  /**
277
- * 把段清单包成**与 memory-source 同形状**的只读数据源(供 `sync --source policy` / `sweep` / `recall` 直接消费)。
278
- * @param {{segments:Array, dbPath?:string}} p
294
+ * 把段清单包成与记忆源同形状的只读数据源(同步 / 清扫 / 召回按方法名直接消费)。
295
+ * @param {{segments?:Array, dbPath?:string}} [p]
279
296
  */
280
- export function openPolicySource({ segments = [], dbPath = null } = {}) {
297
+ export const openPolicySource = ({ segments = [], dbPath = null } = {}) => {
281
298
  const rows = segments
282
- .map((s) => ({
283
- id: String(s.refId),
284
- text: s.text,
285
- tags: s.section || '',
299
+ .map((seg) => ({
300
+ id: String(seg.refId),
301
+ text: seg.text,
302
+ tags: seg.section || '',
286
303
  pinned: 0,
287
- updated_at: Number(s.mtime) || 0,
288
- docPath: s.docPath,
289
- section: s.section,
290
- textHash: s.textHash,
304
+ updated_at: Number(seg.mtime) || 0,
305
+ docPath: seg.docPath,
306
+ section: seg.section,
307
+ textHash: seg.textHash,
291
308
  }))
292
309
  .sort((a, b) => a.updated_at - b.updated_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
293
- const byId = new Map(rows.map((r) => [r.id, r]));
294
- let closed = false;
310
+
311
+ const byId = new Map(rows.map((row) => [row.id, row]));
312
+ let shut = false;
295
313
 
296
314
  return {
297
315
  dbPath: dbPath || '(scanned:policy)',
298
316
  readOnly: true,
299
317
  sourceName: 'policy',
300
318
  n: rows.length,
301
- /** 原始段对象(triage 侧要 docPath/section 做「出处可追溯」) */
319
+ // 原始段对象(分诊侧要 docPath / section 做「出处可追溯」)
302
320
  segments,
303
321
 
304
- count() {
305
- return rows.length;
306
- },
307
- maxUpdatedAt() {
308
- let m = 0;
309
- for (const r of rows) if (r.updated_at > m) m = r.updated_at;
310
- return m;
311
- },
312
- idRange() {
313
- if (!rows.length) return { lo: null, hi: null, n: 0 };
314
- return { lo: 0, hi: rows.length - 1, n: rows.length };
315
- },
316
- pinnedIds() {
317
- return [];
322
+ count: () => rows.length,
323
+
324
+ maxUpdatedAt: () => {
325
+ let high = 0;
326
+ for (const row of rows) if (row.updated_at > high) high = row.updated_at;
327
+ return high;
318
328
  },
319
- incrementalPage(sinceTs, afterId, limit) {
329
+
330
+ idRange: () => (rows.length ? { lo: 0, hi: rows.length - 1, n: rows.length } : { lo: null, hi: null, n: 0 }),
331
+
332
+ pinnedIds: () => [],
333
+
334
+ // 增量页:先按 since 过滤,再按游标(时间, id)往后取,最多 lim 条
335
+ incrementalPage: (sinceTs, afterId, limit) => {
320
336
  const since = Number(sinceTs) || 0;
321
337
  const lim = Number(limit) || 0;
322
338
  if (!rows.length || lim <= 0) return [];
323
- const cur = afterId == null || afterId === '' ? null : String(afterId);
339
+
340
+ const cursorId = afterId == null || afterId === '' ? null : String(afterId);
324
341
  let floor = '';
325
- if (cur != null) {
326
- const hit = byId.get(cur);
327
- floor = hit ? cursorKey(hit.updated_at, hit.id) : cursorKey(0, cur);
342
+ if (cursorId != null) {
343
+ const anchorRow = byId.get(cursorId);
344
+ floor = anchorRow ? cursorOf(anchorRow.updated_at, anchorRow.id) : cursorOf(0, cursorId);
328
345
  }
329
- const out = [];
330
- for (const r of rows) {
331
- if (r.updated_at < since) continue;
332
- if (floor && cursorKey(r.updated_at, r.id) <= floor) continue;
333
- out.push({ ...r });
334
- if (out.length >= lim) break;
346
+
347
+ const page = [];
348
+ for (const row of rows) {
349
+ if (row.updated_at < since) continue;
350
+ if (floor && cursorOf(row.updated_at, row.id) <= floor) continue;
351
+ page.push({ ...row });
352
+ if (page.length >= lim) break;
335
353
  }
336
- return out;
337
- },
338
- segment(lo, hi) {
339
- const a = Number(lo);
340
- const b = Number(hi);
341
- if (!Number.isFinite(a) || !Number.isFinite(b)) return [];
342
- return rows.slice(Math.max(0, a), Math.max(0, b) + 1).map((r) => ({ ...r }));
354
+ return page;
343
355
  },
344
- getById(id) {
345
- const r = byId.get(String(id));
346
- return r ? { ...r } : null;
347
- },
348
- allTexts() {
349
- return rows.map((r) => ({ id: r.id, text: r.text }));
350
- },
351
- allTags() {
352
- return rows.map((r) => ({ id: r.id, tags: r.tags }));
356
+
357
+ // 按序号取一段(含 lo、hi 两端)
358
+ segment: (lo, hi) => {
359
+ const from = Number(lo);
360
+ const to = Number(hi);
361
+ if (!Number.isFinite(from) || !Number.isFinite(to)) return [];
362
+ return rows.slice(Math.max(0, from), Math.max(0, to) + 1).map((row) => ({ ...row }));
353
363
  },
354
- /** policy 段无 FTS5 ⇒ 必须返回对象(`recall.mjs:167` 期望对象;返回数组才会走补位路) */
355
- ftsSearch() {
356
- return { error: 'policy 段源无 FTS5(policy source 无 FTS 补位路)' };
364
+
365
+ getById: (id) => {
366
+ const row = byId.get(String(id));
367
+ return row ? { ...row } : null;
357
368
  },
358
- close() {
359
- if (closed) return;
360
- closed = true;
369
+
370
+ allTexts: () => rows.map((row) => ({ id: row.id, text: row.text })),
371
+
372
+ allTags: () => rows.map((row) => ({ id: row.id, tags: row.tags })),
373
+
374
+ // 段源没有 FTS5:必须回对象(召回侧期望对象;回数组才会被当成「有补位路」)
375
+ ftsSearch: () => ({ error: 'policy 段源无 FTS5(policy source 无 FTS 补位路)' }),
376
+
377
+ close: () => {
378
+ shut = true;
361
379
  },
380
+
362
381
  get closed() {
363
- return closed;
382
+ return shut;
364
383
  },
365
384
  };
366
- }
385
+ };
367
386
 
368
- /** 便捷:扫描 + 打包一步到位 */
369
- export function openPolicySourceFromDisk(opts = {}) {
370
- return openPolicySource({ segments: scanPolicySources(opts), dbPath: opts.dbPath || null });
371
- }
387
+ // 便捷口:扫描 + 打包一步到位
388
+ export const openPolicySourceFromDisk = (opts = {}) =>
389
+ openPolicySource({ segments: scanPolicySources(opts), dbPath: opts.dbPath || null });