@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,241 +1,253 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 离线层 —— policy 只读段源(把「判据落地处」编成可检索的段)
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// ⇒ 自动产出「疑似已落地 / 确属新根 / 中性(需人看)」的**建议**。
|
|
3
|
+
// 用途:候选池里那条坑「防复发措施到底落地没」原本只能人工翻文档。这里把落地处
|
|
4
|
+
// (persona 铁律段 + 技能段 + 少数指定文档段)切成段,候选正文拿去比相似度,
|
|
5
|
+
// 产出「疑似已落地 / 确实是新根 / 中性待看」的**建议**。
|
|
7
6
|
//
|
|
8
|
-
//
|
|
9
|
-
// **向量判据只产出建议,不自动改候选池 status、不自动采纳、永不自动改 persona/技能。**
|
|
7
|
+
// 边界(写死,别越):只出建议,不改候选状态、不自动采纳、永不改 persona 或技能文件。
|
|
10
8
|
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
9
|
+
// 接口形状与记忆源 / 坑源保持一致(同步、清扫、召回都是按方法名直接调的):
|
|
10
|
+
// segment() / incrementalPage() 返回的每一行都必须带 id / text / updated_at 三个字段,
|
|
11
|
+
// 缺了不会报错,只会把 undefined 悄悄写进向量库。
|
|
14
12
|
//
|
|
15
|
-
//
|
|
13
|
+
// 段化口径:按 markdown 标题或 【…】 标记切段,每段 ≤400 字,超长段落再按句子切。
|
|
16
14
|
import fs from 'node:fs';
|
|
17
15
|
import path from 'node:path';
|
|
18
16
|
import crypto from 'node:crypto';
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
// 每段硬上限(超长先按句切,单句还超长就硬截)
|
|
21
19
|
export const POLICY_MAX_CHARS = 400;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return
|
|
40
|
-
}
|
|
20
|
+
|
|
21
|
+
// 段身份锚里的分隔符:用 NUL,保证不会跟正文内容撞
|
|
22
|
+
const ANCHOR_SEP = '\u0000';
|
|
23
|
+
|
|
24
|
+
// 段文本哈希(sha256 hex,与增量同步的跳过判据同一套算法)
|
|
25
|
+
export const policyTextHash = (text) =>
|
|
26
|
+
crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
|
|
27
|
+
|
|
28
|
+
// 段身份锚(16 位 hex,与坑库 id 同形):只由「文件 + 段锚 + 分片号」决定,与正文无关。
|
|
29
|
+
// ⇒ 正文改了是同一条 ref_id 换 text_hash(增量重编码),不会留孤儿行。
|
|
30
|
+
export const policyRefId = (relPath, anchor, chunkIndex = 1) =>
|
|
31
|
+
crypto.createHash('sha1').update([relPath, anchor, chunkIndex].join(ANCHOR_SEP), 'utf8').digest('hex').slice(0, 16);
|
|
32
|
+
|
|
33
|
+
// 路径归一:反斜杠转正斜杠 + 盘符小写(跨平台可复现;policy 库本机自用、不参与同步)
|
|
34
|
+
export const normalizeDocPath = (p) => {
|
|
35
|
+
const slashed = String(p).replace(/\\/g, '/');
|
|
36
|
+
if (slashed.length < 2 || slashed[1] !== ':') return slashed;
|
|
37
|
+
return slashed[0].toLowerCase() + slashed.slice(1);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
// ── persona 块标量抽取 ──────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
// 块起始行:`prefix: |` / `persona: >-` 这类写法都认
|
|
43
|
+
const BLOCK_MARK = /^(\s*)(?:prefix|persona|text)\s*:\s*[|>][-+]?\s*$/;
|
|
44
|
+
|
|
45
|
+
const indentWidth = (line) => line.length - line.replace(/^[ \t]*/, '').length;
|
|
41
46
|
|
|
42
47
|
/**
|
|
43
|
-
* 从
|
|
44
|
-
*
|
|
48
|
+
* 从 agent.cordis.yml 里抽出 persona 段(YAML 块标量的内容)。
|
|
49
|
+
* 抽不到就返回整份文本 —— 宁可编粗一点,也不让整条链断在这。
|
|
45
50
|
*/
|
|
46
|
-
export
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
break;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
if (start < 0) return String(yamlText ?? '');
|
|
59
|
-
const body = [];
|
|
60
|
-
for (let i = start; i < lines.length; i += 1) {
|
|
51
|
+
export const extractPersonaText = (yamlText) => {
|
|
52
|
+
const source = String(yamlText ?? '');
|
|
53
|
+
const lines = source.split(/\r?\n/);
|
|
54
|
+
const at = lines.findIndex((line) => BLOCK_MARK.test(line));
|
|
55
|
+
if (at < 0) return source;
|
|
56
|
+
|
|
57
|
+
const markerIndent = indentWidth(lines[at]);
|
|
58
|
+
const kept = [];
|
|
59
|
+
for (let i = at + 1; i < lines.length; i += 1) {
|
|
61
60
|
const line = lines[i];
|
|
62
61
|
if (!line.trim()) {
|
|
63
|
-
|
|
62
|
+
kept.push('');
|
|
64
63
|
continue;
|
|
65
64
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
body.push(line);
|
|
65
|
+
if (indentWidth(line) <= markerIndent) break; // 缩进退回标记行 ⇒ 块结束
|
|
66
|
+
kept.push(line);
|
|
69
67
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
68
|
+
|
|
69
|
+
// 去掉块标量的公共缩进(YAML 默认取首行缩进)
|
|
70
|
+
const firstKept = kept.find((line) => line.trim());
|
|
71
|
+
const base = firstKept ? indentWidth(firstKept) : 0;
|
|
72
|
+
return kept.map((line) => (line.trim() ? line.slice(Math.min(base, line.length)) : '')).join('\n').trim();
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// ── 切块 ────────────────────────────────────────────────────────────────────
|
|
76
|
+
|
|
77
|
+
// 按「行首标记」切块,返回 [{title, body}];第一个标记之前的非空内容归到 (前言)
|
|
78
|
+
const splitBlocks = (text, markRe) => {
|
|
78
79
|
const lines = String(text ?? '').split(/\r?\n/);
|
|
79
|
-
const
|
|
80
|
-
let
|
|
81
|
-
let
|
|
80
|
+
const groups = [];
|
|
81
|
+
let current = null;
|
|
82
|
+
let loose = [];
|
|
83
|
+
|
|
82
84
|
for (const line of lines) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
} else if (cur) {
|
|
90
|
-
cur.body.push(line);
|
|
91
|
-
} else {
|
|
92
|
-
preamble.push(line);
|
|
85
|
+
if (markRe.test(line)) {
|
|
86
|
+
if (current) groups.push(current);
|
|
87
|
+
else if (loose.join('').trim()) groups.push({ title: '(前言)', body: loose });
|
|
88
|
+
loose = [];
|
|
89
|
+
current = { title: line.trim(), body: [line] };
|
|
90
|
+
continue;
|
|
93
91
|
}
|
|
92
|
+
(current ? current.body : loose).push(line);
|
|
94
93
|
}
|
|
95
|
-
if (
|
|
96
|
-
else if (
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
94
|
+
if (current) groups.push(current);
|
|
95
|
+
else if (loose.join('').trim()) groups.push({ title: '(前言)', body: loose });
|
|
96
|
+
|
|
97
|
+
return groups.map((group) => ({ title: group.title, body: group.body.join('\n').trim() }));
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
// 超长文本按句切(句末标点与换行都算断点);单句仍超长就按上限硬截
|
|
101
|
+
const sliceLong = (text, maxChars) => {
|
|
102
|
+
const whole = String(text ?? '').trim();
|
|
103
|
+
if (!whole) return [];
|
|
104
|
+
if (whole.length <= maxChars) return [whole];
|
|
105
|
+
|
|
106
|
+
const sentences = whole.split(/(?<=[。!?;\n])/u).filter((piece) => piece.trim());
|
|
107
|
+
const slices = [];
|
|
108
|
+
let buffer = '';
|
|
109
|
+
|
|
110
|
+
for (const sentence of sentences.length ? sentences : [whole]) {
|
|
111
|
+
if (buffer && (buffer + sentence).length > maxChars) {
|
|
112
|
+
slices.push(buffer.trim());
|
|
113
|
+
buffer = '';
|
|
112
114
|
}
|
|
113
|
-
if (
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
buf = '';
|
|
117
|
-
}
|
|
118
|
-
for (let i = 0; i < p.length; i += maxChars) out.push(p.slice(i, i + maxChars).trim());
|
|
119
|
-
} else {
|
|
120
|
-
buf += p;
|
|
115
|
+
if (sentence.length > maxChars) {
|
|
116
|
+
for (let at = 0; at < sentence.length; at += maxChars) slices.push(sentence.slice(at, at + maxChars).trim());
|
|
117
|
+
continue;
|
|
121
118
|
}
|
|
119
|
+
buffer += sentence;
|
|
122
120
|
}
|
|
123
|
-
if (
|
|
124
|
-
return
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const markerRe = kind === 'persona' ? /^\s*-?\s*【[^】]*】.*$/u : /^#{2,3}\s+.*$/u;
|
|
134
|
-
const blocks = splitByLineMarker(text, markerRe);
|
|
135
|
-
|
|
136
|
-
// 无标记大块(例:另一个 preset 的 persona 完全没有 【】)先按**空行段**拆,
|
|
137
|
-
// 再对仍超长的段落做句级硬切 —— 避免把一整份 19 KB 文本切成一串半句碎片。
|
|
138
|
-
const pre = [];
|
|
139
|
-
for (const b of blocks) {
|
|
140
|
-
const title = b.title || '(无标题)';
|
|
141
|
-
const body = String(b.body ?? '');
|
|
121
|
+
if (buffer.trim()) slices.push(buffer.trim());
|
|
122
|
+
return slices.filter(Boolean);
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
// 块太大(超过上限 4 倍)且内部有空行分段 ⇒ 按空行先拆开,免得切出一串半句碎片
|
|
126
|
+
const splitOversized = (blocks, maxChars) => {
|
|
127
|
+
const pieces = [];
|
|
128
|
+
for (const block of blocks) {
|
|
129
|
+
const base = block.title || '(无标题)';
|
|
130
|
+
const body = String(block.body ?? '');
|
|
142
131
|
if (body.length > maxChars * 4) {
|
|
143
132
|
const paras = body.split(/\n\s*\n/).map((s) => s.trim()).filter(Boolean);
|
|
144
133
|
if (paras.length > 1) {
|
|
145
|
-
paras.forEach((
|
|
134
|
+
paras.forEach((para, i) => pieces.push({ base, sub: i + 1, body: para }));
|
|
146
135
|
continue;
|
|
147
136
|
}
|
|
148
137
|
}
|
|
149
|
-
|
|
138
|
+
pieces.push({ base, sub: 0, body });
|
|
150
139
|
}
|
|
140
|
+
return pieces;
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
// 段展示名:`<标题>`;前言块后面补首行摘要,段内换行折成空格,让 docPath#section 可读
|
|
144
|
+
const sectionLabel = (piece) => {
|
|
145
|
+
const firstLine = piece.body.trim().split('\n')[0];
|
|
146
|
+
const head = firstLine.slice(0, 40).replace(/\s+/g, ' ').trim();
|
|
147
|
+
return piece.base.startsWith('(') ? `${piece.base} ${head}` : piece.base;
|
|
148
|
+
};
|
|
151
149
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
150
|
+
// persona 段:行首 【…】 与列表式 `- 【…】` 都算新段起点(协作偏好写成列表,只认行首 【 会切不动)
|
|
151
|
+
const PERSONA_MARK = /^\s*-?\s*【[^】]*】.*$/u;
|
|
152
|
+
const HEADING_MARK = /^#{2,3}\s+.*$/u;
|
|
153
|
+
|
|
154
|
+
/** 单份文本 → 段清单(顺序稳定:文件出现序;同名小节用 #N 区分) */
|
|
155
|
+
export const segmentText = (text, { kind = 'markdown', maxChars = POLICY_MAX_CHARS } = {}) => {
|
|
156
|
+
const markRe = kind === 'persona' ? PERSONA_MARK : HEADING_MARK;
|
|
157
|
+
const pieces = splitOversized(splitBlocks(text, markRe), maxChars);
|
|
158
|
+
const occurrences = new Map();
|
|
159
|
+
const segments = [];
|
|
160
|
+
|
|
161
|
+
for (const piece of pieces) {
|
|
162
|
+
const nth = (occurrences.get(piece.base) ?? 0) + 1;
|
|
163
|
+
occurrences.set(piece.base, nth);
|
|
164
|
+
const anchor = piece.sub ? `${piece.base}#${nth}.${piece.sub}` : `${piece.base}#${nth}`;
|
|
165
|
+
const slices = sliceLong(piece.body, maxChars);
|
|
166
|
+
const label = sectionLabel(piece);
|
|
167
|
+
slices.forEach((chunk, i) => {
|
|
168
|
+
segments.push({
|
|
164
169
|
anchor,
|
|
165
|
-
section:
|
|
170
|
+
section: slices.length > 1 ? `${label} (part ${i + 1})` : label,
|
|
166
171
|
chunkIndex: i + 1,
|
|
167
|
-
text:
|
|
172
|
+
text: chunk,
|
|
168
173
|
});
|
|
169
174
|
});
|
|
170
175
|
}
|
|
171
|
-
return
|
|
172
|
-
}
|
|
176
|
+
return segments;
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
// ── 文件遍历与扫描 ──────────────────────────────────────────────────────────
|
|
173
180
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
let entries
|
|
181
|
+
// 递归收文件(跳过隐藏目录与 node_modules;ext 为空表示不过滤后缀)
|
|
182
|
+
const collectFiles = (dir, { ext = null, out = [], maxFiles = 2000 } = {}) => {
|
|
183
|
+
let entries;
|
|
177
184
|
try {
|
|
178
185
|
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
179
186
|
} catch {
|
|
180
|
-
return out;
|
|
187
|
+
return out; // 目录读不动就跳过,不阻断整条链
|
|
181
188
|
}
|
|
182
|
-
for (const
|
|
189
|
+
for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
|
|
183
190
|
if (out.length >= maxFiles) break;
|
|
184
|
-
if (
|
|
185
|
-
const full = path.join(dir,
|
|
186
|
-
if (
|
|
187
|
-
else if (!ext ||
|
|
191
|
+
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
|
192
|
+
const full = path.join(dir, entry.name);
|
|
193
|
+
if (entry.isDirectory()) collectFiles(full, { ext, out, maxFiles });
|
|
194
|
+
else if (!ext || entry.name.toLowerCase().endsWith(ext)) out.push(full);
|
|
188
195
|
}
|
|
189
196
|
return out;
|
|
190
|
-
}
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
// ① persona 铁律段:显式给的文件优先,否则扫目录里的 agent.cordis.yml
|
|
200
|
+
const personaEntries = ({ personaPaths, personaDir, maxFiles }) => {
|
|
201
|
+
if (Array.isArray(personaPaths) && personaPaths.length) {
|
|
202
|
+
return personaPaths.map((p) => ({ file: path.resolve(String(p)), kind: 'persona' }));
|
|
203
|
+
}
|
|
204
|
+
if (!personaDir || !fs.existsSync(personaDir)) return [];
|
|
205
|
+
return collectFiles(personaDir, { ext: '.yml', maxFiles })
|
|
206
|
+
.filter((file) => path.basename(file).toLowerCase() === 'agent.cordis.yml')
|
|
207
|
+
.map((file) => ({ file, kind: 'persona' }));
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
// ② 技能段:每个技能的 SKILL.md,按 ## 小节切
|
|
211
|
+
const skillEntries = ({ skillsRoot, maxFiles }) => {
|
|
212
|
+
if (!skillsRoot || !fs.existsSync(skillsRoot)) return [];
|
|
213
|
+
return collectFiles(skillsRoot, { ext: null, maxFiles })
|
|
214
|
+
.filter((file) => path.basename(file).toLowerCase() === 'skill.md')
|
|
215
|
+
.map((file) => ({ file, kind: 'markdown' }));
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
// ③ 文档段:只收点名的少数几篇(不做全量 docs —— 那是文档索引的活)
|
|
219
|
+
const docEntries = ({ docsPaths, maxFiles }) => {
|
|
220
|
+
const picked = [];
|
|
221
|
+
for (const item of Array.isArray(docsPaths) ? docsPaths : []) {
|
|
222
|
+
if (!item) continue;
|
|
223
|
+
const resolved = path.resolve(String(item));
|
|
224
|
+
if (!fs.existsSync(resolved)) continue;
|
|
225
|
+
if (fs.statSync(resolved).isDirectory()) {
|
|
226
|
+
for (const file of collectFiles(resolved, { ext: '.md', maxFiles })) picked.push({ file, kind: 'markdown' });
|
|
227
|
+
} else {
|
|
228
|
+
picked.push({ file: resolved, kind: 'markdown' });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return picked;
|
|
232
|
+
};
|
|
191
233
|
|
|
192
234
|
/**
|
|
193
|
-
* 扫描 policy 语料 ⇒
|
|
194
|
-
*
|
|
195
|
-
* 语料三份(方案 §4 B.3-1,**按判据强度排序**;MVP 只做前两份,第三份默认空):
|
|
196
|
-
* ① persona 铁律段:`.agent-presets\**\agent.cordis.yml` 的 persona 文本 → 按 `【…】` 切
|
|
197
|
-
* ② 技能段:`skills\*\SKILL.md` → 按 `##` 小节切
|
|
198
|
-
* ③ 文档段(可选/后置):`docsRoots` 里被点名的少数几篇 —— **不做全量 docs**(那是 doc-index 的活)
|
|
199
|
-
*
|
|
200
|
-
* @param {object} p
|
|
201
|
-
* @param {string[]} [p.personaPaths] 显式 persona 文件(缺省 = 扫 personaDir 下的 agent.cordis.yml)
|
|
202
|
-
* @param {string} [p.personaDir]
|
|
203
|
-
* @param {string} [p.skillsRoot]
|
|
204
|
-
* @param {string[]} [p.docsPaths]
|
|
205
|
-
* @param {number} [p.maxChars]
|
|
235
|
+
* 扫描 policy 语料 ⇒ 段清单(只读、零写入)。
|
|
206
236
|
* @returns {Array<{refId:string, docPath:string, relPath:string, section:string, text:string, mtime:number, textHash:string}>}
|
|
207
237
|
*/
|
|
208
|
-
export
|
|
238
|
+
export const scanPolicySources = ({
|
|
209
239
|
personaPaths = null,
|
|
210
240
|
personaDir = null,
|
|
211
241
|
skillsRoot = null,
|
|
212
242
|
docsPaths = null,
|
|
213
243
|
maxChars = POLICY_MAX_CHARS,
|
|
214
244
|
maxFiles = 2000,
|
|
215
|
-
} = {}) {
|
|
216
|
-
const files = [
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
if (path.basename(f).toLowerCase() === 'agent.cordis.yml') files.push({ file: f, kind: 'persona' });
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
if (skillsRoot && fs.existsSync(skillsRoot)) {
|
|
225
|
-
for (const f of walkFiles(skillsRoot, { ext: null, maxFiles })) {
|
|
226
|
-
if (path.basename(f).toLowerCase() === 'skill.md') files.push({ file: f, kind: 'markdown' });
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
for (const d of Array.isArray(docsPaths) ? docsPaths : []) {
|
|
230
|
-
if (!d) continue;
|
|
231
|
-
const resolved = path.resolve(String(d));
|
|
232
|
-
if (!fs.existsSync(resolved)) continue;
|
|
233
|
-
if (fs.statSync(resolved).isDirectory()) {
|
|
234
|
-
for (const f of walkFiles(resolved, { ext: '.md', maxFiles })) files.push({ file: f, kind: 'markdown' });
|
|
235
|
-
} else {
|
|
236
|
-
files.push({ file: resolved, kind: 'markdown' });
|
|
237
|
-
}
|
|
238
|
-
}
|
|
245
|
+
} = {}) => {
|
|
246
|
+
const files = [
|
|
247
|
+
...personaEntries({ personaPaths, personaDir, maxFiles }),
|
|
248
|
+
...skillEntries({ skillsRoot, maxFiles }),
|
|
249
|
+
...docEntries({ docsPaths, maxFiles }),
|
|
250
|
+
];
|
|
239
251
|
|
|
240
252
|
const segments = [];
|
|
241
253
|
for (const { file, kind } of files) {
|
|
@@ -245,15 +257,17 @@ export function scanPolicySources({
|
|
|
245
257
|
raw = fs.readFileSync(file, 'utf8');
|
|
246
258
|
mtime = Math.round(fs.statSync(file).mtimeMs);
|
|
247
259
|
} catch {
|
|
248
|
-
continue; //
|
|
260
|
+
continue; // 读不到就跳过 —— 一份文件坏掉不该拖死整条链
|
|
249
261
|
}
|
|
250
|
-
|
|
262
|
+
|
|
251
263
|
const text = kind === 'persona' ? extractPersonaText(raw) : raw;
|
|
252
264
|
if (!String(text).trim()) continue;
|
|
265
|
+
|
|
266
|
+
const relPath = normalizeDocPath(file);
|
|
253
267
|
for (const seg of segmentText(text, { kind, maxChars })) {
|
|
254
268
|
segments.push({
|
|
255
269
|
refId: policyRefId(relPath, seg.anchor, seg.chunkIndex),
|
|
256
|
-
docPath:
|
|
270
|
+
docPath: relPath,
|
|
257
271
|
relPath,
|
|
258
272
|
section: seg.section,
|
|
259
273
|
text: seg.text,
|
|
@@ -262,110 +276,114 @@ export function scanPolicySources({
|
|
|
262
276
|
});
|
|
263
277
|
}
|
|
264
278
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
279
|
+
|
|
280
|
+
// 稳定排序:文件名(字典序)× 文件内出现序 —— 增量游标依赖这个顺序
|
|
281
|
+
segments.sort(
|
|
282
|
+
(a, b) =>
|
|
283
|
+
(a.docPath < b.docPath ? -1 : a.docPath > b.docPath ? 1 : 0) ||
|
|
284
|
+
a.mtime - b.mtime ||
|
|
285
|
+
(a.refId < b.refId ? -1 : a.refId > b.refId ? 1 : 0),
|
|
286
|
+
);
|
|
269
287
|
return segments;
|
|
270
|
-
}
|
|
288
|
+
};
|
|
271
289
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
290
|
+
// 游标键:updated_at 左补零后拼 id,字符串比较即等价于 (时间, id) 的字典序
|
|
291
|
+
const cursorOf = (updatedAt, id) => `${String(updatedAt).padStart(16, '0')}|${id}`;
|
|
275
292
|
|
|
276
293
|
/**
|
|
277
|
-
*
|
|
278
|
-
* @param {{segments
|
|
294
|
+
* 把段清单包成与记忆源同形状的只读数据源(同步 / 清扫 / 召回按方法名直接消费)。
|
|
295
|
+
* @param {{segments?:Array, dbPath?:string}} [p]
|
|
279
296
|
*/
|
|
280
|
-
export
|
|
297
|
+
export const openPolicySource = ({ segments = [], dbPath = null } = {}) => {
|
|
281
298
|
const rows = segments
|
|
282
|
-
.map((
|
|
283
|
-
id: String(
|
|
284
|
-
text:
|
|
285
|
-
tags:
|
|
299
|
+
.map((seg) => ({
|
|
300
|
+
id: String(seg.refId),
|
|
301
|
+
text: seg.text,
|
|
302
|
+
tags: seg.section || '',
|
|
286
303
|
pinned: 0,
|
|
287
|
-
updated_at: Number(
|
|
288
|
-
docPath:
|
|
289
|
-
section:
|
|
290
|
-
textHash:
|
|
304
|
+
updated_at: Number(seg.mtime) || 0,
|
|
305
|
+
docPath: seg.docPath,
|
|
306
|
+
section: seg.section,
|
|
307
|
+
textHash: seg.textHash,
|
|
291
308
|
}))
|
|
292
309
|
.sort((a, b) => a.updated_at - b.updated_at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
293
|
-
|
|
294
|
-
|
|
310
|
+
|
|
311
|
+
const byId = new Map(rows.map((row) => [row.id, row]));
|
|
312
|
+
let shut = false;
|
|
295
313
|
|
|
296
314
|
return {
|
|
297
315
|
dbPath: dbPath || '(scanned:policy)',
|
|
298
316
|
readOnly: true,
|
|
299
317
|
sourceName: 'policy',
|
|
300
318
|
n: rows.length,
|
|
301
|
-
|
|
319
|
+
// 原始段对象(分诊侧要 docPath / section 做「出处可追溯」)
|
|
302
320
|
segments,
|
|
303
321
|
|
|
304
|
-
count()
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
return m;
|
|
311
|
-
},
|
|
312
|
-
idRange() {
|
|
313
|
-
if (!rows.length) return { lo: null, hi: null, n: 0 };
|
|
314
|
-
return { lo: 0, hi: rows.length - 1, n: rows.length };
|
|
315
|
-
},
|
|
316
|
-
pinnedIds() {
|
|
317
|
-
return [];
|
|
322
|
+
count: () => rows.length,
|
|
323
|
+
|
|
324
|
+
maxUpdatedAt: () => {
|
|
325
|
+
let high = 0;
|
|
326
|
+
for (const row of rows) if (row.updated_at > high) high = row.updated_at;
|
|
327
|
+
return high;
|
|
318
328
|
},
|
|
319
|
-
|
|
329
|
+
|
|
330
|
+
idRange: () => (rows.length ? { lo: 0, hi: rows.length - 1, n: rows.length } : { lo: null, hi: null, n: 0 }),
|
|
331
|
+
|
|
332
|
+
pinnedIds: () => [],
|
|
333
|
+
|
|
334
|
+
// 增量页:先按 since 过滤,再按游标(时间, id)往后取,最多 lim 条
|
|
335
|
+
incrementalPage: (sinceTs, afterId, limit) => {
|
|
320
336
|
const since = Number(sinceTs) || 0;
|
|
321
337
|
const lim = Number(limit) || 0;
|
|
322
338
|
if (!rows.length || lim <= 0) return [];
|
|
323
|
-
|
|
339
|
+
|
|
340
|
+
const cursorId = afterId == null || afterId === '' ? null : String(afterId);
|
|
324
341
|
let floor = '';
|
|
325
|
-
if (
|
|
326
|
-
const
|
|
327
|
-
floor =
|
|
342
|
+
if (cursorId != null) {
|
|
343
|
+
const anchorRow = byId.get(cursorId);
|
|
344
|
+
floor = anchorRow ? cursorOf(anchorRow.updated_at, anchorRow.id) : cursorOf(0, cursorId);
|
|
328
345
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
if (
|
|
333
|
-
|
|
334
|
-
|
|
346
|
+
|
|
347
|
+
const page = [];
|
|
348
|
+
for (const row of rows) {
|
|
349
|
+
if (row.updated_at < since) continue;
|
|
350
|
+
if (floor && cursorOf(row.updated_at, row.id) <= floor) continue;
|
|
351
|
+
page.push({ ...row });
|
|
352
|
+
if (page.length >= lim) break;
|
|
335
353
|
}
|
|
336
|
-
return
|
|
337
|
-
},
|
|
338
|
-
segment(lo, hi) {
|
|
339
|
-
const a = Number(lo);
|
|
340
|
-
const b = Number(hi);
|
|
341
|
-
if (!Number.isFinite(a) || !Number.isFinite(b)) return [];
|
|
342
|
-
return rows.slice(Math.max(0, a), Math.max(0, b) + 1).map((r) => ({ ...r }));
|
|
354
|
+
return page;
|
|
343
355
|
},
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
allTags() {
|
|
352
|
-
return rows.map((r) => ({ id: r.id, tags: r.tags }));
|
|
356
|
+
|
|
357
|
+
// 按序号取一段(含 lo、hi 两端)
|
|
358
|
+
segment: (lo, hi) => {
|
|
359
|
+
const from = Number(lo);
|
|
360
|
+
const to = Number(hi);
|
|
361
|
+
if (!Number.isFinite(from) || !Number.isFinite(to)) return [];
|
|
362
|
+
return rows.slice(Math.max(0, from), Math.max(0, to) + 1).map((row) => ({ ...row }));
|
|
353
363
|
},
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
364
|
+
|
|
365
|
+
getById: (id) => {
|
|
366
|
+
const row = byId.get(String(id));
|
|
367
|
+
return row ? { ...row } : null;
|
|
357
368
|
},
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
369
|
+
|
|
370
|
+
allTexts: () => rows.map((row) => ({ id: row.id, text: row.text })),
|
|
371
|
+
|
|
372
|
+
allTags: () => rows.map((row) => ({ id: row.id, tags: row.tags })),
|
|
373
|
+
|
|
374
|
+
// 段源没有 FTS5:必须回对象(召回侧期望对象;回数组才会被当成「有补位路」)
|
|
375
|
+
ftsSearch: () => ({ error: 'policy 段源无 FTS5(policy source 无 FTS 补位路)' }),
|
|
376
|
+
|
|
377
|
+
close: () => {
|
|
378
|
+
shut = true;
|
|
361
379
|
},
|
|
380
|
+
|
|
362
381
|
get closed() {
|
|
363
|
-
return
|
|
382
|
+
return shut;
|
|
364
383
|
},
|
|
365
384
|
};
|
|
366
|
-
}
|
|
385
|
+
};
|
|
367
386
|
|
|
368
|
-
|
|
369
|
-
export
|
|
370
|
-
|
|
371
|
-
}
|
|
387
|
+
// 便捷口:扫描 + 打包一步到位
|
|
388
|
+
export const openPolicySourceFromDisk = (opts = {}) =>
|
|
389
|
+
openPolicySource({ segments: scanPolicySources(opts), dbPath: opts.dbPath || null });
|