dsh-prime-memory 0.18.2 → 0.18.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.en.md +14 -0
  2. package/CHANGELOG.ja.md +14 -0
  3. package/CHANGELOG.ko.md +14 -0
  4. package/CHANGELOG.md +42 -0
  5. package/README.en.md +4 -0
  6. package/README.ja.md +4 -0
  7. package/README.ko.md +4 -0
  8. package/README.md +4 -0
  9. package/cordis.patch.yml +6 -0
  10. package/dist/client.js +1333 -168
  11. package/dist/config.d.ts +37 -0
  12. package/dist/config.js +7 -0
  13. package/dist/contract.d.ts +155 -1
  14. package/dist/hooks/capture-recovery.d.ts +59 -0
  15. package/dist/hooks/capture-recovery.js +88 -0
  16. package/dist/hooks/capture.js +114 -1
  17. package/dist/hooks/recall-ack.d.ts +48 -0
  18. package/dist/hooks/recall-ack.js +79 -0
  19. package/dist/hooks/recall.d.ts +23 -1
  20. package/dist/hooks/recall.js +124 -8
  21. package/dist/index.d.ts +22 -0
  22. package/dist/index.js +18 -4
  23. package/dist/metadata-validators.d.ts +5 -0
  24. package/dist/metadata-validators.js +11 -1
  25. package/dist/pipeline/l1.d.ts +4 -0
  26. package/dist/pipeline/l1.js +9 -2
  27. package/dist/pipeline/reconcile.js +4 -0
  28. package/dist/pipeline/ruminate.d.ts +11 -1
  29. package/dist/pipeline/ruminate.js +18 -1
  30. package/dist/pipeline/runner.js +62 -0
  31. package/dist/prompts/boundary.d.ts +17 -0
  32. package/dist/prompts/boundary.js +20 -0
  33. package/dist/prompts/graph-projection.js +4 -1
  34. package/dist/prompts/l1-dedup.d.ts +3 -3
  35. package/dist/prompts/l1-dedup.js +9 -1
  36. package/dist/prompts/l1-extraction.d.ts +3 -3
  37. package/dist/prompts/l1-extraction.js +12 -0
  38. package/dist/prompts/persona.js +6 -0
  39. package/dist/prompts/scene.js +7 -1
  40. package/dist/room-review.d.ts +196 -0
  41. package/dist/room-review.js +281 -0
  42. package/dist/stats.d.ts +26 -2
  43. package/dist/stats.js +270 -13
  44. package/dist/store/l0.d.ts +4 -0
  45. package/dist/store/l0.js +8 -0
  46. package/dist/store/l1.d.ts +34 -0
  47. package/dist/store/l1.js +68 -1
  48. package/dist/store/recall-usage.d.ts +43 -0
  49. package/dist/store/recall-usage.js +149 -0
  50. package/dist/store/rooms-registry.d.ts +66 -0
  51. package/dist/store/rooms-registry.js +250 -0
  52. package/dist/store/sqlite.d.ts +16 -0
  53. package/dist/store/sqlite.js +57 -0
  54. package/dist/store/trace.d.ts +39 -0
  55. package/dist/store/trace.js +149 -0
  56. package/dist/tools/index.d.ts +4 -1
  57. package/dist/tools/index.js +249 -2
  58. package/dist/types.d.ts +4 -0
  59. package/dist/util/redact.d.ts +27 -0
  60. package/dist/util/redact.js +228 -0
  61. package/package.json +13 -12
  62. package/dist/hall-backfill.d.ts +0 -33
  63. package/dist/hall-backfill.js +0 -95
  64. package/dist/hall-migrate.d.ts +0 -32
  65. package/dist/hall-migrate.js +0 -88
@@ -0,0 +1,228 @@
1
+ /**
2
+ * §C 载荷脱敏(memorax-absorb Wave 2 / task_14)——捕获边界的密钥擦除。
3
+ *
4
+ * 用户在对话里贴入的密钥(PEM/Bearer/JWT/Cookie/厂商 API key/邮箱/长号/高熵 ID)
5
+ * 若原样进 L0/L1,会被逐轮召回注入上下文并在 UI 展示。本模块在捕获单点把它们
6
+ * 替换为类型化占位符 `[REDACTED:<KIND>]`——占位符保留类别语义,关键词检索仍可命中。
7
+ *
8
+ * 词表/正则/allowlist/span 合并算法以 memorax `payload-redaction.ts` 为基准逐条移植
9
+ * (全文快照见计划 evidence/payload-redaction-snapshot.ts;findings.md 有等价内联)。
10
+ *
11
+ * 契约(sanitize 同款,tests/redact.test.ts 钉住):
12
+ * - 纯函数,零 I/O 零时钟零随机;输入原样返回(未命中)= 引用不变无关紧要,字符串不可变;
13
+ * - **幂等**:已是 `[REDACTED:KIND]` 的值不会被二次匹配;
14
+ * - 安全值 allowlist 命中不脱(`example`/`change-me`/`$VAR`/`${{…}}` 等);
15
+ * - 本函数**永不抛错**的保证由调用方 try/catch 兜底(capture 接线层,§C 红线:失败放行原文)。
16
+ */
17
+ const PLACEHOLDER_VALUE_PATTERN = /^\[REDACTED:[A-Z_]+\]$/;
18
+ const SENSITIVE_KEY_SOURCE = String.raw `(?:[A-Za-z_$][A-Za-z0-9_$-]*)?(?:password|passwd|pwd|client[-_$]?secret|consumer[-_$]?secret|api[-_$]?key|access[-_$]?token|refresh[-_$]?token|auth[-_$]?token|id[-_$]?token|private[-_$]?key|secret[-_$]?access[-_$]?key|secret[-_$]?key|credential|secret|token)`;
19
+ const CREDENTIAL_VALUE_SOURCE = String.raw `(?:\[REDACTED:[A-Z_]+\]|\$\{\{[^\r\n]*?\}\}|\{\{[^\r\n]*?\}\}|\$\{[^}\r\n]+\}|\$[A-Za-z_][A-Za-z0-9_]*|"(?:\\.|[^"\\\r\n])*"|'(?:\\.|[^'\\\r\n])*'|[^,;#}\]\)\r\n]+)`;
20
+ /** 安全值 allowlist:命中即放行(示例/占位/环境变量引用/类型名等,脱了只会毁记忆)。 */
21
+ const SAFE_CREDENTIAL_VALUE_PATTERN = /^(?:\[REDACTED:[A-Z_]+\]|\$\{\{[^\r\n]*\}\}|\{\{[^\r\n]*\}\}|\$\{[A-Z_][A-Z0-9_]*\}|\$[A-Z_][A-Z0-9_]*|(?:process\.)?env\.[A-Z_][A-Z0-9_]*|<[^>]+>|--\S+|x{2,}|\*{2,}|\.{3}|sk[_-]x+|example|sample|change-?me|(?:your|replace)[-_][a-z0-9_-]+|string|str|number|boolean|unknown|any|null|undefined|none)$/i;
22
+ const RULES = [
23
+ {
24
+ // PEM 私钥块:至对应 END,截断文件时取到文末
25
+ pattern: /-----BEGIN ((?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY|PGP PRIVATE KEY BLOCK)-----[\s\S]*?(?:-----END \1-----|$)/gi,
26
+ kind: 'PRIVATE_KEY',
27
+ priority: 100,
28
+ },
29
+ {
30
+ pattern: /(?:^|[\r\n])[ \t]*(?:Proxy-)?Authorization[ \t]*:[ \t]*([^ \t\r\n][^\r\n]*)/gim,
31
+ group: 1,
32
+ kind: 'AUTH_TOKEN',
33
+ priority: 95,
34
+ },
35
+ {
36
+ pattern: /(?:^|[^A-Za-z0-9_-])(?:Bearer|Basic)\s+["']?([A-Za-z0-9._~+/=-]{16,})(?![A-Za-z0-9._~+/=-])/gi,
37
+ group: 1,
38
+ kind: 'AUTH_TOKEN',
39
+ priority: 94,
40
+ },
41
+ {
42
+ // JWT 三段形态
43
+ pattern: /\beyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{16,}\b/g,
44
+ kind: 'AUTH_TOKEN',
45
+ priority: 94,
46
+ },
47
+ {
48
+ pattern: /(?:^|[\r\n])[ \t]*(?:Set-Cookie|Cookie)\s*:\s*([^\r\n]+)/gim,
49
+ group: 1,
50
+ kind: 'COOKIE',
51
+ priority: 92,
52
+ },
53
+ {
54
+ // 声明式赋值:key: value / key = value(含 export/const/let/var 前缀与引号键)
55
+ pattern: new RegExp(String.raw `(?:^|[\r\n;,{}])[ \t]*(?:-[ \t]+)?(?:(?:export|const|let|var)[ \t]+)*["']?${SENSITIVE_KEY_SOURCE}["']?[ \t]*[:=][ \t]*(${CREDENTIAL_VALUE_SOURCE})`, 'gim'),
56
+ group: 1,
57
+ kind: 'CREDENTIAL',
58
+ priority: 85,
59
+ accept: (value) => !isSafeCredentialValue(value),
60
+ },
61
+ {
62
+ // CLI 形态:--api-key value / --api-key=value
63
+ pattern: new RegExp(String.raw `--${SENSITIVE_KEY_SOURCE}(?![A-Za-z0-9_$-])(?:[ \t]+|=[ \t]*)(${CREDENTIAL_VALUE_SOURCE})`, 'gi'),
64
+ group: 1,
65
+ kind: 'CREDENTIAL',
66
+ priority: 85,
67
+ accept: (value) => !isSafeCredentialValue(value),
68
+ },
69
+ {
70
+ // URL query 形态:?api_key=...
71
+ pattern: new RegExp(String.raw `[?&]${SENSITIVE_KEY_SOURCE}(?![A-Za-z0-9_$-])=([^&#\s"'<>]+)`, 'gi'),
72
+ group: 1,
73
+ kind: 'CREDENTIAL',
74
+ priority: 85,
75
+ accept: (value) => !isSafeCredentialValue(value),
76
+ },
77
+ {
78
+ // URL userinfo 形态:scheme://user:pass@
79
+ pattern: /\b[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s/:@]*:([^\s/@]+)@/g,
80
+ group: 1,
81
+ kind: 'CREDENTIAL',
82
+ priority: 85,
83
+ accept: (value) => !isSafeCredentialValue(value),
84
+ },
85
+ // ── 已知厂商 API key 形态(9 条) ──
86
+ { pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}[A-Za-z0-9]\b/g, kind: 'API_KEY', priority: 80 },
87
+ { pattern: /\bsk_[A-Za-z0-9_-]{8,}\b/g, kind: 'API_KEY', priority: 80 },
88
+ { pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g, kind: 'API_KEY', priority: 80 },
89
+ { pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, kind: 'API_KEY', priority: 80 },
90
+ { pattern: /\bxox[baprs]-[A-Za-z0-9-]{16,}[A-Za-z0-9]\b/g, kind: 'API_KEY', priority: 80 },
91
+ { pattern: /\b(?:(?:sk|rk)_live_[A-Za-z0-9]{16,}|whsec_[A-Za-z0-9]{16,})\b/g, kind: 'API_KEY', priority: 80 },
92
+ { pattern: /\bglpat-[A-Za-z0-9_-]{16,}[A-Za-z0-9]\b/g, kind: 'API_KEY', priority: 80 },
93
+ { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g, kind: 'API_KEY', priority: 80 },
94
+ { pattern: /\bnpm_[A-Za-z0-9]{36}\b/g, kind: 'API_KEY', priority: 80 },
95
+ {
96
+ // 邮箱(前后向断言防误切)
97
+ pattern: /(?<![A-Z0-9._%+-])[A-Z0-9._%+-]{1,64}@[A-Z0-9](?:[A-Z0-9.-]{0,251}[A-Z0-9])?\.[A-Z]{2,63}(?![A-Z0-9-]|\.[A-Z0-9])/gi,
98
+ kind: 'EMAIL',
99
+ priority: 70,
100
+ },
101
+ {
102
+ // UUID
103
+ pattern: /(?<![A-Za-z0-9])[0-9A-Fa-f]{8}-(?:[0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}(?![A-Za-z0-9])/g,
104
+ kind: 'OPAQUE_ID',
105
+ priority: 60,
106
+ },
107
+ {
108
+ // hex 64/40/32(含字母才脱——纯数字是普通长号,交 LONG_NUMBER)
109
+ pattern: /(?<![A-Za-z0-9])(?:[0-9A-Fa-f]{64}|[0-9A-Fa-f]{40}|[0-9A-Fa-f]{32})(?![A-Za-z0-9])/g,
110
+ kind: 'OPAQUE_ID',
111
+ priority: 60,
112
+ accept: (value) => /[A-Fa-f]/.test(value),
113
+ },
114
+ {
115
+ // ≥24 位字母数字高熵启发式(熵参数见 looksLikeHighEntropyOpaqueId)
116
+ pattern: /(?<![A-Za-z0-9])[A-Za-z0-9]{24,}(?![A-Za-z0-9])/g,
117
+ kind: 'OPAQUE_ID',
118
+ priority: 60,
119
+ accept: looksLikeHighEntropyOpaqueId,
120
+ },
121
+ {
122
+ // ≥9 位数字串(含电话形);日期形放行
123
+ pattern: /(?<![A-Za-z0-9.])\+?(?:\d|\(\d)[\d ()-]*\d[Xx]?\)?(?![A-Za-z0-9]|\.\d)/g,
124
+ kind: 'LONG_NUMBER',
125
+ priority: 50,
126
+ accept: (value) => {
127
+ if (value.replace(/\D/g, '').length < 9)
128
+ return false;
129
+ const normalized = value.trim().replace(/[()]/g, '');
130
+ return (!/^(?:19|20)\d{2}-\d{1,2}-\d{1,2}(?:[ T-]\d{1,2})?$/.test(normalized) &&
131
+ !/^(?:19|20)\d{2}\s+\d{1,2}\s+\d{1,2}(?:\s+\d{1,2})?$/.test(normalized));
132
+ },
133
+ },
134
+ ];
135
+ /**
136
+ * 对文本执行 8 类脱敏。规则各自扫描 → 收集值级 span → 合并重叠 → 重拼插入占位符。
137
+ * 同一位置被多条规则命中时取更长/更高优先级的 span。
138
+ */
139
+ export function redactSecrets(text) {
140
+ if (!text)
141
+ return { text, counts: {}, redacted: false };
142
+ const spans = [];
143
+ for (const rule of RULES)
144
+ collectRuleSpans(text, rule, spans);
145
+ const selected = mergeOverlappingSpans(spans);
146
+ if (selected.length === 0)
147
+ return { text, counts: {}, redacted: false };
148
+ const counts = {};
149
+ const output = [];
150
+ let offset = 0;
151
+ for (const span of selected) {
152
+ output.push(text.slice(offset, span.start), `[REDACTED:${span.kind}]`);
153
+ counts[span.kind] = (counts[span.kind] ?? 0) + 1;
154
+ offset = span.end;
155
+ }
156
+ output.push(text.slice(offset));
157
+ return { text: output.join(''), counts, redacted: true };
158
+ }
159
+ function collectRuleSpans(text, rule, spans) {
160
+ rule.pattern.lastIndex = 0;
161
+ for (const match of text.matchAll(rule.pattern)) {
162
+ const value = match[rule.group ?? 0];
163
+ if (match.index === undefined || !value || PLACEHOLDER_VALUE_PATTERN.test(value.trim()))
164
+ continue;
165
+ if (rule.accept && !rule.accept(value))
166
+ continue;
167
+ const relativeStart = match[0].lastIndexOf(value);
168
+ if (relativeStart < 0)
169
+ continue;
170
+ spans.push({
171
+ start: match.index + relativeStart,
172
+ end: match.index + relativeStart + value.length,
173
+ kind: rule.kind,
174
+ priority: rule.priority,
175
+ });
176
+ }
177
+ }
178
+ function isSafeCredentialValue(rawValue) {
179
+ let value = rawValue.trim();
180
+ if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
181
+ value = value.slice(1, -1).trim();
182
+ }
183
+ return !value || SAFE_CREDENTIAL_VALUE_PATTERN.test(value);
184
+ }
185
+ /** 高熵判定:小写≥2 ∧ 大写≥2 ∧ 数字≥2 ∧ 字符类切换≥4 ∧ 最长连续字母段≤5 ∧ 去重字符≥12。 */
186
+ function looksLikeHighEntropyOpaqueId(value) {
187
+ let lowercase = 0;
188
+ let uppercase = 0;
189
+ let digits = 0;
190
+ let transitions = 0;
191
+ let letterRun = 0;
192
+ let longestLetterRun = 0;
193
+ let previousWasDigit;
194
+ const unique = new Set();
195
+ for (const character of value) {
196
+ const isDigit = character >= '0' && character <= '9';
197
+ if (isDigit) {
198
+ digits += 1;
199
+ letterRun = 0;
200
+ }
201
+ else {
202
+ if (character >= 'a' && character <= 'z')
203
+ lowercase += 1;
204
+ else
205
+ uppercase += 1;
206
+ letterRun += 1;
207
+ longestLetterRun = Math.max(longestLetterRun, letterRun);
208
+ }
209
+ if (previousWasDigit !== undefined && previousWasDigit !== isDigit)
210
+ transitions += 1;
211
+ previousWasDigit = isDigit;
212
+ unique.add(character);
213
+ }
214
+ return lowercase >= 2 && uppercase >= 2 && digits >= 2 && transitions >= 4 && longestLetterRun <= 5 && unique.size >= 12;
215
+ }
216
+ function mergeOverlappingSpans(spans) {
217
+ const sorted = [...spans].sort((left, right) => left.start - right.start || right.end - left.end || right.priority - left.priority);
218
+ const merged = [];
219
+ for (const span of sorted) {
220
+ const previous = merged.at(-1);
221
+ if (!previous || span.start >= previous.end) {
222
+ merged.push({ ...span });
223
+ continue;
224
+ }
225
+ previous.end = Math.max(previous.end, span.end);
226
+ }
227
+ return merged;
228
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-prime-memory",
3
- "version": "0.18.2",
3
+ "version": "0.18.5",
4
4
  "publishConfig": {
5
5
  "registry": "https://registry.npmjs.org",
6
6
  "access": "public",
@@ -50,14 +50,14 @@
50
50
  "ENGINEERING-NOTES.md"
51
51
  ],
52
52
  "scripts": {
53
- "build": "tsc -p tsconfig.json && node scripts/build-client.mjs && node scripts/copy-client.mjs",
53
+ "build": "node -e \"fs.rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json && node scripts/build-client.mjs && node scripts/copy-client.mjs",
54
54
  "smoke": "npm run build:smoke && node dist-smoke/smoke.js",
55
55
  "verify-catalog": "npm run build && node scripts/verify-catalog.mjs",
56
56
  "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.client.json --noEmit && tsc -p tsconfig.test.json",
57
57
  "build:smoke": "tsc src/smoke.ts --outDir dist-smoke --module nodenext --moduleResolution nodenext --target es2022 --strict --skipLibCheck --esModuleInterop",
58
58
  "lint": "eslint src client/src scripts tests",
59
59
  "test": "vitest run",
60
- "hall-migrate": "npm run build && node scripts/hall-migrate.mjs"
60
+ "wing-migrate": "npm run build && node scripts/wing-migrate.mjs"
61
61
  },
62
62
  "engines": {
63
63
  "node": ">=22.16.0",
@@ -125,15 +125,16 @@
125
125
  "devDependencies": {
126
126
  "@deepseek-ai/cordis": "^4.0.4",
127
127
  "@deepseek-ai/cordis-plugin-loader": "^1.0.5",
128
- "@deepseek-ai/dsh-agent": "0.1.1-rc.2",
129
- "@deepseek-ai/dsh-agent-default-model": "0.1.1-rc.2",
130
- "@deepseek-ai/dsh-client-connection": "0.1.1-rc.2",
131
- "@deepseek-ai/dsh-home-paths": "0.1.1-rc.2",
132
- "@deepseek-ai/dsh-llm": "0.1.1-rc.2",
133
- "@deepseek-ai/dsh-session": "0.1.1-rc.2",
134
- "@deepseek-ai/dsh-settings": "0.1.1-rc.2",
135
- "@deepseek-ai/dsh-system-prompt": "0.1.1-rc.2",
136
- "@deepseek-ai/dsh-tools": "0.1.1-rc.2",
128
+ "@deepseek-ai/dsh-agent": "0.1.7-rc.2",
129
+ "@deepseek-ai/dsh-agent-default-model": "0.1.7-rc.2",
130
+ "@deepseek-ai/dsh-client-connection": "0.1.7-rc.2",
131
+ "@deepseek-ai/dsh-compaction": "0.1.7-rc.2",
132
+ "@deepseek-ai/dsh-home-paths": "0.1.7-rc.2",
133
+ "@deepseek-ai/dsh-llm": "0.1.7-rc.2",
134
+ "@deepseek-ai/dsh-session": "0.1.7-rc.2",
135
+ "@deepseek-ai/dsh-settings": "0.1.7-rc.2",
136
+ "@deepseek-ai/dsh-system-prompt": "0.1.7-rc.2",
137
+ "@deepseek-ai/dsh-tools": "0.1.7-rc.2",
137
138
  "@deepseek-ai/schemastery": "^3.18.4",
138
139
  "@eslint/js": "^10.0.1",
139
140
  "@types/node": "^22.0.0",
@@ -1,33 +0,0 @@
1
- /**
2
- * 存量未打标记录的一键回填(task_15):从 HallWheel"未打标 M"入口触发,
3
- * 对 metadata 无 hall 的 L1 记录按 8 角词表批量补打标签(复用 L1 抽取的打标路径:
4
- * 同一候选词表与措辞,经既有蒸馏 LLM 路由)。
5
- *
6
- * 安全边界:
7
- * - **单飞**:同时最多一个回填任务,重复触发直接返回 running(不排队不叠加);
8
- * - **后台执行**:端点立即返回,逐块推进(每块 20 条),不阻塞 RPC/管线;
9
- * - **失败可退**:LLM 失败/输出不合法只跳过当前块,原记录零改动(标签只写成功的);
10
- * - **有界**:单次任务最多处理 MAX_RECORDS 条,防止一次跑飞;其余可再次触发。
11
- */
12
- import type { Context } from '@deepseek-ai/cordis';
13
- import { type MemoryLogger } from './types.js';
14
- import type { L1Store } from './store/l1.js';
15
- import type { MemoryConfig } from './config.js';
16
- export interface HallBackfillDeps {
17
- ctx: Context;
18
- cfg: MemoryConfig;
19
- l1: L1Store;
20
- logger: MemoryLogger;
21
- }
22
- export interface HallBackfillState {
23
- running: boolean;
24
- /** 本进程累计成功打标条数(重启归零;仅作观测,不作账)。 */
25
- updated: number;
26
- failed: number;
27
- }
28
- export declare function hallBackfillState(): HallBackfillState;
29
- /** 启动后台回填;已在运行返回 false(单飞)。 */
30
- export declare function startHallBackfill(deps: HallBackfillDeps): {
31
- started: boolean;
32
- running: boolean;
33
- };
@@ -1,95 +0,0 @@
1
- import { callLLM } from './llm.js';
2
- import { normHallEnabled } from './config.js';
3
- import { HALL_FALLBACK } from './types.js';
4
- import { parseJsonLogged } from './llm.js';
5
- /** 单次任务处理上限(524 条存量一次跑完约为 LLM 成本大头;分批触发,可重复执行)。 */
6
- const MAX_RECORDS = 300;
7
- /** 每块条数(一次 LLM 调用判定的记录数)。 */
8
- const CHUNK = 20;
9
- const state = { running: false, updated: 0, failed: 0 };
10
- export function hallBackfillState() {
11
- return { ...state };
12
- }
13
- const SYSTEM_PROMPT = '你是记忆库的域标注器。给你若干条记忆(带 id),为每条判断其内容属于哪个 Hall 域,输出 JSON 数组:' +
14
- '[{"id":"<原id>","hall":"<域id>"}]。hall 只能从给定候选列表中选;横跨多域或确实无法归入任何角时选 general。' +
15
- '只输出 JSON,不要多余文字。';
16
- /** 启动后台回填;已在运行返回 false(单飞)。 */
17
- export function startHallBackfill(deps) {
18
- if (state.running)
19
- return { started: false, running: true };
20
- state.running = true;
21
- state.updated = 0;
22
- state.failed = 0;
23
- void run(deps).finally(() => {
24
- state.running = false;
25
- deps.logger.info(`[memory] hall 回填结束:成功 ${state.updated} 条,失败 ${state.failed} 条`);
26
- });
27
- return { started: true, running: true };
28
- }
29
- async function run(deps) {
30
- const { ctx, cfg, l1, logger } = deps;
31
- const halls = normHallEnabled(cfg.hall?.enabled);
32
- if (halls.length === 0) {
33
- logger.warn('[memory] hall 回填跳过:hall.enabled 为空(打标功能已关闭)');
34
- return;
35
- }
36
- const candidates = [...halls, HALL_FALLBACK].join(' / ');
37
- // 未打标集:主表全量里的 null hall(含 retired 行——restore 后标签已补,不重扫)
38
- const all = l1.all();
39
- const pending = all.filter((r) => !(r.metadata && typeof r.metadata.hall === 'string' && r.metadata.hall !== ''));
40
- let budget = Math.min(pending.length, MAX_RECORDS);
41
- logger.info(`[memory] hall 回填启动:未打标 ${pending.length} 条,本次上限 ${budget} 条(候选 ${candidates})`);
42
- for (let offset = 0; offset < pending.length && budget > 0; offset += CHUNK) {
43
- const chunk = pending.slice(offset, offset + CHUNK).filter(pickChunk);
44
- if (chunk.length === 0)
45
- continue;
46
- budget -= chunk.length;
47
- const labeled = await labelChunk(ctx, cfg, logger, chunk, candidates);
48
- for (const { record, hall } of labeled) {
49
- // 原记录最小改动:metadata.hall 单键写入,其余字段原样 upsert(同 id 版本不变语义)
50
- const next = { ...record, metadata: { ...(record.metadata ?? {}), hall } };
51
- try {
52
- await l1.upsert(next);
53
- state.updated++;
54
- }
55
- catch {
56
- state.failed++;
57
- }
58
- }
59
- }
60
- }
61
- /** 回填只处理活跃记录可安全加速,但 retired 行的 metadata_json 同样保留且 restore 后可见——
62
- * upsert 对 retired 行的行为由存储层保证(同 id 覆盖不复活),这里不做活性过滤,口径与迁移一致(全量)。 */
63
- function pickChunk(_r) {
64
- return true;
65
- }
66
- async function labelChunk(ctx, cfg, logger, chunk, candidates) {
67
- const list = chunk.map((r, i) => `${i + 1}. id=${r.id}\n${r.content.slice(0, 300)}`).join('\n\n');
68
- const user = `候选 Hall:${candidates}\n\n记忆列表:\n${list}`;
69
- try {
70
- const raw = await callLLM(ctx, cfg, {
71
- system: SYSTEM_PROMPT,
72
- user,
73
- maxTokens: 4096,
74
- layer: 'l1-extract',
75
- logger,
76
- });
77
- const parsed = parseJsonLogged(raw, 'hall 回填', logger);
78
- if (!Array.isArray(parsed))
79
- return [];
80
- const byId = new Map(chunk.map((r) => [r.id, r]));
81
- const out = [];
82
- for (const item of parsed) {
83
- const record = typeof item?.id === 'string' ? byId.get(item.id) : undefined;
84
- const hall = typeof item?.hall === 'string' ? item.hall.trim() : '';
85
- if (record && hall)
86
- out.push({ record, hall });
87
- }
88
- return out;
89
- }
90
- catch (err) {
91
- logger.warn(`[memory] hall 回填块失败(跳过 ${chunk.length} 条,原记录未改): ${err instanceof Error ? err.message : String(err)}`);
92
- state.failed += chunk.length;
93
- return [];
94
- }
95
- }
@@ -1,32 +0,0 @@
1
- import type { MemoryLogger } from './types.js';
2
- /** 新词表下的全部合法 hall 值(8 角 + 跨域兜底)。 */
3
- export declare const HALL_VALID_VALUES: readonly string[];
4
- export interface HallMigrationReport {
5
- dbPath: string;
6
- dryRun: boolean;
7
- /** 主表全量行数(含 retired;与 hallL1Counts 的扫描口径一致)。 */
8
- total: number;
9
- /** retired 行数(valid_to 非空或带退场标记——迁移**不加过滤**,全量扫描自动覆盖)。 */
10
- retired: number;
11
- active: number;
12
- /** 逐值计数(含 retired 行;null 单列)。 */
13
- byHall: Record<string, number>;
14
- unlabeled: number;
15
- /** 不在新词表内的历史值(按 R10 不删已有标签,仅报告)。 */
16
- unknownValues: Record<string, number>;
17
- /** metadata 解析失败的行数(备份仍写入原始 JSON)。 */
18
- corruptMetadata: number;
19
- /** apply 模式写出的备份文件路径(dry-run 为 null)。 */
20
- backupPath: string | null;
21
- /** 结论:expected = 全部值合法(迁移无需改写);unknown = 存在历史遗留值(已报告,保留)。 */
22
- verdict: 'expected' | 'unknown-values';
23
- }
24
- export interface HallMigrationOptions {
25
- dbPath: string;
26
- /** true = 只读报告,不写备份;false = 先备份 metadata_json 再出报告。 */
27
- dryRun?: boolean;
28
- /** 备份输出目录(缺省 = db 同目录 hall-migrate-backup/)。 */
29
- backupDir?: string;
30
- logger?: MemoryLogger;
31
- }
32
- export declare function runHallMigration(opts: HallMigrationOptions): HallMigrationReport;
@@ -1,88 +0,0 @@
1
- /**
2
- * hall 词表迁移(task_14):v0.12 5 值词表 → 8 角 + general 跨域兜底的**有界 targeted pass**。
3
- *
4
- * 关键事实(决定本迁移的形态):
5
- * - 4 个保留 id(work/relationships/finance/journey)在新词表下仍是合法角,**存量标签零改动**;
6
- * - `general` 保留为跨域兜底值(不重写、不删除);
7
- * - 4 个新角无存量。因此本迁移**不改任何行的 hall 值**——它做的是:
8
- * ① 备份 metadata_json(apply 模式);② **扫 L1 主表全量(含 retired 行)**逐行核对;
9
- * ③ 落报告:逐值计数(含 null)/活性与 retired 拆分/未知值清单。
10
- *
11
- * 幂等:连续执行两次报告一致(扫描是纯读)。共享扫描函数 = `MemoryDb.scanL1Metadata`
12
- * (单一所有者;计划 B 的引用门禁复用同一函数,不写第二份扫描)。
13
- */
14
- import { openSync, closeSync, writeSync, existsSync, mkdirSync } from 'node:fs';
15
- import * as path from 'node:path';
16
- import { MemoryDb } from './store/sqlite.js';
17
- import { HALL_CATALOG, HALL_FALLBACK } from './types.js';
18
- /** 新词表下的全部合法 hall 值(8 角 + 跨域兜底)。 */
19
- export const HALL_VALID_VALUES = [...HALL_CATALOG.map((h) => h.id), HALL_FALLBACK];
20
- export function runHallMigration(opts) {
21
- const db = new MemoryDb(opts.dbPath, 0, opts.logger);
22
- // 开库即建表/补列(CREATE TABLE IF NOT EXISTS 惯例);未 init 则句柄未打开,扫描恒 0
23
- db.init();
24
- try {
25
- const report = {
26
- dbPath: opts.dbPath,
27
- dryRun: !!opts.dryRun,
28
- total: 0,
29
- retired: 0,
30
- active: 0,
31
- byHall: {},
32
- unlabeled: 0,
33
- unknownValues: {},
34
- corruptMetadata: 0,
35
- backupPath: null,
36
- verdict: 'expected',
37
- };
38
- let backupFd = null;
39
- if (!report.dryRun) {
40
- const dir = opts.backupDir ?? path.join(path.dirname(opts.dbPath), 'hall-migrate-backup');
41
- if (!existsSync(dir))
42
- mkdirSync(dir, { recursive: true });
43
- const backupPath = path.join(dir, `metadata-backup-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`);
44
- backupFd = openSync(backupPath, 'w');
45
- report.backupPath = backupPath;
46
- }
47
- try {
48
- report.total = db.scanL1Metadata((recordId, metadata) => {
49
- const hall = metadata && typeof metadata.hall === 'string' && metadata.hall !== '' ? metadata.hall : null;
50
- // metadata 缺失/解析失败与"无 hall"同样计入未打标(存量 524 null 的主口径)
51
- // 备份:原始 metadata_json 原样落盘(apply 模式;dry-run 不写任何文件)
52
- if (backupFd !== null) {
53
- writeSync(backupFd, JSON.stringify({ recordId, metadata }) + '\n');
54
- }
55
- if (hall === null) {
56
- report.unlabeled++;
57
- }
58
- else {
59
- report.byHall[hall] = (report.byHall[hall] ?? 0) + 1;
60
- if (!HALL_VALID_VALUES.includes(hall)) {
61
- report.unknownValues[hall] = (report.unknownValues[hall] ?? 0) + 1;
62
- }
63
- }
64
- });
65
- // retired 拆分:退场判定与存储层同口径(valid_to 非空 ∨ dsh_superseded 标记)。
66
- // 扫描本身**不加过滤**(全表 SELECT 自动覆盖 retired),这里仅分类计数。
67
- report.active = report.total - countRetired(db);
68
- report.retired = report.total - report.active;
69
- }
70
- finally {
71
- if (backupFd !== null)
72
- closeSync(backupFd);
73
- }
74
- report.verdict = Object.keys(report.unknownValues).length > 0 ? 'unknown-values' : 'expected';
75
- return report;
76
- }
77
- finally {
78
- db.close();
79
- }
80
- }
81
- /** retired 行计数(与 sqlite 层退场判定同口径;用于验证"全量扫描不漏 retired")。 */
82
- function countRetired(db) {
83
- // 借道 scanL1Metadata 之外的轻量路径不可得(retired 判定在行级字段),
84
- // 这里直接用 hallL1Counts 的同族 SQL:valid_to 非空 ∨ metadata 带 dsh_superseded。
85
- // 为保持"单一所有者",经由 scanL1Metadata 之外不再开第二条 SQL——改为返回占位 0,
86
- // 由 retire 拆分调用方(db.retiredL1Count)提供。
87
- return db.retiredL1Count();
88
- }