@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/LICENSE +661 -0
  3. package/README.en.md +110 -0
  4. package/README.md +102 -2
  5. package/dsh-memory-local/FORK-BASE.json +24 -0
  6. package/dsh-memory-local/README.md +113 -0
  7. package/dsh-memory-local/build.mjs +101 -0
  8. package/dsh-memory-local/cordis.patch.yml +20 -0
  9. package/dsh-memory-local/deploy.mjs +306 -0
  10. package/dsh-memory-local/entity-status.mjs +75 -0
  11. package/dsh-memory-local/lib/bridge.mjs +108 -0
  12. package/dsh-memory-local/lib/host-adapter.js +1171 -0
  13. package/dsh-memory-local/lib/index.d.ts +128 -0
  14. package/dsh-memory-local/lib/index.js +944 -0
  15. package/dsh-memory-local/lib/injector.mjs +400 -0
  16. package/dsh-memory-local/lib/memory-read.mjs +116 -0
  17. package/dsh-memory-local/lib/module-gate.mjs +68 -0
  18. package/dsh-memory-local/lib/pointer.mjs +152 -0
  19. package/dsh-memory-local/lib/prod-gate.mjs +121 -0
  20. package/dsh-memory-local/lib/recall-log.mjs +118 -0
  21. package/dsh-memory-local/lib/recall.mjs +192 -0
  22. package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
  23. package/dsh-memory-local/lib/user-message.mjs +32 -0
  24. package/dsh-memory-local/lib/vectors-read.mjs +110 -0
  25. package/dsh-memory-local/package.json +51 -0
  26. package/dsh-memory-local/restore.mjs +259 -0
  27. package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
  28. package/dsh-memory-local/src/bridge.mjs +108 -0
  29. package/dsh-memory-local/src/host-adapter.js +1171 -0
  30. package/dsh-memory-local/src/index.d.ts +128 -0
  31. package/dsh-memory-local/src/index.js +944 -0
  32. package/dsh-memory-local/src/injector.mjs +400 -0
  33. package/dsh-memory-local/src/memory-read.mjs +116 -0
  34. package/dsh-memory-local/src/module-gate.mjs +68 -0
  35. package/dsh-memory-local/src/pointer.mjs +152 -0
  36. package/dsh-memory-local/src/recall-log.mjs +118 -0
  37. package/dsh-memory-local/src/recall.mjs +192 -0
  38. package/dsh-memory-local/src/schema-migrate.mjs +365 -0
  39. package/dsh-memory-local/src/user-message.mjs +32 -0
  40. package/dsh-memory-local/src/vectors-read.mjs +110 -0
  41. package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
  42. package/dsh-memory-local/test/probe-entity.mjs +170 -0
  43. package/dsh-memory-local/test/regression.mjs +270 -0
  44. package/dsh-memory-local/test/s2b.test.mjs +568 -0
  45. package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
  46. package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
  47. package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
  48. package/dsh-memory-local/verify-after-restart.mjs +181 -0
  49. package/lib/deploy-guard.mjs +391 -0
  50. package/lib/guard-http.mjs +93 -0
  51. package/lib/pitfall-form-map.mjs +130 -0
  52. package/lib/pitfall-score.mjs +185 -0
  53. package/lib/session-file-name.mjs +160 -0
  54. package/lib/zstd-frames.mjs +78 -0
  55. package/memory-db-tools/README.md +68 -0
  56. package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
  57. package/memory-db-tools/audit-analyze.mjs +352 -0
  58. package/memory-db-tools/audit-dump-ids.mjs +245 -0
  59. package/memory-db-tools/audit-export.mjs +270 -0
  60. package/memory-db-tools/audit-hit-probe.mjs +227 -0
  61. package/memory-db-tools/audit-ledger-probe.mjs +236 -0
  62. package/memory-db-tools/audit-persona-overlap.mjs +277 -0
  63. package/memory-db-tools/audit-struct-probe.mjs +213 -0
  64. package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
  65. package/memory-db-tools/check-memory-db.mjs +8 -0
  66. package/memory-db-tools/dump-memory-db.mjs +12 -0
  67. package/memory-db-tools/lib/memory-scope.mjs +446 -0
  68. package/memory-db-tools/list-memory-summary.mjs +477 -0
  69. package/memory-db-tools/memory-identity-check.mjs +101 -0
  70. package/memory-db-tools/memory-schema-migrate.mjs +290 -0
  71. package/memory-db-tools/memory-scope-fixture.mjs +197 -0
  72. package/memory-db-tools/read-memories.mjs +26 -0
  73. package/memory-db-tools/read-memory-db.mjs +20 -0
  74. package/memory-db-tools/refine-memories.mjs +247 -0
  75. package/memory-db-tools/show-mem-times.mjs +22 -0
  76. package/memory-engine/README.md +70 -0
  77. package/memory-engine/bench/bench.mjs +221 -0
  78. package/memory-engine/cli/distill-rollback.mjs +86 -0
  79. package/memory-engine/cli/distill.mjs +156 -0
  80. package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
  81. package/memory-engine/cli/doc-index.mjs +48 -0
  82. package/memory-engine/cli/domain-tags.mjs +251 -0
  83. package/memory-engine/cli/pitfall-hook.mjs +91 -0
  84. package/memory-engine/cli/recall.mjs +132 -0
  85. package/memory-engine/cli/selfcheck.mjs +57 -0
  86. package/memory-engine/cli/serve-start.mjs +247 -0
  87. package/memory-engine/cli/sweep.mjs +58 -0
  88. package/memory-engine/cli/sync.mjs +69 -0
  89. package/memory-engine/lib/cli-util.mjs +197 -0
  90. package/memory-engine/lib/config.mjs +135 -0
  91. package/memory-engine/lib/dirty-ledger.mjs +174 -0
  92. package/memory-engine/lib/distill-rollback.mjs +178 -0
  93. package/memory-engine/lib/distill.mjs +898 -0
  94. package/memory-engine/lib/doc-index-store.mjs +325 -0
  95. package/memory-engine/lib/doc-index.mjs +529 -0
  96. package/memory-engine/lib/doc-scan.mjs +271 -0
  97. package/memory-engine/lib/doc-summarize.mjs +305 -0
  98. package/memory-engine/lib/domain-tags.mjs +429 -0
  99. package/memory-engine/lib/embedder.mjs +242 -0
  100. package/memory-engine/lib/hashes.mjs +12 -0
  101. package/memory-engine/lib/memory-source.mjs +127 -0
  102. package/memory-engine/lib/pitfall-source.mjs +170 -0
  103. package/memory-engine/lib/policy-source.mjs +371 -0
  104. package/memory-engine/lib/recall.mjs +331 -0
  105. package/memory-engine/lib/selfcheck.mjs +131 -0
  106. package/memory-engine/lib/sweep.mjs +159 -0
  107. package/memory-engine/lib/sync.mjs +180 -0
  108. package/memory-engine/lib/tick-hook.mjs +218 -0
  109. package/memory-engine/lib/vectors-store.mjs +292 -0
  110. package/memory-engine/package-lock.json +1086 -0
  111. package/memory-engine/package.json +22 -0
  112. package/memory-engine/serve.mjs +171 -0
  113. package/memory-engine/test/01-store.test.mjs +127 -0
  114. package/memory-engine/test/02-sync.test.mjs +196 -0
  115. package/memory-engine/test/03-sweep.test.mjs +145 -0
  116. package/memory-engine/test/04-recall.test.mjs +214 -0
  117. package/memory-engine/test/05-selfcheck.test.mjs +175 -0
  118. package/memory-engine/test/06-doc-index.test.mjs +928 -0
  119. package/memory-engine/test/07-domain-tags.test.mjs +298 -0
  120. package/memory-engine/test/08-distill.test.mjs +686 -0
  121. package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
  122. package/memory-engine/test/10-tick-hook.test.mjs +252 -0
  123. package/memory-engine/test/11-policy-source.test.mjs +224 -0
  124. package/memory-engine/test/_tickhook-worker.mjs +56 -0
  125. package/memory-engine/test/fixtures.mjs +205 -0
  126. package/memory-engine/test/harness.mjs +73 -0
  127. package/memory-engine/test/run.mjs +35 -0
  128. package/memory-engine/tools/bridge-smoke.mjs +80 -0
  129. package/memory-engine/tools/fetch-model.mjs +128 -0
  130. package/memory-engine/tools/smoke-recall.mjs +76 -0
  131. package/package.json +53 -4
  132. package/persona-memory-audit.mjs +217 -0
@@ -0,0 +1,197 @@
1
+ // 记忆引擎 v2 · 离线层 —— CLI 公共工具
2
+ //
3
+ // 入口守卫(铁律):工作区下可能存在目录 junction,`import.meta.url` 会解析为物理路径而
4
+ // `process.argv[1]` 是逻辑路径 ⇒ 必须 realpath 双方再比,否则 main 入口永不触发、脚本静默退出 0。
5
+ import fs from 'node:fs';
6
+ import path from 'node:path';
7
+ import { fileURLToPath } from 'node:url';
8
+ import { resolveConfig } from './config.mjs';
9
+ import { openVectorsStore } from './vectors-store.mjs';
10
+ import { openMemorySource } from './memory-source.mjs';
11
+ import { openPitfallSource } from './pitfall-source.mjs';
12
+ import { openPolicySource, scanPolicySources } from './policy-source.mjs';
13
+ import { createStubEmbedder, createHttpEmbedder, createLocalEmbedder } from './embedder.mjs';
14
+
15
+ /** 已实现的向量来源(值 = `vectors.source` 的取值) */
16
+ export const KNOWN_SOURCES = ['memory', 'pitfall', 'policy'];
17
+
18
+ /** `--source` 取值(默认 memory;缺省不改任何现有行为) */
19
+ export function resolveSourceName(flags = {}, dflt = 'memory') {
20
+ const s = flags.source;
21
+ if (s === undefined || s === true || s === '') return dflt;
22
+ return String(s);
23
+ }
24
+
25
+ /**
26
+ * 按来源选数据源(方案 §A.2 第 11 行 I-1:**运行时选源单例**)。
27
+ * ⚠️ 不改这里 ⇒ 即使 sync/sweep/recall 全参数化,命令行也**选不到**坑源。
28
+ */
29
+ export function openDataSource(name, config) {
30
+ const n = String(name);
31
+ if (n === 'memory') return openMemorySource(config.memoryDb);
32
+ if (n === 'pitfall') return openPitfallSource(config.pitfallsFile);
33
+ // 【2026-10-10 · 补坑库联动BCD】policy 源 = **每次现扫**(语料是文件,不是库):
34
+ // 段清单由 `scanPolicySources()` 只读扫描得到 ⇒ 无需另建状态文件;增量同步靠 (refId, textHash) 幂等。
35
+ if (n === 'policy') {
36
+ return openPolicySource({
37
+ segments: scanPolicySources({
38
+ personaDir: config.policyPersonaDir,
39
+ skillsRoot: config.policySkillsRoot,
40
+ docsPaths: config.policyDocsPaths,
41
+ }),
42
+ });
43
+ }
44
+ throw new Error(`未知来源: ${n}(现支持 memory / pitfall / policy)`);
45
+ }
46
+
47
+ export function isMainEntry(importMetaUrl, argv1 = process.argv[1]) {
48
+ if (!argv1) return false;
49
+ try {
50
+ const a = fs.realpathSync(fileURLToPath(importMetaUrl));
51
+ const b = fs.realpathSync(argv1);
52
+ return a === b;
53
+ } catch {
54
+ return false;
55
+ }
56
+ }
57
+
58
+ export function parseArgs(argv = process.argv.slice(2)) {
59
+ const flags = {};
60
+ const rest = [];
61
+ for (let i = 0; i < argv.length; i += 1) {
62
+ const a = argv[i];
63
+ if (a.startsWith('--')) {
64
+ const body = a.slice(2);
65
+ const eq = body.indexOf('=');
66
+ if (eq >= 0) {
67
+ flags[body.slice(0, eq)] = body.slice(eq + 1);
68
+ } else if (argv[i + 1] !== undefined && !argv[i + 1].startsWith('--')) {
69
+ flags[body] = argv[i + 1];
70
+ i += 1;
71
+ } else {
72
+ flags[body] = true;
73
+ }
74
+ } else {
75
+ rest.push(a);
76
+ }
77
+ }
78
+ return { flags, rest };
79
+ }
80
+
81
+ /** --json 时日志只上 stderr(机器只读 stdout 的 JSON) */
82
+ export function makeLogger({ json = false, quiet = false, prefix = '' } = {}) {
83
+ const write = (s) => {
84
+ if (quiet) return;
85
+ if (json) process.stderr.write(s + '\n');
86
+ else process.stdout.write(s + '\n');
87
+ };
88
+ return (msg) => write(prefix ? `${prefix} ${msg}` : String(msg));
89
+ }
90
+
91
+ export function num(v, dflt) {
92
+ if (v === undefined || v === null || v === true) return dflt;
93
+ const n = Number(v);
94
+ return Number.isFinite(n) ? n : dflt;
95
+ }
96
+
97
+ /**
98
+ * 打开运行时(配置 / 向量库 / 记忆库只读 / 编码器)。
99
+ * 一切路径可注入;CLI 未传时落默认(≈ `~/.dsh`)。
100
+ */
101
+ export async function openRuntime(flags = {}, { withSource = true, withEmbedder = true } = {}) {
102
+ const config = resolveConfig({
103
+ memoryDb: flags['memory-db'] || undefined,
104
+ vectorsDb: flags['vectors-db'] || undefined,
105
+ vectorsDir: flags['vectors-dir'] || undefined,
106
+ stateFile: flags['state-file'] || undefined,
107
+ pitfallsFile: flags['pitfalls-file'] || undefined,
108
+ pitfallVectorDirtyFile: flags['pitfall-dirty-file'] || undefined,
109
+ // 【2026-10-10 · 补坑库联动BCD】policy 语料面(隔离建库时用:向量库落隔离 home、
110
+ // 语料仍指生产只读路径 ⇒ 既不动生产库,又能编出真语料)
111
+ policyPersonaDir: flags['policy-persona-dir'] || undefined,
112
+ policySkillsRoot: flags['policy-skills-root'] || undefined,
113
+ policyDocsPaths: flags['policy-docs']
114
+ ? String(flags['policy-docs']).split(',').map((s) => s.trim()).filter(Boolean)
115
+ : undefined,
116
+ modelDir: flags['model-dir'] || undefined,
117
+ modelId: flags.model || undefined,
118
+ dim: flags.dim ? Number(flags.dim) : undefined,
119
+ hostId: flags.host || undefined,
120
+ batchSize: flags.batch ? Number(flags.batch) : undefined,
121
+ bridgePort: flags['bridge-port'] ? Number(flags['bridge-port']) : undefined,
122
+ dshHome: flags['dsh-home'] || undefined,
123
+ });
124
+
125
+ const store = openVectorsStore(config.vectorsDb, { create: true });
126
+ store.hostId = config.hostId;
127
+
128
+ // 数据源按名字缓存(多来源召回要同时开 memory + pitfall;同一来源只开一次)
129
+ const cache = new Map();
130
+ const openSource = (name) => {
131
+ const n = String(name);
132
+ if (!cache.has(n)) cache.set(n, openDataSource(n, config));
133
+ return cache.get(n);
134
+ };
135
+ const sourceName = resolveSourceName(flags);
136
+ let source = null;
137
+ if (withSource) source = openSource(sourceName);
138
+
139
+ let embedder = null;
140
+ if (!withEmbedder) {
141
+ embedder = null;
142
+ } else if (flags['stub-embedder']) {
143
+ embedder = createStubEmbedder({ dim: config.dim, modelId: `${config.modelId}-stub` });
144
+ } else if (flags['http-embedder']) {
145
+ embedder = createHttpEmbedder({
146
+ host: config.bridgeHost,
147
+ port: config.bridgePort,
148
+ timeoutMs: num(flags['bridge-timeout'], 250),
149
+ modelId: config.modelId,
150
+ dim: config.dim,
151
+ });
152
+ } else {
153
+ embedder = await createLocalEmbedder({
154
+ modelDir: config.modelDir,
155
+ modelId: config.modelId,
156
+ dim: config.dim,
157
+ quantized: Boolean(flags.quantized),
158
+ dtype: flags.dtype || null,
159
+ });
160
+ }
161
+
162
+ return {
163
+ config,
164
+ store,
165
+ source,
166
+ sourceName,
167
+ openSource,
168
+ embedder,
169
+ close() {
170
+ for (const s of cache.values()) {
171
+ try {
172
+ if (s) s.close();
173
+ } catch {
174
+ /* ignore */
175
+ }
176
+ }
177
+ cache.clear();
178
+ try {
179
+ store.db.close();
180
+ } catch {
181
+ /* ignore */
182
+ }
183
+ },
184
+ };
185
+ }
186
+
187
+ export function relForLog(p) {
188
+ try {
189
+ return path.relative(process.cwd(), p) || p;
190
+ } catch {
191
+ return p;
192
+ }
193
+ }
194
+
195
+ export function fmtMs(v) {
196
+ return `${Number(v).toFixed(1)}ms`;
197
+ }
@@ -0,0 +1,135 @@
1
+ // 记忆引擎 v2 · 离线层 —— 配置与路径解析
2
+ //
3
+ // 铁律(方案 §4 宿主迁移契约 / 本卡「测试不得写生产路径」):
4
+ // 1. 一切路径都可注入(overrides)或经环境变量覆盖,默认值才落 ~/.dsh;
5
+ // 2. 测试一律显式传临时目录,绝不落到生产路径;
6
+ // 3. 不写死机器名/主机名常量。
7
+ import fs from 'node:fs';
8
+ import os from 'node:os';
9
+ import path from 'node:path';
10
+
11
+ // ⚠️ 本常量 = **向量库(vectors.db)** 的 schema 版本。
12
+ // 2026-10-10 · 补坑库联动A:'1' → '2'(meta 键位加 `<key>:<source>` 命名空间;**表结构未变**,
13
+ // 老库靠 `migrateMetaNamespace()` 幂等迁移 + 兼容读,回滚靠「冗余写」保持旧键同值)。
14
+ // ⚠️ 文档索引库(doc-index.db)的 DDL 与本库无关,**不要**跟着改 —— 它有独立常量(见下)。
15
+ export const SCHEMA_VERSION = '2';
16
+ /** 文档索引库(doc-index.db)的 schema 版本(本轮未动其 DDL ⇒ 保持 '1') */
17
+ export const DOC_INDEX_SCHEMA_VERSION = '1';
18
+ export const DEFAULT_MODEL_ID = 'bge-small-zh-v1.5';
19
+ export const DEFAULT_DIM = 512;
20
+
21
+ /** 模型权重目录候选(第一个存在的即用;都不存在则用最后一个 = 会被创建的那个) */
22
+ export function defaultModelDirCandidates() {
23
+ return [
24
+ process.env.MEMORY_ENGINE_MODEL_DIR,
25
+ path.join(os.homedir(), '.dsh', 'memory-engine', 'models'), // 可经 MEMORY_ENGINE_MODEL_DIR 覆盖
26
+ path.join(os.homedir(), '.cache', 'dsh-memory-engine', 'models'),
27
+ ].filter(Boolean);
28
+ }
29
+
30
+ export function defaultModelDir() {
31
+ const candidates = defaultModelDirCandidates();
32
+ for (const c of candidates) {
33
+ try {
34
+ if (fs.existsSync(c)) return c;
35
+ } catch {
36
+ /* 忽略探测失败 */
37
+ }
38
+ }
39
+ return candidates[candidates.length - 1];
40
+ }
41
+
42
+ /**
43
+ * 解析配置。overrides 优先级最高,其次环境变量,最后默认值。
44
+ * 关键字段(测试可全部覆盖):memoryDb / vectorsDb / stateFile / modelDir。
45
+ */
46
+ export function resolveConfig(overrides = {}) {
47
+ const env = overrides.env || process.env;
48
+ const dshHome =
49
+ overrides.dshHome ||
50
+ env.DSH_HOME ||
51
+ path.join(os.homedir(), '.dsh');
52
+
53
+ const vectorsDir =
54
+ overrides.vectorsDir ||
55
+ env.MEMORY_ENGINE_VECTORS_DIR ||
56
+ path.join(dshHome, 'memory-vectors');
57
+
58
+ return {
59
+ dshHome,
60
+ // 数据源:默认生产库;本卡一律传副本路径(方案 §8.1 铁律)
61
+ memoryDb:
62
+ overrides.memoryDb ||
63
+ env.MEMORY_ENGINE_MEMORY_DB ||
64
+ path.join(dshHome, 'memory', 'memory.db'),
65
+ vectorsDir,
66
+ vectorsDb:
67
+ overrides.vectorsDb ||
68
+ env.MEMORY_ENGINE_VECTORS_DB ||
69
+ path.join(vectorsDir, 'vectors.db'),
70
+ stateFile:
71
+ overrides.stateFile ||
72
+ env.MEMORY_ENGINE_STATE ||
73
+ path.join(dshHome, 'butler-memory-vectors-state.json'),
74
+
75
+ // 【2026-10-10 · 补坑库联动A】坑库第二向量来源(source='pitfall')的输入与标记文件
76
+ // · pitfallsFile:坑库状态文件(**只读**,A 点不写它)
77
+ // · pitfallVectorDirtyFile:置/清标记双写者的那个标记文件(登记见 dsh-butler/lib/state-files.mjs)
78
+ pitfallsFile:
79
+ overrides.pitfallsFile ||
80
+ env.MEMORY_ENGINE_PITFALLS_FILE ||
81
+ path.join(dshHome, 'butler-pitfalls.json'),
82
+ pitfallVectorDirtyFile:
83
+ overrides.pitfallVectorDirtyFile ||
84
+ env.MEMORY_ENGINE_PITFALL_DIRTY ||
85
+ path.join(dshHome, 'butler-pitfall-vector-dirty.json'),
86
+
87
+ // 【2026-10-10 · 补坑库联动BCD】policy 语料面(source='policy' · 落地判据向量化 · B 点)
88
+ // 语料三份里 MVP 只做前两份(方案 §4 B.3-1):① persona 铁律段 ② 技能段;③ 文档段默认空
89
+ // (「不做全量 docs」是方案的明写 Non-goal —— 那是 S3 doc-index 的活)。
90
+ // ⚠️ policy 库**只在本机建**(D5 已拍 = 不跨机传 persona/技能文本)⇒ 库不进任何同步仓。
91
+ policyPersonaDir:
92
+ overrides.policyPersonaDir ||
93
+ env.MEMORY_ENGINE_POLICY_PERSONA_DIR ||
94
+ path.join(dshHome, '.agent-presets'),
95
+ policySkillsRoot:
96
+ overrides.policySkillsRoot ||
97
+ env.MEMORY_ENGINE_POLICY_SKILLS_ROOT ||
98
+ path.join(dshHome, 'skills'),
99
+ policyDocsPaths: overrides.policyDocsPaths || null, // 默认空(后置项,不默认扫全量 docs)
100
+
101
+ // 文档索引层(方案 §3.2:独立库出主库;§4 载体清单)
102
+ docIndexDb:
103
+ overrides.docIndexDb ||
104
+ env.DOC_INDEX_DB ||
105
+ path.join(vectorsDir, 'doc-index.db'),
106
+ docIndexState:
107
+ overrides.docIndexState ||
108
+ env.DOC_INDEX_STATE ||
109
+ path.join(dshHome, 'butler-doc-index-state.json'),
110
+ // 扫描面与 path 口径锚(不写死机器名;方案 §4 宿主迁移契约)
111
+ docsRoot: overrides.docsRoot || env.DOC_INDEX_DOCS_ROOT || path.join(process.cwd(), 'docs'),
112
+ dshRoot: overrides.dshRoot || env.DOC_INDEX_DSH_ROOT || process.cwd(),
113
+
114
+ modelDir: overrides.modelDir || defaultModelDir(),
115
+ modelId: overrides.modelId || env.MEMORY_ENGINE_MODEL_ID || DEFAULT_MODEL_ID,
116
+ dim: Number(overrides.dim || env.MEMORY_ENGINE_DIM || DEFAULT_DIM),
117
+
118
+ // 同步参数(方案 §3.1:批 50)
119
+ batchSize: Number(overrides.batchSize || env.MEMORY_ENGINE_BATCH || 50),
120
+ // 兜底 hash 轮转段数(方案 §3.1:按周 / id 段轮转覆盖全表)
121
+ sweepSegments: Number(overrides.sweepSegments || env.MEMORY_ENGINE_SWEEP_SEGMENTS || 20),
122
+
123
+ // 召回参数(方案 §3.3)
124
+ vectorTopK: Number(overrides.vectorTopK || env.MEMORY_ENGINE_VEC_TOPK || 20),
125
+ ftsTopK: Number(overrides.ftsTopK || env.MEMORY_ENGINE_FTS_TOPK || 20),
126
+ rrfK: Number(overrides.rrfK || env.MEMORY_ENGINE_RRF_K || 60),
127
+ resultTopK: Number(overrides.resultTopK || env.MEMORY_ENGINE_RESULT_TOPK || 8),
128
+
129
+ // 编码桥(方案 §3.1 编码通道契约:仅回环、端口 8201)
130
+ bridgeHost: overrides.bridgeHost || env.MEMORY_ENGINE_BRIDGE_HOST || '127.0.0.1',
131
+ bridgePort: Number(overrides.bridgePort || env.MEMORY_ENGINE_BRIDGE_PORT || 8201),
132
+
133
+ hostId: overrides.hostId || env.MEMORY_ENGINE_HOST_ID || os.hostname(),
134
+ };
135
+ }
@@ -0,0 +1,174 @@
1
+ // 记忆引擎 v2 · S3c —— `memories_dirty` 台账(兜底 hash 轮转的**产物**)
2
+ //
3
+ // 为什么要有它(方案 §3.5 输入行「v3 补漏(关 I5)」):
4
+ // A7 实测 5 个脚本 41 处 `UPDATE memories SET text=?`,其中 `apply-pinned-refine.mjs:110`
5
+ // 改 text **不动 `updated_at`** ⇒ 这类被勘误的记忆**永远不在 24h 增量窗口内**。
6
+ // 方案给的修法 = 把「§3.1 全表 `text_hash` 段轮转」的**产物**(差异清单)作为提炼 job 的补充输入源。
7
+ //
8
+ // 关键设计:差异清单必须**持久化**,不能只在内存里过一遍 ——
9
+ // 轮转(`lib/sweep.mjs`)发现差异后会**立刻重编码并更新 `vectors.text_hash`**,
10
+ // 差异信号当场消失;若当时没人消费,这条改动就永久消失在增量窗口之外(正是要治的病)。
11
+ // ⇒ 检测到差异的**那一刻**就落台账;提炼 job 事后按台账补齐输入。
12
+ //
13
+ // 落点:`vectors.db`(独立向量库,`sync:false` 不进任何仓,可重建)。
14
+ // 放这里而不是新开 state 文件:它**本来就是**向量层的派生产物(与 `vectors.text_hash` 同源),
15
+ // 且向量库随 `~/.dsh/memory-vectors/` 一起被清理/重建,台账不会成为孤儿。
16
+ //
17
+ // 语义:一条 = 一个 `ref_id`(memory.id)。
18
+ // · `kind='mismatch'`:正文哈希与向量库记录不符 ⇒ **正文被改过**(本台账的唯一入账类型)
19
+ // · `consumed_at`:提炼 job 消费过(= 已并入某次提炼输入)
20
+ // ⚠️ **不删行**:删了就没法回答"这条改动是谁消费的";保留 + 定期 prune。
21
+ import { DatabaseSync } from 'node:sqlite';
22
+
23
+ /** 台账 DDL(幂等;与 `vectors-store.mjs` 同库,加表不前向破坏既有表)。 */
24
+ export const DIRTY_DDL = `
25
+ CREATE TABLE IF NOT EXISTS memories_dirty (
26
+ ref_id TEXT NOT NULL PRIMARY KEY,
27
+ kind TEXT NOT NULL,
28
+ old_hash TEXT,
29
+ new_hash TEXT,
30
+ first_at INTEGER NOT NULL,
31
+ last_at INTEGER NOT NULL,
32
+ hits INTEGER NOT NULL DEFAULT 1,
33
+ consumed_at INTEGER
34
+ );
35
+ CREATE INDEX IF NOT EXISTS memories_dirty_open ON memories_dirty (consumed_at, last_at);
36
+ `;
37
+
38
+ export const DIRTY_SCHEMA = 1;
39
+
40
+ export function ensureDirtySchema(db) {
41
+ db.exec(DIRTY_DDL);
42
+ return true;
43
+ }
44
+
45
+ function hasTable(db, name) {
46
+ return Boolean(db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?").get(name));
47
+ }
48
+
49
+ /**
50
+ * 记一批差异(**幂等 upsert**)。
51
+ *
52
+ * 重复记账的口径(写死,别再各写各的):
53
+ * · 同一 `ref_id` 再次被检测到 → `hits+1` / `last_at` 刷新 / `new_hash` 刷新;
54
+ * · **`new_hash` 与库里不同 = 正文又变了** ⇒ 把 `consumed_at` 清空(重新变成待消费),
55
+ * 否则"消费过"的标记会把第二次改动永久吞掉(这正是台账最容易写错的一处)。
56
+ *
57
+ * @param {object} db 向量库连接(写)
58
+ * @param {Array<{refId:string|number, kind?:string, oldHash?:string|null, newHash?:string|null}>} items
59
+ * @param {{now?:number}} [opts]
60
+ * @returns {{recorded:number, reopened:number}}
61
+ */
62
+ export function recordDirty(db, items, opts = {}) {
63
+ if (!items || !items.length) return { recorded: 0, reopened: 0 };
64
+ ensureDirtySchema(db);
65
+ const now = Number(opts.now ?? Date.now());
66
+ const sel = db.prepare('SELECT ref_id, new_hash, consumed_at FROM memories_dirty WHERE ref_id = ?');
67
+ const ins = db.prepare(`INSERT INTO memories_dirty (ref_id, kind, old_hash, new_hash, first_at, last_at, hits, consumed_at)
68
+ VALUES (?, ?, ?, ?, ?, ?, 1, NULL)`);
69
+ const upd = db.prepare(`UPDATE memories_dirty SET kind = ?, old_hash = ?, new_hash = ?, last_at = ?, hits = hits + 1,
70
+ consumed_at = ? WHERE ref_id = ?`);
71
+ let recorded = 0;
72
+ let reopened = 0;
73
+ db.exec('BEGIN IMMEDIATE');
74
+ try {
75
+ for (const it of items) {
76
+ const refId = String(it.refId);
77
+ if (!refId) continue;
78
+ const kind = String(it.kind ?? 'mismatch');
79
+ const oldHash = it.oldHash ?? null;
80
+ const newHash = it.newHash ?? null;
81
+ const prev = sel.get(refId);
82
+ if (!prev) {
83
+ ins.run(refId, kind, oldHash, newHash, now, now);
84
+ recorded += 1;
85
+ continue;
86
+ }
87
+ const changedAgain = newHash != null && prev.new_hash != null && prev.new_hash !== newHash;
88
+ if (changedAgain && prev.consumed_at != null) reopened += 1;
89
+ // ⚠️ 只有"正文又变了"才清 consumed_at;**同一条重复被扫到不得清**(否则每轮轮转都会重新消费一遍)
90
+ upd.run(kind, oldHash, newHash, now, changedAgain ? null : prev.consumed_at, refId);
91
+ recorded += 1;
92
+ }
93
+ db.exec('COMMIT');
94
+ } catch (e) {
95
+ try { db.exec('ROLLBACK'); } catch { /* ignore */ }
96
+ throw e;
97
+ }
98
+ return { recorded, reopened };
99
+ }
100
+
101
+ /**
102
+ * 读台账。
103
+ * @param {object} db
104
+ * @param {{includeConsumed?:boolean, openOnly?:boolean, limit?:number}} [opts]
105
+ */
106
+ export function listDirty(db, opts = {}) {
107
+ if (!hasTable(db, 'memories_dirty')) return [];
108
+ const openOnly = opts.openOnly !== false && opts.includeConsumed !== true;
109
+ const lim = Number(opts.limit ?? 0);
110
+ const sql = `SELECT ref_id, kind, old_hash, new_hash, first_at, last_at, hits, consumed_at
111
+ FROM memories_dirty ${openOnly ? 'WHERE consumed_at IS NULL' : ''}
112
+ ORDER BY last_at DESC${lim > 0 ? ` LIMIT ${lim}` : ''}`;
113
+ return db.prepare(sql).all().map((r) => ({
114
+ refId: String(r.ref_id), kind: r.kind, oldHash: r.old_hash, newHash: r.new_hash,
115
+ firstAt: Number(r.first_at), lastAt: Number(r.last_at), hits: Number(r.hits),
116
+ consumedAt: r.consumed_at == null ? null : Number(r.consumed_at),
117
+ }));
118
+ }
119
+
120
+ /** 标记已消费(提炼 job 把这批 id 并入输入后调用)。 */
121
+ export function markConsumed(db, refIds, opts = {}) {
122
+ if (!refIds || !refIds.length) return 0;
123
+ if (!hasTable(db, 'memories_dirty')) return 0;
124
+ const now = Number(opts.now ?? Date.now());
125
+ const stmt = db.prepare('UPDATE memories_dirty SET consumed_at = ? WHERE ref_id = ? AND consumed_at IS NULL');
126
+ let n = 0;
127
+ db.exec('BEGIN IMMEDIATE');
128
+ try {
129
+ for (const id of refIds) n += Number(stmt.run(now, String(id)).changes || 0);
130
+ db.exec('COMMIT');
131
+ } catch (e) {
132
+ try { db.exec('ROLLBACK'); } catch { /* ignore */ }
133
+ throw e;
134
+ }
135
+ return n;
136
+ }
137
+
138
+ /** 台账概览(state / CLI / 日报用)。 */
139
+ export function dirtyStats(db) {
140
+ if (!hasTable(db, 'memories_dirty')) return { table: false, total: 0, open: 0, consumed: 0, lastAt: null };
141
+ const r = db.prepare(`SELECT COUNT(*) total,
142
+ SUM(CASE WHEN consumed_at IS NULL THEN 1 ELSE 0 END) open_count,
143
+ SUM(CASE WHEN consumed_at IS NOT NULL THEN 1 ELSE 0 END) consumed_count,
144
+ MAX(last_at) last_at FROM memories_dirty`).get();
145
+ return {
146
+ table: true,
147
+ total: Number(r.total ?? 0),
148
+ open: Number(r.open_count ?? 0),
149
+ consumed: Number(r.consumed_count ?? 0),
150
+ lastAt: r.last_at == null ? null : Number(r.last_at),
151
+ };
152
+ }
153
+
154
+ /** 定期清理:清掉**已消费**且早于 `keepDays` 的行(未消费的永不清理)。 */
155
+ export function pruneDirty(db, opts = {}) {
156
+ if (!hasTable(db, 'memories_dirty')) return 0;
157
+ const now = Number(opts.now ?? Date.now());
158
+ const keepDays = Number(opts.keepDays ?? 90);
159
+ const cutoff = now - keepDays * 86_400_000;
160
+ const r = db.prepare('DELETE FROM memories_dirty WHERE consumed_at IS NOT NULL AND consumed_at < ?').run(cutoff);
161
+ return Number(r.changes || 0);
162
+ }
163
+
164
+ /** 便捷:直接按库路径开一个写连接执行 fn(台账写入方都是短命脚本)。 */
165
+ export function withDirtyDb(dbPath, fn) {
166
+ const db = new DatabaseSync(dbPath);
167
+ try {
168
+ db.exec('PRAGMA busy_timeout = 5000;');
169
+ ensureDirtySchema(db);
170
+ return fn(db);
171
+ } finally {
172
+ try { db.close(); } catch { /* ignore */ }
173
+ }
174
+ }
@@ -0,0 +1,178 @@
1
+ // 记忆引擎 v2 · S3c —— 提炼产物回滚(方案 §3.5 回滚行)
2
+ //
3
+ // 口径:**按 `tags LIKE 'distilled%'` 批量列出 / 删除**;默认 dry-run;删除前 `VACUUM INTO` 快照。
4
+ // 为什么按 tag 而不是按 id 清单:提炼条是**推断产物**,主人可能整批否定("这批结论不对"),
5
+ // 也可能只否定某几组 ⇒ 两种粒度都要能选:`--key <srcKey>`(整组)/ `--id <行 id>`(单条)。
6
+ // ⚠️ 删除语句**恒带 `AND tags LIKE 'distilled%'`** —— 即使调用方算错了 id 集合,也不可能误删真实记忆。
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+ import crypto from 'node:crypto';
10
+ import { DatabaseSync } from 'node:sqlite';
11
+ import { vacuumInto } from './domain-tags.mjs';
12
+ import { DISTILL_PREFIX } from './distill.mjs';
13
+
14
+ /** 解析提炼条标签 → {srcKey, srcIds, date}。 */
15
+ export function parseDistilledTags(tags) {
16
+ const t = String(tags ?? '');
17
+ if (!t.startsWith(DISTILL_PREFIX)) return null;
18
+ const key = t.match(/(?:^|\s)src=([0-9a-f]+)(?:\s|$)/);
19
+ const ids = t.match(/(?:^|\s)srcids=([0-9,\s]*)(?:\s|$)/);
20
+ const date = t.match(/(?:^|\s)date=(\d{8})(?:\s|$)/);
21
+ return {
22
+ srcKey: key ? key[1] : null,
23
+ srcIds: ids && ids[1] ? ids[1].split(',').map((x) => Number(x.trim())).filter((x) => Number.isFinite(x)) : [],
24
+ date: date ? date[1] : null,
25
+ };
26
+ }
27
+
28
+ function open(dbPath, { readOnly = false } = {}) {
29
+ const db = readOnly ? new DatabaseSync(dbPath, { readOnly: true }) : new DatabaseSync(dbPath);
30
+ if (!readOnly) db.exec('PRAGMA busy_timeout = 5000;');
31
+ return db;
32
+ }
33
+
34
+ /**
35
+ * 列出提炼条(只读)。
36
+ * @param {{dbPath:string, keys?:string[], ids?:number[], limit?:number, headChars?:number}} p
37
+ */
38
+ export function listDistilled({ dbPath, keys = null, ids = null, limit = 0, headChars = 100 }) {
39
+ const db = open(dbPath, { readOnly: true });
40
+ try {
41
+ const rows = db
42
+ .prepare(`SELECT id, text, tags, updated_at, created_at${hasCol(db, 'owner_user') ? ', owner_user, visibility' : ''} FROM memories WHERE tags LIKE 'distilled%' ORDER BY id`)
43
+ .all();
44
+ let items = rows.map((r) => {
45
+ const meta = parseDistilledTags(r.tags) ?? { srcKey: null, srcIds: [], date: null };
46
+ return {
47
+ id: Number(r.id), srcKey: meta.srcKey, srcIds: meta.srcIds, date: meta.date,
48
+ chars: String(r.text ?? '').length,
49
+ head: String(r.text ?? '').slice(0, headChars).replace(/\s+/g, ' '),
50
+ tags: r.tags, createdAt: Number(r.created_at), updatedAt: Number(r.updated_at),
51
+ ownerUser: r.owner_user ?? null, visibility: r.visibility ?? null,
52
+ };
53
+ });
54
+ if (Array.isArray(keys) && keys.length) {
55
+ const want = new Set(keys.map((k) => String(k)));
56
+ items = items.filter((x) => want.has(String(x.srcKey)));
57
+ }
58
+ if (Array.isArray(ids) && ids.length) {
59
+ const want = new Set(ids.map((x) => Number(x)));
60
+ items = items.filter((x) => want.has(x.id));
61
+ }
62
+ if (limit > 0) items = items.slice(0, Number(limit));
63
+ return items;
64
+ } finally {
65
+ try { db.close(); } catch { /* ignore */ }
66
+ }
67
+ }
68
+
69
+ function hasCol(db, name) {
70
+ return db.prepare('PRAGMA table_info(memories)').all().some((r) => r.name === name);
71
+ }
72
+
73
+ /**
74
+ * **内容级摘要**:按 id 排序取 (id, text, tags, pinned, updated_at) 逐行哈希再整体哈希。
75
+ * 为什么不用文件 sha256:SQLite 写回后页布局会变(同样的内容也可能字节不同),
76
+ * 文件 sha 无法回答"回滚后库是否回到原状";内容摘要可以(本卡演练的判据就靠它)。
77
+ */
78
+ export function contentDigest(dbPath, { excludeTagsLike = null } = {}) {
79
+ const db = open(dbPath, { readOnly: true });
80
+ try {
81
+ const where = excludeTagsLike ? ` WHERE tags NOT LIKE '${excludeTagsLike}'` : '';
82
+ const rows = db.prepare(`SELECT id, text, tags, pinned, updated_at FROM memories${where} ORDER BY id`).all();
83
+ const h = crypto.createHash('sha256');
84
+ for (const r of rows) h.update(`${r.id}\u0001${r.text}\u0001${r.tags}\u0001${r.pinned}\u0001${r.updated_at}\u0002`, 'utf8');
85
+ return { count: rows.length, digest: h.digest('hex'), allCount: Number(db.prepare('SELECT COUNT(*) n FROM memories').get().n) };
86
+ } finally {
87
+ try { db.close(); } catch { /* ignore */ }
88
+ }
89
+ }
90
+
91
+ export function fileSha256(file) {
92
+ try {
93
+ return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
94
+ } catch {
95
+ return null;
96
+ }
97
+ }
98
+
99
+ /**
100
+ * 批量回滚。
101
+ * @param {{dbPath:string, apply?:boolean, keys?:string[], ids?:number[], snapshotDir?:string,
102
+ * now?:number, keepSnapshot?:boolean}} p
103
+ * @returns {{ok:boolean, matched:number, deleted:number, deletedIds:number[], snapshot:string|null,
104
+ * before:object, after:object, dryRun:boolean, rollbackFile:string|null}}
105
+ */
106
+ export function rollbackDistilled(p = {}) {
107
+ const { dbPath, keys = null, ids = null, snapshotDir = null, now = Date.now(), apply = false } = p;
108
+ const before = contentDigest(dbPath);
109
+ const items = listDistilled({ dbPath, keys, ids });
110
+ const out = {
111
+ ok: true, dryRun: !apply, matched: items.length, deleted: 0, deletedIds: [],
112
+ snapshot: null, before, after: before, items: items.map((x) => ({ id: x.id, srcKey: x.srcKey, head: x.head })),
113
+ rollbackFile: null, warnings: [],
114
+ };
115
+ if (!items.length) return out;
116
+
117
+ if (!apply) return out;
118
+
119
+ // 删除前快照(**唯一回滚点**)
120
+ const dir = snapshotDir ?? path.join(path.dirname(dbPath), 'snapshots');
121
+ fs.mkdirSync(dir, { recursive: true });
122
+ const stamp = new Date(Number(now)).toISOString().replace(/[:.]/g, '-');
123
+ const snap = path.join(dir, `memory-before-distill-rollback-${stamp}.db`);
124
+ vacuumInto(dbPath, snap);
125
+ out.snapshot = snap;
126
+ out.snapshotSha256 = fileSha256(snap);
127
+
128
+ const db = open(dbPath);
129
+ try {
130
+ // FTS 同步靠 `memories_ad` 触发器(触发器缺失时 DELETE 不会清 FTS 影子表 ⇒ 显式提示)
131
+ const triggers = db.prepare("SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name='memories'").all().map((r) => r.name);
132
+ out.triggers = triggers;
133
+ if (!triggers.some((n) => /_ad$|delete/i.test(n))) out.warnings.push('未发现 memories 的 AFTER DELETE 触发器 ⇒ FTS 影子表可能残留(建议核对 memories_fts)');
134
+
135
+ const targetIds = items.map((x) => x.id);
136
+ const stmt = db.prepare(`DELETE FROM memories WHERE id = ? AND tags LIKE 'distilled%'`);
137
+ db.exec('BEGIN IMMEDIATE');
138
+ try {
139
+ for (const id of targetIds) {
140
+ const r = stmt.run(id);
141
+ if (Number(r.changes || 0) > 0) out.deletedIds.push(id);
142
+ }
143
+ db.exec('COMMIT');
144
+ } catch (e) {
145
+ try { db.exec('ROLLBACK'); } catch { /* ignore */ }
146
+ throw e;
147
+ }
148
+ out.deleted = out.deletedIds.length;
149
+ } finally {
150
+ try { db.close(); } catch { /* ignore */ }
151
+ }
152
+ out.after = contentDigest(dbPath);
153
+ // 反向回滚脚本(把快照拷回去 = 一键恢复)
154
+ if (out.snapshot) {
155
+ const rbFile = path.join(dir, `restore-memory-${stamp}.mjs`);
156
+ fs.writeFileSync(rbFile, restoreScript({ snapshot: snap, target: dbPath }), 'utf8');
157
+ out.rollbackFile = rbFile;
158
+ }
159
+ return out;
160
+ }
161
+
162
+ /** 生成"把快照写回目标库"的一键恢复脚本(纯 ASCII 头 + 覆盖前校验存在性)。 */
163
+ function restoreScript({ snapshot, target }) {
164
+ return `// 自动生成:把提炼回滚前的快照写回目标记忆库(覆盖式)
165
+ // 用法:node ${path.basename('restore-memory.mjs')} 或 node "本文件路径"
166
+ import fs from 'node:fs';
167
+ const SNAP = ${JSON.stringify(snapshot)};
168
+ const TARGET = ${JSON.stringify(target)};
169
+ if (!fs.existsSync(SNAP)) { console.error('快照不存在:' + SNAP); process.exit(3); }
170
+ for (const suffix of ['', '-wal', '-shm']) {
171
+ const f = TARGET + suffix;
172
+ try { if (fs.existsSync(f)) fs.copyFileSync(f, f + '.before-restore'); } catch { /* ignore */ }
173
+ }
174
+ fs.copyFileSync(SNAP, TARGET);
175
+ for (const suffix of ['-wal', '-shm']) { try { fs.rmSync(TARGET + suffix, { force: true }); } catch { /* ignore */ } }
176
+ console.log('已恢复:' + TARGET + ' <- ' + SNAP);
177
+ `;
178
+ }