dsh-prime-memory 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/CHANGELOG.en.md +28 -0
  2. package/CHANGELOG.ja.md +30 -0
  3. package/CHANGELOG.ko.md +30 -0
  4. package/CHANGELOG.md +1220 -0
  5. package/ENGINEERING-NOTES.md +452 -0
  6. package/INSTALL.en.md +92 -0
  7. package/INSTALL.ja.md +92 -0
  8. package/INSTALL.ko.md +92 -0
  9. package/INSTALL.md +92 -0
  10. package/LICENSE +21 -0
  11. package/README.en.md +458 -0
  12. package/README.ja.md +306 -0
  13. package/README.ko.md +306 -0
  14. package/README.md +425 -0
  15. package/assets/changelog/0.8.10/01-write-only-pill.png +0 -0
  16. package/assets/changelog/0.8.9/01-panel.png +0 -0
  17. package/assets/changelog/0.8.9/02-halo.png +0 -0
  18. package/assets/changelog/0.8.9/03-layer-segmented-panel.png +0 -0
  19. package/assets/changelog/0.8.9/04-layer-l1-panel.png +0 -0
  20. package/assets/img/EmbeddingSource.png +0 -0
  21. package/assets/img/Hero.png +0 -0
  22. package/assets/img/Layers.png +0 -0
  23. package/assets/img/MemoryTools.png +0 -0
  24. package/assets/img/Modes.png +0 -0
  25. package/assets/img/ToolTrajectory.png +0 -0
  26. package/assets/img/ui-dark.jpg +0 -0
  27. package/assets/img/ui-light.jpg +0 -0
  28. package/assets/readme/bench-dialog.svg +70 -0
  29. package/assets/readme/bench-workflow.svg +79 -0
  30. package/assets/readme/flow.svg +189 -0
  31. package/assets/readme/storage.svg +115 -0
  32. package/cordis.patch.yml +16 -0
  33. package/dist/bench-control.d.ts +34 -0
  34. package/dist/bench-control.js +16 -0
  35. package/dist/client.js +4293 -0
  36. package/dist/config.d.ts +683 -0
  37. package/dist/config.js +129 -0
  38. package/dist/contract.d.ts +820 -0
  39. package/dist/contract.js +1 -0
  40. package/dist/embedding-worker.cjs +176 -0
  41. package/dist/graph/apply.d.ts +37 -0
  42. package/dist/graph/apply.js +270 -0
  43. package/dist/graph/constraints.d.ts +47 -0
  44. package/dist/graph/constraints.js +38 -0
  45. package/dist/graph/search.d.ts +16 -0
  46. package/dist/graph/search.js +115 -0
  47. package/dist/graph/types.d.ts +142 -0
  48. package/dist/graph/types.js +14 -0
  49. package/dist/hooks/capture.d.ts +32 -0
  50. package/dist/hooks/capture.js +194 -0
  51. package/dist/hooks/recall.d.ts +63 -0
  52. package/dist/hooks/recall.js +429 -0
  53. package/dist/index.d.ts +534 -0
  54. package/dist/index.js +344 -0
  55. package/dist/llm-usage.d.ts +26 -0
  56. package/dist/llm-usage.js +37 -0
  57. package/dist/llm.d.ts +153 -0
  58. package/dist/llm.js +530 -0
  59. package/dist/pipeline/graph.d.ts +35 -0
  60. package/dist/pipeline/graph.js +104 -0
  61. package/dist/pipeline/l1.d.ts +19 -0
  62. package/dist/pipeline/l1.js +271 -0
  63. package/dist/pipeline/l2.d.ts +13 -0
  64. package/dist/pipeline/l2.js +83 -0
  65. package/dist/pipeline/l3.d.ts +15 -0
  66. package/dist/pipeline/l3.js +78 -0
  67. package/dist/pipeline/rebuild.d.ts +61 -0
  68. package/dist/pipeline/rebuild.js +307 -0
  69. package/dist/pipeline/ruminate.d.ts +89 -0
  70. package/dist/pipeline/ruminate.js +298 -0
  71. package/dist/pipeline/runner.d.ts +167 -0
  72. package/dist/pipeline/runner.js +638 -0
  73. package/dist/pipeline/trigger.d.ts +40 -0
  74. package/dist/pipeline/trigger.js +75 -0
  75. package/dist/prompts/graph-projection.d.ts +70 -0
  76. package/dist/prompts/graph-projection.js +167 -0
  77. package/dist/prompts/l1-dedup.d.ts +22 -0
  78. package/dist/prompts/l1-dedup.js +251 -0
  79. package/dist/prompts/l1-extraction.d.ts +22 -0
  80. package/dist/prompts/l1-extraction.js +457 -0
  81. package/dist/prompts/persona.d.ts +23 -0
  82. package/dist/prompts/persona.js +240 -0
  83. package/dist/prompts/scene.d.ts +32 -0
  84. package/dist/prompts/scene.js +414 -0
  85. package/dist/runtime-package-lock.json +982 -0
  86. package/dist/settings.d.ts +50 -0
  87. package/dist/settings.js +355 -0
  88. package/dist/stats.d.ts +109 -0
  89. package/dist/stats.js +929 -0
  90. package/dist/store/bm25.d.ts +19 -0
  91. package/dist/store/bm25.js +63 -0
  92. package/dist/store/cost-ledger.d.ts +75 -0
  93. package/dist/store/cost-ledger.js +171 -0
  94. package/dist/store/download-queue.d.ts +79 -0
  95. package/dist/store/download-queue.js +424 -0
  96. package/dist/store/embedding-source.d.ts +118 -0
  97. package/dist/store/embedding-source.js +443 -0
  98. package/dist/store/embedding.d.ts +90 -0
  99. package/dist/store/embedding.js +206 -0
  100. package/dist/store/graph-store.d.ts +94 -0
  101. package/dist/store/graph-store.js +641 -0
  102. package/dist/store/l0.d.ts +40 -0
  103. package/dist/store/l0.js +197 -0
  104. package/dist/store/l1.d.ts +93 -0
  105. package/dist/store/l1.js +297 -0
  106. package/dist/store/local-embedding.d.ts +89 -0
  107. package/dist/store/local-embedding.js +227 -0
  108. package/dist/store/model-catalog.d.ts +48 -0
  109. package/dist/store/model-catalog.js +81 -0
  110. package/dist/store/occupancy.d.ts +30 -0
  111. package/dist/store/occupancy.js +134 -0
  112. package/dist/store/pending.d.ts +36 -0
  113. package/dist/store/pending.js +103 -0
  114. package/dist/store/persona.d.ts +15 -0
  115. package/dist/store/persona.js +60 -0
  116. package/dist/store/recall-dedupe.d.ts +26 -0
  117. package/dist/store/recall-dedupe.js +138 -0
  118. package/dist/store/runtime-installer.d.ts +59 -0
  119. package/dist/store/runtime-installer.js +243 -0
  120. package/dist/store/scenes.d.ts +24 -0
  121. package/dist/store/scenes.js +160 -0
  122. package/dist/store/search-utils.d.ts +38 -0
  123. package/dist/store/search-utils.js +100 -0
  124. package/dist/store/session-modes.d.ts +35 -0
  125. package/dist/store/session-modes.js +144 -0
  126. package/dist/store/sqlite.d.ts +246 -0
  127. package/dist/store/sqlite.js +1491 -0
  128. package/dist/store/state.d.ts +41 -0
  129. package/dist/store/state.js +72 -0
  130. package/dist/token-cost.d.ts +23 -0
  131. package/dist/token-cost.js +185 -0
  132. package/dist/tools/index.d.ts +34 -0
  133. package/dist/tools/index.js +758 -0
  134. package/dist/types.d.ts +139 -0
  135. package/dist/types.js +38 -0
  136. package/dist/util/context-occupancy.d.ts +68 -0
  137. package/dist/util/context-occupancy.js +92 -0
  138. package/dist/util/filelog.d.ts +6 -0
  139. package/dist/util/filelog.js +108 -0
  140. package/dist/util/io.d.ts +18 -0
  141. package/dist/util/io.js +97 -0
  142. package/dist/util/recall-budget.d.ts +31 -0
  143. package/dist/util/recall-budget.js +84 -0
  144. package/dist/util/sanitize.d.ts +11 -0
  145. package/dist/util/sanitize.js +67 -0
  146. package/dist/util/text.d.ts +16 -0
  147. package/dist/util/text.js +61 -0
  148. package/dist/util/tokenizer.d.ts +9 -0
  149. package/dist/util/tokenizer.js +50 -0
  150. package/dsh.plugin.json +22 -0
  151. package/package.json +118 -0
@@ -0,0 +1,227 @@
1
+ /**
2
+ * 本地嵌入服务(worker 线程化版):transformers.js 的模型加载与 ONNX 推理全部在
3
+ * worker_threads 子线程执行(resources/embedding-worker.cjs),本类只是主线程侧的
4
+ * 协议代理——onnxruntime-node 的 run/loadModel 是主线程同步调用,留在宿主事件循环
5
+ * 会冻结整个 dsh 页面(0.8.6 修复的真实事故)。
6
+ *
7
+ * - 懒加载:首次嵌入/warmup 才让 worker 加载模型,close()=terminate 释放线程与
8
+ * 模型(嵌入源切走/关闭时调用),terminated 后不可复用;
9
+ * - 模型从数据目录 models/<id>/ 本地加载(worker 侧 env.allowRemoteModels=false);
10
+ * - channel 可注入(测试缝):用假通道验证协议与状态机,不触真模型;
11
+ * - callOpts.timeoutMs 经 Promise.race 钳制(迟到回复按 id 丢弃——推理在
12
+ * worker 线程无法真正取消,但主线程可以停止等待并降级 FTS);
13
+ * - 池化方式来自模型目录(BGE 系 CLS / Gemma 系 MEAN),normalize 交给
14
+ * pipeline 内建 L2 归一(与远程路径的 sanitizeAndNormalize 语义一致)。
15
+ */
16
+ import * as path from 'node:path';
17
+ import { fileURLToPath } from 'node:url';
18
+ import { Worker } from 'node:worker_threads';
19
+ /** 默认 worker 资产路径:dist/store/ → dist/embedding-worker.cjs(构建期拷入)。 */
20
+ function defaultWorkerPath() {
21
+ return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'embedding-worker.cjs');
22
+ }
23
+ /** 真实通道:spawn worker_threads + 自增 id 配对 + 崩溃兜底拒绝。 */
24
+ class RealWorkerChannel {
25
+ worker;
26
+ pending = new Map();
27
+ nextId = 1;
28
+ terminated = false;
29
+ crashed;
30
+ crashCb;
31
+ constructor(workerPath, workerData) {
32
+ this.worker = new Worker(workerPath, { workerData });
33
+ this.worker.on('message', (msg) => {
34
+ if (msg && msg.type === 'fatal') {
35
+ this.failAll(`本地嵌入 worker 致命错误: ${msg.error ?? '未知'}`);
36
+ return;
37
+ }
38
+ const id = msg.id;
39
+ if (typeof id !== 'number')
40
+ return;
41
+ const entry = this.pending.get(id);
42
+ if (!entry)
43
+ return; // 迟到回复(调用方已超时放弃)
44
+ this.pending.delete(id);
45
+ entry.resolve(msg);
46
+ });
47
+ // error(未捕获异常且 worker 未自处理)与 exit(含 fatal 后的退出)都兜底拒绝
48
+ this.worker.on('error', (err) => this.failAll(`本地嵌入 worker 线程错误: ${err.message}`));
49
+ this.worker.on('exit', (code) => {
50
+ if (!this.terminated)
51
+ this.failAll(`本地嵌入 worker 线程退出(code=${code})`);
52
+ });
53
+ }
54
+ request(call) {
55
+ // 已释放/已崩溃的通道快速拒绝——postMessage 到死线程是静默无回应,调用方会挂到超时
56
+ if (this.terminated)
57
+ return Promise.reject(new Error('嵌入 worker 已释放'));
58
+ if (this.crashed)
59
+ return Promise.reject(new Error(this.crashed));
60
+ const id = this.nextId++;
61
+ return new Promise((resolve, reject) => {
62
+ this.pending.set(id, { resolve, reject });
63
+ this.worker.postMessage({ ...call, id });
64
+ });
65
+ }
66
+ terminate() {
67
+ if (this.terminated)
68
+ return;
69
+ this.terminated = true;
70
+ this.failAll('嵌入 worker 已释放');
71
+ void this.worker.terminate();
72
+ }
73
+ setOnCrash(cb) {
74
+ this.crashCb = cb;
75
+ // 构造与回调注册之间发生的崩溃(spawn 即失败等)不丢通知
76
+ if (this.crashed)
77
+ cb(this.crashed);
78
+ }
79
+ failAll(error) {
80
+ for (const [, entry] of this.pending)
81
+ entry.reject(new Error(error));
82
+ this.pending.clear();
83
+ if (!this.terminated && !this.crashed) {
84
+ this.crashed = error;
85
+ this.crashCb?.(error);
86
+ }
87
+ }
88
+ }
89
+ export class LocalEmbeddingService {
90
+ state = 'idle';
91
+ loadError = null;
92
+ channel;
93
+ entry;
94
+ logger;
95
+ constructor(entry, modelDir, opts) {
96
+ this.entry = entry;
97
+ this.logger = opts.logger;
98
+ const maxInputChars = opts.maxInputChars && opts.maxInputChars > 0 ? opts.maxInputChars : 5000;
99
+ this.channel =
100
+ opts.channel ??
101
+ new RealWorkerChannel(opts.workerPath ?? defaultWorkerPath(), {
102
+ runtimeDir: opts.runtimeDir,
103
+ modelDir,
104
+ pooling: entry.pooling,
105
+ dtype: 'q8',
106
+ maxInputChars,
107
+ });
108
+ // 崩溃不自愈(换源/重启恢复):拒绝语义沿 EmbedHelper 降级链走 FTS
109
+ this.channel.setOnCrash((error) => {
110
+ if (this.state === 'terminated')
111
+ return;
112
+ this.state = 'failed';
113
+ this.loadError = error;
114
+ this.logger?.warn(`[memory] ${error}(本地嵌入转入 failed 态,换源或重启可恢复)`);
115
+ });
116
+ }
117
+ getDimensions() {
118
+ return this.entry.dims;
119
+ }
120
+ getProviderInfo() {
121
+ // dimensions 语义与远程路径一致(sqlite 的 providerInfo 比对含维度)
122
+ return { provider: 'local', model: this.entry.id, dimensions: this.entry.dims };
123
+ }
124
+ isReady() {
125
+ return this.state === 'ready';
126
+ }
127
+ /** 状态(进度展示用)。 */
128
+ getState() {
129
+ return this.state;
130
+ }
131
+ getLoadError() {
132
+ return this.loadError;
133
+ }
134
+ /** 后台预热:启动后让 worker 立即加载模型(幂等;失败态可重试)。 */
135
+ startWarmup() {
136
+ void this.waitForReady().catch(() => { });
137
+ }
138
+ /** 等待模型就绪(warmup 协议;applyChain 的 warming 阶段与测试用)。 */
139
+ async waitForReady() {
140
+ if (this.state === 'ready')
141
+ return;
142
+ if (this.state === 'terminated') {
143
+ throw new Error('本地嵌入服务已释放(嵌入源已切换);本实例不可复用');
144
+ }
145
+ if (this.state !== 'failed')
146
+ this.state = 'loading';
147
+ const reply = await this.channel.request({ type: 'warmup' });
148
+ if (!reply.ok) {
149
+ this.applyLoadFailure(reply.error);
150
+ throw new Error(reply.error);
151
+ }
152
+ this.markReady();
153
+ }
154
+ async embed(text, callOpts) {
155
+ const [vec] = await this.embedBatch([text], callOpts);
156
+ return vec;
157
+ }
158
+ async embedBatch(texts, callOpts) {
159
+ if (texts.length === 0)
160
+ return [];
161
+ if (this.state === 'terminated') {
162
+ throw new Error('本地嵌入服务已释放(嵌入源已切换);本实例不可复用');
163
+ }
164
+ if (this.state === 'failed') {
165
+ // 只有明确失败过的服务才抛错(EmbedHelper 捕获后降级 FTS);warmup 可重试
166
+ throw new Error(`本地嵌入模型加载失败: ${this.loadError ?? '未知原因'}(重启插件或重新下载模型可重试)`);
167
+ }
168
+ if (this.state !== 'ready')
169
+ this.state = 'loading';
170
+ // 单条请求(召回 query)带优先标记:worker 侧插队,不被 reindex 批次堵队尾
171
+ const reply = await this.requestWithTimeout({ type: 'embed', texts, priority: texts.length === 1 }, callOpts?.timeoutMs);
172
+ if (!reply.ok) {
173
+ if (reply.stage === 'load')
174
+ this.applyLoadFailure(reply.error);
175
+ else
176
+ this.markReady(); // 推理失败说明模型已加载成功(loading → ready),失败只属于这一次调用
177
+ throw new Error(reply.error);
178
+ }
179
+ if (reply.type !== 'embedded')
180
+ throw new Error(`嵌入 worker 返回异常消息类型: ${reply.type}`);
181
+ this.markReady();
182
+ for (const v of reply.vectors) {
183
+ if (v.length !== this.entry.dims) {
184
+ throw new Error(`本地嵌入维度不匹配:期望 ${this.entry.dims},得到 ${v.length}`);
185
+ }
186
+ }
187
+ return reply.vectors;
188
+ }
189
+ /** 释放 worker 线程与模型(嵌入源切走/关闭时调用;幂等)。terminated 后不可
190
+ * 再复用——防止插件卸载/切走后残留的重嵌循环把模型重新加载常驻(内存泄漏)。 */
191
+ close() {
192
+ this.state = 'terminated';
193
+ this.loadError = null;
194
+ this.channel.terminate();
195
+ }
196
+ /** 内层钳制(仅缩短):超时放弃等待(迟到回复由通道按 id 丢弃),调用方降级。 */
197
+ async requestWithTimeout(call, timeoutMs) {
198
+ if (!(timeoutMs && timeoutMs > 0))
199
+ return this.channel.request(call);
200
+ let timer;
201
+ try {
202
+ return await Promise.race([
203
+ this.channel.request(call),
204
+ new Promise((_, reject) => {
205
+ timer = setTimeout(() => reject(new Error(`本地嵌入调用超时(${timeoutMs}ms),已放弃等待`)), timeoutMs);
206
+ }),
207
+ ]);
208
+ }
209
+ finally {
210
+ // 先到者胜出后清掉另一个定时器(不清理会挂住引用至自然到期)
211
+ if (timer)
212
+ clearTimeout(timer);
213
+ }
214
+ }
215
+ /** loading → ready 一次性日志(memory.log 时序可读性:启动到模型就绪的间隔)。 */
216
+ markReady() {
217
+ if (this.state === 'ready')
218
+ return;
219
+ this.state = 'ready';
220
+ this.logger?.info(`[memory] 本地嵌入模型就绪: ${this.entry.id}(dims=${this.entry.dims},pooling=${this.entry.pooling})`);
221
+ }
222
+ applyLoadFailure(error) {
223
+ this.state = 'failed';
224
+ this.loadError = error;
225
+ this.logger?.warn(`[memory] 本地嵌入模型加载失败(${this.entry.id}): ${error}`);
226
+ }
227
+ }
@@ -0,0 +1,48 @@
1
+ /**
2
+ * 本地嵌入模型目录:插件内置白名单,不提供任意 repo 下载。
3
+ * 每款锁定 revision(commit sha)+ 每文件 sha256——目录即完整性契约,
4
+ * 升级模型 = 改这里(维度变化会触发使用方全量重嵌入)。
5
+ *
6
+ * 数据来源(2026-08-17 采集):
7
+ * - revision 与 LFS 大文件 sha256:HF tree API(LFS oid 即 sha256);
8
+ * - 小文件 sha256:从镜像按锁定 revision 下载后本地实测。
9
+ * embeddinggemma 是 ONNX 外部权重格式:model_quantized.onnx(图,~0.5MB)与
10
+ * model_quantized.onnx_data(权重,~294MB)必须成对存在,onnxruntime 同目录自动加载。
11
+ *
12
+ * 注:以下 revision/size/sha256 数据表为上游模型文件的完整性事实数据,
13
+ * 净室重写按原值逐字保留(改动任何一位都会破坏校验)。
14
+ */
15
+ export interface CatalogFile {
16
+ /** 仓库内相对路径(同时是 models/<id>/ 下的落盘路径)。 */
17
+ path: string;
18
+ /** 字节数(磁盘检查的进度分母)。 */
19
+ size: number;
20
+ /** sha256 hex(64 位)。 */
21
+ sha256: string;
22
+ }
23
+ export interface CatalogEntry {
24
+ /** 稳定 id:嵌入源状态文件(embedding-source.json)引用它。 */
25
+ id: string;
26
+ /** 展示名。 */
27
+ name: string;
28
+ /** HF 仓库 repo id。 */
29
+ repo: string;
30
+ /** 锁定的 revision。 */
31
+ revision: string;
32
+ /** 向量维度(vec0 建表与切换重嵌的判据)。 */
33
+ dims: number;
34
+ /** 上下文窗口(token),仅 UI 展示。 */
35
+ contextTokens: number;
36
+ /** 池化方式:BGE 系 CLS;embeddinggemma 系(Gemma 解码器底座)MEAN。 */
37
+ pooling: 'cls' | 'mean';
38
+ /** UI 标签。 */
39
+ tags: string[];
40
+ /** 一句话说明(设置页模型卡)。 */
41
+ description: string;
42
+ files: CatalogFile[];
43
+ }
44
+ export declare const MODEL_CATALOG: CatalogEntry[];
45
+ /** 按 id 取目录项。 */
46
+ export declare function catalogById(id: string): CatalogEntry | undefined;
47
+ /** 模型总字节数(磁盘检查与整体进度分母)。 */
48
+ export declare function catalogTotalBytes(entry: CatalogEntry): number;
@@ -0,0 +1,81 @@
1
+ /**
2
+ * 本地嵌入模型目录:插件内置白名单,不提供任意 repo 下载。
3
+ * 每款锁定 revision(commit sha)+ 每文件 sha256——目录即完整性契约,
4
+ * 升级模型 = 改这里(维度变化会触发使用方全量重嵌入)。
5
+ *
6
+ * 数据来源(2026-08-17 采集):
7
+ * - revision 与 LFS 大文件 sha256:HF tree API(LFS oid 即 sha256);
8
+ * - 小文件 sha256:从镜像按锁定 revision 下载后本地实测。
9
+ * embeddinggemma 是 ONNX 外部权重格式:model_quantized.onnx(图,~0.5MB)与
10
+ * model_quantized.onnx_data(权重,~294MB)必须成对存在,onnxruntime 同目录自动加载。
11
+ *
12
+ * 注:以下 revision/size/sha256 数据表为上游模型文件的完整性事实数据,
13
+ * 净室重写按原值逐字保留(改动任何一位都会破坏校验)。
14
+ */
15
+ export const MODEL_CATALOG = [
16
+ {
17
+ id: 'bge-small-zh-v1.5',
18
+ name: 'BGE small 中文',
19
+ repo: 'Xenova/bge-small-zh-v1.5',
20
+ revision: '75c43b069aac4d136ba6bc1122f995fedcfd2781',
21
+ dims: 512,
22
+ contextTokens: 512,
23
+ pooling: 'cls',
24
+ tags: ['中文', '轻量'],
25
+ description: '中文专用小模型:约 25MB,CPU 嵌入最快,适合先体验语义检索。',
26
+ files: [
27
+ { path: 'config.json', size: 716, sha256: 'd4193ead3a810fd694fa8a31d7fc72fbaebc0668b603e398734bf2f6538ff42f' },
28
+ { path: 'tokenizer_config.json', size: 367, sha256: 'e6f3b96db926a37d4039995fbf5ad17de158dfb8f6343d607e4dbaad18d75f5a' },
29
+ { path: 'special_tokens_map.json', size: 125, sha256: 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3' },
30
+ { path: 'tokenizer.json', size: 439125, sha256: '48cea5d44424912a6fd1ea647bf4fe50b55ab8b1e5879c3275f80e339e8fae26' },
31
+ { path: 'onnx/model_quantized.onnx', size: 24010842, sha256: '15b717c382bcb518ba457b93ea6850ede7f4f1cd8937454aa06972366cd19bcc' },
32
+ ],
33
+ },
34
+ {
35
+ id: 'embeddinggemma-300m',
36
+ name: 'EmbeddingGemma 300M',
37
+ repo: 'onnx-community/embeddinggemma-300m-ONNX',
38
+ revision: '5090578d9565bb06545b4552f76e6bc2c93e4a66',
39
+ dims: 768,
40
+ contextTokens: 2048,
41
+ pooling: 'mean',
42
+ tags: ['多语言', '均衡'],
43
+ description: 'Google 官方嵌入模型:100+ 语言含中文,约 295MB,质量与开销均衡(上游 MemoryCore 同款)。',
44
+ files: [
45
+ { path: 'config.json', size: 1765, sha256: '6e1f06404b7163e0325ed2ea3e6781cde50f4a50b31780a95ad0d30e8404d77b' },
46
+ { path: 'added_tokens.json', size: 35, sha256: '50b2f405ba56a26d4913fd772089992252d7f942123cc0a034d96424221ba946' },
47
+ { path: 'special_tokens_map.json', size: 662, sha256: '2f7b0adf4fb469770bb1490e3e35df87b1dc578246c5e7e6fc76ecf33213a397' },
48
+ { path: 'tokenizer_config.json', size: 1156830, sha256: '3ca953eea6c3c9fcda9cf3df22949ff18b216f7c74bd6459230f3f1013953f3a' },
49
+ { path: 'generation_config.json', size: 133, sha256: '1fb1efd221c1ca88a736d1b36cb47d754c177677e222acb3b1e5424c5d664870' },
50
+ { path: 'tokenizer.json', size: 20323312, sha256: '4dda02faaf32bc91031dc8c88457ac272b00c1016cc679757d1c441b248b9c47' },
51
+ { path: 'onnx/model_quantized.onnx', size: 567874, sha256: '172efde319fe1542dc41f31be6154910b05b78f7a861c265c4600eec906bd6d8' },
52
+ { path: 'onnx/model_quantized.onnx_data', size: 308890624, sha256: '705626e28e4c23c82ade34566b4197d97f534c12275fa406dfb71e9937d388c0' },
53
+ ],
54
+ },
55
+ {
56
+ id: 'bge-m3',
57
+ name: 'BGE-M3 大杯',
58
+ repo: 'Xenova/bge-m3',
59
+ revision: '4de13258303883538bd53b696b452bf8099f0858',
60
+ dims: 1024,
61
+ contextTokens: 8192,
62
+ pooling: 'cls',
63
+ tags: ['中文', '长上下文'],
64
+ description: '中文质量最强:8192 token 上下文,约 543MB,CPU 嵌入较慢(单条可达秒级)。',
65
+ files: [
66
+ { path: 'config.json', size: 770, sha256: '734a79bf12d388c1467a4e3ab625f45de7f6906cffcfb93a1eca1787504bed95' },
67
+ { path: 'special_tokens_map.json', size: 964, sha256: '8c785abebea9ae3257b61681b4e6fd8365ceafde980c21970d001e834cf10835' },
68
+ { path: 'tokenizer_config.json', size: 1173, sha256: '7e4c1cc848840aeccdd763458c18dd525eb0f795c992e00ebe9c28554e7db2d4' },
69
+ { path: 'tokenizer.json', size: 17082821, sha256: '6710678b12670bc442b99edc952c4d996ae309a7020c1fa0096dd245c2faf790' },
70
+ { path: 'onnx/model_quantized.onnx', size: 569694530, sha256: '0826f8c1ab9edf1801db86c61919d4d108e8bfc0b809ec823ad366882ff0b77d' },
71
+ ],
72
+ },
73
+ ];
74
+ /** 按 id 取目录项。 */
75
+ export function catalogById(id) {
76
+ return MODEL_CATALOG.find((m) => m.id === id);
77
+ }
78
+ /** 模型总字节数(磁盘检查与整体进度分母)。 */
79
+ export function catalogTotalBytes(entry) {
80
+ return entry.files.reduce((sum, f) => sum + f.size, 0);
81
+ }
@@ -0,0 +1,30 @@
1
+ import type { MemoryLogger } from '../types.js';
2
+ import type { OccupancyLedger } from '../util/context-occupancy.js';
3
+ /** 会话条目上限(按 updatedAt 淘汰最旧;防文件无限增长)。 */
4
+ export declare const OCCUPANCY_SESSION_CAP = 200;
5
+ export declare class OccupancyStore {
6
+ private readonly logger?;
7
+ private readonly file;
8
+ private readonly entries;
9
+ private persistFailed;
10
+ /** 串行化持久化写(避免并发原子写撞临时文件名);init 链最前(先载入再落盘,防丢更新)。 */
11
+ private writeChain;
12
+ constructor(dataDir: string, logger?: MemoryLogger | undefined);
13
+ /** 载入持久化账目(合并进内存——构造与载入之间发生的 save 不丢);失败降级内存态。 */
14
+ private init;
15
+ /**
16
+ * 该会话的持久化账目(热路径同步读;从未注入/已复位返回 null)。
17
+ * 返回浅拷贝——调用方(ledgerFor 复生)会在其上做迁移;若交出内部引用,
18
+ * 原地修改会让 save() 的数值比较误判"未变"而跳过写穿。
19
+ */
20
+ load(sessionId: string): OccupancyLedger | null;
21
+ /**
22
+ * 迁移后写穿。数值未变只刷新内存时间戳不落盘(profile 稳定区每次请求组装都
23
+ * 触发迁移,但内容不变时不应产生文件写);stock 归零即删除条目。
24
+ */
25
+ save(sessionId: string, ledger: OccupancyLedger): void;
26
+ /** 等待在途持久化写完成(测试/停机用)。 */
27
+ flush(): Promise<void>;
28
+ private persist;
29
+ private serialize;
30
+ }
@@ -0,0 +1,134 @@
1
+ /**
2
+ * 记忆占用流水存储:sessionId → OccupancyLedger 的持久化映射。
3
+ *
4
+ * 动机:占用账本是进程内存态,dsh 重启后历史会话"窗口里明明有记忆、账面却为零",
5
+ * 指示器对重启前注入的存量失明。本存储让账目跨进程复生:
6
+ * - 每次账本迁移后写穿(串行化原子写 + 失败降级内存态);
7
+ * - 查询热路径仍是同步内存读——文件只在迁移时刻与启动载入时触碰;
8
+ * - stock 归零的条目不落盘(compaction 复位即天然删除);
9
+ * - agent 销毁不删记录(持久化语义与召回去重一致:会话恢复后账目继续);
10
+ * - 90 天未更新过期清理 + 会话条目上限(session-modes/dedupe 同款量级)。
11
+ */
12
+ import * as path from 'node:path';
13
+ import { errDetail } from '../util/filelog.js';
14
+ import { atomicWriteJson, ensureDir, readJsonIfExists } from '../util/io.js';
15
+ /** 会话条目上限(按 updatedAt 淘汰最旧;防文件无限增长)。 */
16
+ export const OCCUPANCY_SESSION_CAP = 200;
17
+ /** 条目过期清理(90 天未更新即丢弃,与 session-modes/dedupe 同款量级)。 */
18
+ const PRUNE_MS = 90 * 24 * 3600_000;
19
+ export class OccupancyStore {
20
+ logger;
21
+ file;
22
+ entries = new Map();
23
+ persistFailed = false;
24
+ /** 串行化持久化写(避免并发原子写撞临时文件名);init 链最前(先载入再落盘,防丢更新)。 */
25
+ writeChain;
26
+ constructor(dataDir, logger) {
27
+ this.logger = logger;
28
+ this.file = path.join(dataDir, 'occupancy.json');
29
+ this.writeChain = this.init();
30
+ }
31
+ /** 载入持久化账目(合并进内存——构造与载入之间发生的 save 不丢);失败降级内存态。 */
32
+ async init() {
33
+ const data = await readJsonIfExists(this.file);
34
+ if (!data?.sessions || typeof data.sessions !== 'object')
35
+ return;
36
+ const now = Date.now();
37
+ let count = 0;
38
+ for (const [sid, entry] of Object.entries(data.sessions)) {
39
+ if (typeof entry?.stockTokens !== 'number')
40
+ continue;
41
+ if (now - (entry.updatedAt ?? 0) > PRUNE_MS)
42
+ continue;
43
+ const existing = this.entries.get(sid);
44
+ if (existing) {
45
+ if (entry.updatedAt > existing.updatedAt)
46
+ this.entries.set(sid, { ...entry });
47
+ }
48
+ else {
49
+ this.entries.set(sid, { ...entry });
50
+ }
51
+ count++;
52
+ }
53
+ if (count > 0)
54
+ this.logger?.info(`[memory] 记忆占用账目载入 ${count} 个会话`);
55
+ }
56
+ /**
57
+ * 该会话的持久化账目(热路径同步读;从未注入/已复位返回 null)。
58
+ * 返回浅拷贝——调用方(ledgerFor 复生)会在其上做迁移;若交出内部引用,
59
+ * 原地修改会让 save() 的数值比较误判"未变"而跳过写穿。
60
+ */
61
+ load(sessionId) {
62
+ const e = this.entries.get(sessionId);
63
+ return e ? { ...e } : null;
64
+ }
65
+ /**
66
+ * 迁移后写穿。数值未变只刷新内存时间戳不落盘(profile 稳定区每次请求组装都
67
+ * 触发迁移,但内容不变时不应产生文件写);stock 归零即删除条目。
68
+ */
69
+ save(sessionId, ledger) {
70
+ const prev = this.entries.get(sessionId);
71
+ const sameNumbers = prev !== undefined &&
72
+ prev.stockTokens === ledger.stockTokens &&
73
+ prev.recallTokens === ledger.recallTokens &&
74
+ prev.profileTokens === ledger.profileTokens &&
75
+ prev.lastInjectTokens === ledger.lastInjectTokens;
76
+ if (sameNumbers) {
77
+ prev.updatedAt = ledger.updatedAt;
78
+ return;
79
+ }
80
+ if (ledger.stockTokens <= 0) {
81
+ if (prev === undefined)
82
+ return;
83
+ this.entries.delete(sessionId);
84
+ }
85
+ else {
86
+ this.entries.set(sessionId, { ...ledger });
87
+ }
88
+ this.writeChain = this.writeChain.then(() => this.persist());
89
+ }
90
+ /** 等待在途持久化写完成(测试/停机用)。 */
91
+ flush() {
92
+ return this.writeChain;
93
+ }
94
+ async persist() {
95
+ try {
96
+ await ensureDir(path.dirname(this.file));
97
+ await atomicWriteJson(this.file, this.serialize());
98
+ this.persistFailed = false;
99
+ }
100
+ catch (err) {
101
+ if (!this.persistFailed) {
102
+ this.persistFailed = true;
103
+ this.logger?.warn(`[memory] 记忆占用持久化失败(降级内存态): ${errDetail(err)}`);
104
+ }
105
+ }
106
+ }
107
+ serialize() {
108
+ const now = Date.now();
109
+ for (const [sid, e] of this.entries) {
110
+ if (now - e.updatedAt > PRUNE_MS && e.updatedAt > 0)
111
+ this.entries.delete(sid);
112
+ }
113
+ while (this.entries.size > OCCUPANCY_SESSION_CAP) {
114
+ let oldest;
115
+ let oldestAt = Infinity;
116
+ for (const [sid, e] of this.entries) {
117
+ if (e.updatedAt > 0 && e.updatedAt < oldestAt) {
118
+ oldest = sid;
119
+ oldestAt = e.updatedAt;
120
+ }
121
+ }
122
+ if (oldest === undefined)
123
+ break;
124
+ this.entries.delete(oldest);
125
+ }
126
+ const sessions = {};
127
+ for (const [sid, e] of this.entries) {
128
+ if (e.stockTokens <= 0)
129
+ continue; // 归零条目不落盘
130
+ sessions[sid] = { ...e };
131
+ }
132
+ return { version: 1, sessions };
133
+ }
134
+ }
@@ -0,0 +1,36 @@
1
+ import type { ConversationMessage, ExtractMode, MemoryLogger } from '../types.js';
2
+ /** 带会话标识的未蒸馏消息(会话切片的成员;CONTEXT.md「会话切片」「捕获档位」)。 */
3
+ export interface PendingMessage extends ConversationMessage {
4
+ /** 捕获该消息的会话;旧格式数据无此字段,加载时归 legacy 组一次性蒸馏。 */
5
+ sessionId: string;
6
+ }
7
+ /** 三档蒸馏缓冲桶(off 在捕获侧已被拦截,永远不到这里)。 */
8
+ export interface PendingBuckets {
9
+ auto: PendingMessage[];
10
+ chat: PendingMessage[];
11
+ work: PendingMessage[];
12
+ }
13
+ /** 旧格式(无 sessionId 字段)条目加载时归属的会话组。 */
14
+ export declare const LEGACY_SESSION = "legacy";
15
+ /** 各档位桶的渐进阈值状态(0 = 已毕业用稳态值;1 = 爬坡起点;ADR-0003)。 */
16
+ export type WarmupState = Record<ExtractMode, number>;
17
+ /** 全新起步:三档都从 1 爬坡(首轮即触发抽取)。 */
18
+ export declare function freshWarmup(): WarmupState;
19
+ export declare function emptyPending(): PendingBuckets;
20
+ /** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
21
+ * 旧格式条目(无 sessionId)归 legacy 组;warmup 缺省 = 全新起步。 */
22
+ export declare function loadPending(file: string, logger?: MemoryLogger): Promise<{
23
+ buckets: PendingBuckets;
24
+ warmup: WarmupState;
25
+ }>;
26
+ /** 按会话分组(会话切片):组按首条时间排序、组内按时间稳定排序——
27
+ * 蒸馏的一切触发都以切片为单位,切片内永不跨会话混装(ADR-0003)。 */
28
+ export declare function groupPendingBySession(messages: PendingMessage[]): Array<{
29
+ sessionId: string;
30
+ messages: PendingMessage[];
31
+ }>;
32
+ /** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
33
+ export declare function savePending(file: string, buckets: PendingBuckets, warmup?: WarmupState): Promise<void>;
34
+ export declare function pendingPathFor(dataDir: string): string;
35
+ /** 三档 key(调度/遍历用)。 */
36
+ export declare const PENDING_MODES: readonly ExtractMode[];
@@ -0,0 +1,103 @@
1
+ /**
2
+ * 未蒸馏缓冲持久化(pending.json):
3
+ * 按档分桶的"已捕获但尚未成功蒸馏"消息——既包括抽取失败的待重试消息,
4
+ * 也包括攒够触发阈值之前的消息。跨重启不丢(CONTEXT.md「未蒸馏缓冲」)。
5
+ *
6
+ * 与 state.json / session-modes.json 同款原子写;读取宽容(坏行丢弃、坏文件空桶起步)。
7
+ */
8
+ import * as fs from 'node:fs/promises';
9
+ import * as path from 'node:path';
10
+ import { atomicWriteJson } from '../util/io.js';
11
+ /** 旧格式(无 sessionId 字段)条目加载时归属的会话组。 */
12
+ export const LEGACY_SESSION = 'legacy';
13
+ /** 全新起步:三档都从 1 爬坡(首轮即触发抽取)。 */
14
+ export function freshWarmup() {
15
+ return { auto: 1, chat: 1, work: 1 };
16
+ }
17
+ export function emptyPending() {
18
+ return { auto: [], chat: [], work: [] };
19
+ }
20
+ function isMessage(m) {
21
+ if (!m || typeof m !== 'object')
22
+ return false;
23
+ const r = m;
24
+ return typeof r.id === 'string' && typeof r.content === 'string' && (r.role === 'user' || r.role === 'assistant');
25
+ }
26
+ /** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
27
+ * 旧格式条目(无 sessionId)归 legacy 组;warmup 缺省 = 全新起步。 */
28
+ export async function loadPending(file, logger) {
29
+ const out = emptyPending();
30
+ // 直接读文件而非 readJsonIfExists:后者把"不存在"、"不可读"、"JSON 损坏"压成同一个 undefined,
31
+ // 会让形状/内容损坏静默降级为空桶(与"合法空 pending"不可区分)。此处区分 ENOENT(正常)与其余(需告警)。
32
+ let raw;
33
+ try {
34
+ raw = JSON.parse(await fs.readFile(file, 'utf-8'));
35
+ }
36
+ catch (err) {
37
+ if (err?.code !== 'ENOENT') {
38
+ logger?.warn(`[memory] 未蒸馏缓冲读取失败(按空桶起步): ${file} — ${err instanceof Error ? err.message : String(err)}`);
39
+ }
40
+ return { buckets: out, warmup: freshWarmup() };
41
+ }
42
+ if (!raw || typeof raw !== 'object' || !raw.buckets || typeof raw.buckets !== 'object') {
43
+ logger?.warn(`[memory] 未蒸馏缓冲格式不符(缺 buckets,按空桶起步): ${file}`);
44
+ return { buckets: out, warmup: freshWarmup() };
45
+ }
46
+ let dropped = 0;
47
+ let legacy = 0;
48
+ for (const key of ['auto', 'chat', 'work']) {
49
+ const arr = raw.buckets[key];
50
+ if (!Array.isArray(arr))
51
+ continue;
52
+ for (const m of arr) {
53
+ if (!isMessage(m)) {
54
+ dropped++;
55
+ continue;
56
+ }
57
+ const sid = m.sessionId;
58
+ if (typeof sid === 'string' && sid) {
59
+ out[key].push({ ...m, sessionId: sid });
60
+ }
61
+ else {
62
+ legacy++;
63
+ out[key].push({ ...m, sessionId: LEGACY_SESSION });
64
+ }
65
+ }
66
+ }
67
+ if (dropped > 0)
68
+ logger?.warn(`[memory] 未蒸馏缓冲文件含 ${dropped} 条坏记录,已丢弃`);
69
+ if (legacy > 0)
70
+ logger?.info(`[memory] 未蒸馏缓冲含 ${legacy} 条旧格式条目,归入 legacy 会话组`);
71
+ const warmup = freshWarmup();
72
+ for (const key of ['auto', 'chat', 'work']) {
73
+ const w = raw.warmup?.[key];
74
+ if (typeof w === 'number' && Number.isFinite(w) && w >= 0)
75
+ warmup[key] = Math.floor(w);
76
+ }
77
+ return { buckets: out, warmup };
78
+ }
79
+ /** 按会话分组(会话切片):组按首条时间排序、组内按时间稳定排序——
80
+ * 蒸馏的一切触发都以切片为单位,切片内永不跨会话混装(ADR-0003)。 */
81
+ export function groupPendingBySession(messages) {
82
+ const groups = new Map();
83
+ for (const m of messages) {
84
+ const g = groups.get(m.sessionId);
85
+ if (g)
86
+ g.push(m);
87
+ else
88
+ groups.set(m.sessionId, [m]);
89
+ }
90
+ return [...groups.entries()]
91
+ .map(([sessionId, msgs]) => ({ sessionId, messages: [...msgs].sort((a, b) => a.timestamp - b.timestamp) }))
92
+ .sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
93
+ }
94
+ /** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
95
+ export async function savePending(file, buckets, warmup) {
96
+ const payload = { version: 1, buckets, ...(warmup ? { warmup } : {}) };
97
+ await atomicWriteJson(file, payload);
98
+ }
99
+ export function pendingPathFor(dataDir) {
100
+ return path.join(dataDir, 'pending.json');
101
+ }
102
+ /** 三档 key(调度/遍历用)。 */
103
+ export const PENDING_MODES = ['auto', 'chat', 'work'];