dsh-layered-memory 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +43 -13
- package/README.md +28 -13
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/bench-dialog.svg +49 -55
- package/dist/bench-control.d.ts +35 -0
- package/dist/bench-control.js +16 -0
- package/dist/client.js +173 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.js +4 -0
- package/dist/embedding-worker.cjs +176 -0
- package/dist/hooks/recall.d.ts +28 -1
- package/dist/hooks/recall.js +72 -13
- package/dist/index.d.ts +6 -0
- package/dist/index.js +18 -3
- package/dist/llm-usage.d.ts +27 -0
- package/dist/llm-usage.js +39 -0
- package/dist/llm.d.ts +3 -0
- package/dist/llm.js +6 -0
- package/dist/pipeline/l1.js +4 -2
- package/dist/pipeline/l2.js +1 -0
- package/dist/pipeline/l3.js +1 -0
- package/dist/pipeline/runner.d.ts +21 -0
- package/dist/pipeline/runner.js +83 -4
- package/dist/pipeline/trigger.d.ts +2 -0
- package/dist/pipeline/trigger.js +11 -0
- package/dist/prompts/l1-extraction.d.ts +7 -1
- package/dist/prompts/l1-extraction.js +12 -3
- package/dist/stats.d.ts +27 -1
- package/dist/stats.js +37 -2
- package/dist/store/embedding-source.d.ts +2 -1
- package/dist/store/embedding-source.js +7 -2
- package/dist/store/embedding.d.ts +2 -1
- package/dist/store/l0.d.ts +2 -0
- package/dist/store/l0.js +14 -4
- package/dist/store/l1.d.ts +11 -1
- package/dist/store/l1.js +26 -6
- package/dist/store/local-embedding.d.ts +69 -46
- package/dist/store/local-embedding.js +179 -75
- package/dist/store/recall-dedupe.d.ts +26 -0
- package/dist/store/recall-dedupe.js +138 -0
- package/dist/store/runtime-installer.d.ts +0 -2
- package/dist/store/runtime-installer.js +0 -6
- package/dist/store/search-utils.d.ts +17 -0
- package/dist/store/search-utils.js +29 -0
- package/dist/store/sqlite.d.ts +2 -0
- package/dist/store/sqlite.js +37 -9
- package/dist/types.d.ts +8 -0
- package/dist/types.js +8 -0
- package/dist/util/recall-budget.d.ts +2 -2
- package/dist/util/recall-budget.js +2 -2
- package/package.json +1 -1
|
@@ -1,20 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地嵌入服务(D1/D6 决策,worker 线程化版):transformers.js 的模型加载与
|
|
3
|
+
* ONNX 推理全部在 worker_threads 子线程执行(resources/embedding-worker.cjs),
|
|
4
|
+
* 本类只是主线程侧的协议代理——onnxruntime-node 的 run/loadModel 是主线程
|
|
5
|
+
* 同步调用,留在宿主事件循环会冻结整个 dsh 页面(0.8.6 修复的真实事故)。
|
|
6
|
+
*
|
|
7
|
+
* - 懒加载(D6):首次嵌入/warmup 才让 worker 加载模型,close()=terminate
|
|
8
|
+
* 释放线程与模型(嵌入源切走/关闭时调用),terminated 后不可复用;
|
|
9
|
+
* - 模型从数据目录 models/<id>/ 本地加载(worker 侧 env.allowRemoteModels=false);
|
|
10
|
+
* - channel 可注入(测试缝):smoke 用假通道验证协议与状态机,不触真模型;
|
|
11
|
+
* - callOpts.timeoutMs 经 Promise.race 钳制(迟到回复按 id 丢弃——推理在
|
|
12
|
+
* worker 线程无法真正取消,但主线程可以停止等待并降级 FTS);
|
|
13
|
+
* - 池化方式来自模型目录(BGE 系 CLS / Gemma 系 MEAN),normalize 交给
|
|
14
|
+
* pipeline 内建 L2 归一(与远程路径的 sanitizeAndNormalize 语义一致)。
|
|
15
|
+
*/
|
|
16
|
+
import * as path from 'node:path';
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
import { Worker } from 'node:worker_threads';
|
|
19
|
+
/** 默认 worker 资产路径:dist/store/ → dist/embedding-worker.cjs(构建期拷入)。 */
|
|
20
|
+
function defaultWorkerPath() {
|
|
21
|
+
return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'embedding-worker.cjs');
|
|
22
|
+
}
|
|
23
|
+
/** 真实通道:spawn worker_threads + 自增 id 配对 + 崩溃兜底拒绝。 */
|
|
24
|
+
class RealWorkerChannel {
|
|
25
|
+
worker;
|
|
26
|
+
pending = new Map();
|
|
27
|
+
nextId = 1;
|
|
28
|
+
terminated = false;
|
|
29
|
+
crashed;
|
|
30
|
+
crashCb;
|
|
31
|
+
constructor(workerPath, workerData) {
|
|
32
|
+
this.worker = new Worker(workerPath, { workerData });
|
|
33
|
+
this.worker.on('message', (msg) => {
|
|
34
|
+
if (msg && msg.type === 'fatal') {
|
|
35
|
+
this.failAll(`本地嵌入 worker 致命错误: ${msg.error ?? '未知'}`);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const id = msg.id;
|
|
39
|
+
if (typeof id !== 'number')
|
|
40
|
+
return;
|
|
41
|
+
const entry = this.pending.get(id);
|
|
42
|
+
if (!entry)
|
|
43
|
+
return; // 迟到回复(调用方已超时放弃)
|
|
44
|
+
this.pending.delete(id);
|
|
45
|
+
entry.resolve(msg);
|
|
46
|
+
});
|
|
47
|
+
// error(未捕获异常且 worker 未自处理)与 exit(含 fatal 后的退出)都兜底拒绝
|
|
48
|
+
this.worker.on('error', (err) => this.failAll(`本地嵌入 worker 线程错误: ${err.message}`));
|
|
49
|
+
this.worker.on('exit', (code) => {
|
|
50
|
+
if (!this.terminated)
|
|
51
|
+
this.failAll(`本地嵌入 worker 线程退出(code=${code})`);
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
request(call) {
|
|
55
|
+
// 已释放/已崩溃的通道快速拒绝——postMessage 到死线程是静默无回应,调用方会挂到超时
|
|
56
|
+
if (this.terminated)
|
|
57
|
+
return Promise.reject(new Error('嵌入 worker 已释放'));
|
|
58
|
+
if (this.crashed)
|
|
59
|
+
return Promise.reject(new Error(this.crashed));
|
|
60
|
+
const id = this.nextId++;
|
|
61
|
+
return new Promise((resolve, reject) => {
|
|
62
|
+
this.pending.set(id, { resolve, reject });
|
|
63
|
+
this.worker.postMessage({ ...call, id });
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
terminate() {
|
|
67
|
+
if (this.terminated)
|
|
68
|
+
return;
|
|
69
|
+
this.terminated = true;
|
|
70
|
+
this.failAll('嵌入 worker 已释放');
|
|
71
|
+
void this.worker.terminate();
|
|
72
|
+
}
|
|
73
|
+
setOnCrash(cb) {
|
|
74
|
+
this.crashCb = cb;
|
|
75
|
+
// 构造与回调注册之间发生的崩溃(spawn 即失败等)不丢通知
|
|
76
|
+
if (this.crashed)
|
|
77
|
+
cb(this.crashed);
|
|
78
|
+
}
|
|
79
|
+
failAll(error) {
|
|
80
|
+
for (const [, entry] of this.pending)
|
|
81
|
+
entry.reject(new Error(error));
|
|
82
|
+
this.pending.clear();
|
|
83
|
+
if (!this.terminated && !this.crashed) {
|
|
84
|
+
this.crashed = error;
|
|
85
|
+
this.crashCb?.(error);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
1
89
|
export class LocalEmbeddingService {
|
|
2
90
|
state = 'idle';
|
|
3
|
-
extractor = null;
|
|
4
|
-
loadPromise = null;
|
|
5
91
|
loadError = null;
|
|
6
|
-
|
|
92
|
+
channel;
|
|
7
93
|
entry;
|
|
8
|
-
loader;
|
|
9
94
|
logger;
|
|
10
|
-
|
|
11
|
-
maxInputChars;
|
|
12
|
-
constructor(entry, modelDir, loader, logger, maxInputChars) {
|
|
95
|
+
constructor(entry, modelDir, opts) {
|
|
13
96
|
this.entry = entry;
|
|
14
|
-
this.
|
|
15
|
-
|
|
16
|
-
this.
|
|
17
|
-
|
|
97
|
+
this.logger = opts.logger;
|
|
98
|
+
const maxInputChars = opts.maxInputChars && opts.maxInputChars > 0 ? opts.maxInputChars : 5000;
|
|
99
|
+
this.channel =
|
|
100
|
+
opts.channel ??
|
|
101
|
+
new RealWorkerChannel(opts.workerPath ?? defaultWorkerPath(), {
|
|
102
|
+
runtimeDir: opts.runtimeDir,
|
|
103
|
+
modelDir,
|
|
104
|
+
pooling: entry.pooling,
|
|
105
|
+
dtype: 'q8',
|
|
106
|
+
maxInputChars,
|
|
107
|
+
});
|
|
108
|
+
// 崩溃不自愈(换源/重启恢复):拒绝语义沿 EmbedHelper 降级链走 FTS
|
|
109
|
+
this.channel.setOnCrash((error) => {
|
|
110
|
+
if (this.state === 'terminated')
|
|
111
|
+
return;
|
|
112
|
+
this.state = 'failed';
|
|
113
|
+
this.loadError = error;
|
|
114
|
+
this.logger?.warn(`[memory] ${error}(本地嵌入转入 failed 态,换源或重启可恢复)`);
|
|
115
|
+
});
|
|
18
116
|
}
|
|
19
117
|
getDimensions() {
|
|
20
118
|
return this.entry.dims;
|
|
@@ -24,7 +122,7 @@ export class LocalEmbeddingService {
|
|
|
24
122
|
return { provider: 'local', model: this.entry.id, dimensions: this.entry.dims };
|
|
25
123
|
}
|
|
26
124
|
isReady() {
|
|
27
|
-
return this.state === 'ready'
|
|
125
|
+
return this.state === 'ready';
|
|
28
126
|
}
|
|
29
127
|
/** 状态(进度展示用)。 */
|
|
30
128
|
getState() {
|
|
@@ -33,91 +131,97 @@ export class LocalEmbeddingService {
|
|
|
33
131
|
getLoadError() {
|
|
34
132
|
return this.loadError;
|
|
35
133
|
}
|
|
36
|
-
/**
|
|
134
|
+
/** 后台预热:启动后让 worker 立即加载模型(幂等;失败态可重试)。 */
|
|
37
135
|
startWarmup() {
|
|
38
|
-
void this.
|
|
136
|
+
void this.waitForReady().catch(() => { });
|
|
39
137
|
}
|
|
40
|
-
/**
|
|
138
|
+
/** 等待模型就绪(warmup 协议;applyChain 的 warming 阶段与测试用)。 */
|
|
41
139
|
async waitForReady() {
|
|
42
|
-
|
|
140
|
+
if (this.state === 'ready')
|
|
141
|
+
return;
|
|
142
|
+
if (this.state === 'terminated') {
|
|
143
|
+
throw new Error('本地嵌入服务已释放(嵌入源已切换);本实例不可复用');
|
|
144
|
+
}
|
|
145
|
+
if (this.state !== 'failed')
|
|
146
|
+
this.state = 'loading';
|
|
147
|
+
const reply = await this.channel.request({ type: 'warmup' });
|
|
148
|
+
if (!reply.ok) {
|
|
149
|
+
this.applyLoadFailure(reply.error);
|
|
150
|
+
throw new Error(reply.error);
|
|
151
|
+
}
|
|
152
|
+
this.markReady();
|
|
43
153
|
}
|
|
44
|
-
async embed(text) {
|
|
45
|
-
const [vec] = await this.embedBatch([text]);
|
|
154
|
+
async embed(text, callOpts) {
|
|
155
|
+
const [vec] = await this.embedBatch([text], callOpts);
|
|
46
156
|
return vec;
|
|
47
157
|
}
|
|
48
|
-
async embedBatch(texts) {
|
|
158
|
+
async embedBatch(texts, callOpts) {
|
|
49
159
|
if (texts.length === 0)
|
|
50
160
|
return [];
|
|
51
161
|
if (this.state === 'terminated') {
|
|
52
162
|
throw new Error('本地嵌入服务已释放(嵌入源已切换);本实例不可复用');
|
|
53
163
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
164
|
+
if (this.state === 'failed') {
|
|
165
|
+
// 只有明确失败过的服务才抛错(EmbedHelper 捕获后降级 FTS);warmup 可重试
|
|
166
|
+
throw new Error(`本地嵌入模型加载失败: ${this.loadError ?? '未知原因'}(重启插件或重新下载模型可重试)`);
|
|
167
|
+
}
|
|
168
|
+
if (this.state !== 'ready')
|
|
169
|
+
this.state = 'loading';
|
|
170
|
+
// 单条请求(召回 query)带优先标记:worker 侧插队,不被 reindex 批次堵队尾
|
|
171
|
+
const reply = await this.requestWithTimeout({ type: 'embed', texts, priority: texts.length === 1 }, callOpts?.timeoutMs);
|
|
172
|
+
if (!reply.ok) {
|
|
173
|
+
if (reply.stage === 'load')
|
|
174
|
+
this.applyLoadFailure(reply.error);
|
|
175
|
+
else
|
|
176
|
+
this.markReady(); // 推理失败说明模型已加载成功(loading → ready),失败只属于这一次调用
|
|
177
|
+
throw new Error(reply.error);
|
|
61
178
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
179
|
+
if (reply.type !== 'embedded')
|
|
180
|
+
throw new Error(`嵌入 worker 返回异常消息类型: ${reply.type}`);
|
|
181
|
+
this.markReady();
|
|
182
|
+
for (const v of reply.vectors) {
|
|
183
|
+
if (v.length !== this.entry.dims) {
|
|
184
|
+
throw new Error(`本地嵌入维度不匹配:期望 ${this.entry.dims},得到 ${v.length}`);
|
|
185
|
+
}
|
|
67
186
|
}
|
|
68
|
-
return
|
|
187
|
+
return reply.vectors;
|
|
69
188
|
}
|
|
70
|
-
/**
|
|
71
|
-
*
|
|
189
|
+
/** 释放 worker 线程与模型(嵌入源切走/关闭时调用;幂等)。terminated 后不可
|
|
190
|
+
* 再复用——防止插件卸载/切走后残留的重嵌循环把模型重新加载常驻(内存泄漏)。 */
|
|
72
191
|
close() {
|
|
73
|
-
const ext = this.extractor;
|
|
74
|
-
this.extractor = null;
|
|
75
|
-
this.loadPromise = null;
|
|
76
192
|
this.state = 'terminated';
|
|
77
193
|
this.loadError = null;
|
|
194
|
+
this.channel.terminate();
|
|
195
|
+
}
|
|
196
|
+
/** 内层钳制(仅缩短):超时放弃等待(迟到回复由通道按 id 丢弃),调用方降级。 */
|
|
197
|
+
async requestWithTimeout(call, timeoutMs) {
|
|
198
|
+
if (!(timeoutMs && timeoutMs > 0))
|
|
199
|
+
return this.channel.request(call);
|
|
200
|
+
let timer;
|
|
78
201
|
try {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
202
|
+
return await Promise.race([
|
|
203
|
+
this.channel.request(call),
|
|
204
|
+
new Promise((_, reject) => {
|
|
205
|
+
timer = setTimeout(() => reject(new Error(`本地嵌入调用超时(${timeoutMs}ms),已放弃等待`)), timeoutMs);
|
|
206
|
+
}),
|
|
207
|
+
]);
|
|
82
208
|
}
|
|
83
|
-
|
|
84
|
-
|
|
209
|
+
finally {
|
|
210
|
+
// 先到者胜出后清掉另一个定时器(不清理会挂住引用至自然到期,raceRecallTimeout 同款)
|
|
211
|
+
if (timer)
|
|
212
|
+
clearTimeout(timer);
|
|
85
213
|
}
|
|
86
214
|
}
|
|
87
|
-
|
|
215
|
+
/** loading → ready 一次性日志(memory.log 时序可读性:启动到模型就绪的间隔)。 */
|
|
216
|
+
markReady() {
|
|
88
217
|
if (this.state === 'ready')
|
|
89
218
|
return;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
this.
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (mod.env) {
|
|
98
|
-
mod.env.allowRemoteModels = false;
|
|
99
|
-
if (mod.env.allowLocalModels !== undefined)
|
|
100
|
-
mod.env.allowLocalModels = true;
|
|
101
|
-
}
|
|
102
|
-
const extractor = await mod.pipeline('feature-extraction', this.modelDir, { dtype: 'q8' });
|
|
103
|
-
// loading 中被 close()(terminated):不得覆写状态复活(review C)——直接释放丢弃
|
|
104
|
-
if (this.state === 'terminated') {
|
|
105
|
-
const dispose = extractor.dispose;
|
|
106
|
-
if (dispose)
|
|
107
|
-
void Promise.resolve(dispose.call(extractor)).catch(() => { });
|
|
108
|
-
throw new Error('加载期间服务已释放');
|
|
109
|
-
}
|
|
110
|
-
this.extractor = extractor;
|
|
111
|
-
this.state = 'ready';
|
|
112
|
-
this.logger?.info(`[memory] 本地嵌入模型就绪: ${this.entry.id}(dims=${this.entry.dims},pooling=${this.entry.pooling})`);
|
|
113
|
-
}
|
|
114
|
-
catch (err) {
|
|
115
|
-
this.state = 'failed';
|
|
116
|
-
this.loadError = err instanceof Error ? err.message : String(err);
|
|
117
|
-
this.logger?.warn(`[memory] 本地嵌入模型加载失败(${this.entry.id}): ${this.loadError}`);
|
|
118
|
-
throw err;
|
|
119
|
-
}
|
|
120
|
-
})();
|
|
121
|
-
return this.loadPromise;
|
|
219
|
+
this.state = 'ready';
|
|
220
|
+
this.logger?.info(`[memory] 本地嵌入模型就绪: ${this.entry.id}(dims=${this.entry.dims},pooling=${this.entry.pooling})`);
|
|
221
|
+
}
|
|
222
|
+
applyLoadFailure(error) {
|
|
223
|
+
this.state = 'failed';
|
|
224
|
+
this.loadError = error;
|
|
225
|
+
this.logger?.warn(`[memory] 本地嵌入模型加载失败(${this.entry.id}): ${error}`);
|
|
122
226
|
}
|
|
123
227
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { MemoryLogger } from '../types.js';
|
|
2
|
+
/** 会话条目上限(按 updatedAt 淘汰最旧;防文件无限增长)。 */
|
|
3
|
+
export declare const RECALL_DEDUPE_SESSION_CAP = 200;
|
|
4
|
+
/** 单会话记录 id 上限(按插入序淘汰最旧;Set 迭代序即插入序)。 */
|
|
5
|
+
export declare const RECALL_DEDUPE_IDS_CAP = 512;
|
|
6
|
+
export declare class RecallDedupeStore {
|
|
7
|
+
private readonly logger?;
|
|
8
|
+
private readonly file;
|
|
9
|
+
private readonly entries;
|
|
10
|
+
private persistFailed;
|
|
11
|
+
/** 串行化持久化写(避免并发原子写撞临时文件名);init 链最前(先载入再落盘,防丢更新)。 */
|
|
12
|
+
private writeChain;
|
|
13
|
+
constructor(dataDir: string, logger?: MemoryLogger | undefined);
|
|
14
|
+
/** 载入持久化映射(合并进内存——构造与载入之间发生的 mark 不丢);失败降级内存态。 */
|
|
15
|
+
private init;
|
|
16
|
+
/** 该会话的已注入集合(热路径同步读;未出现过的会话返回空集合,惰性建条)。 */
|
|
17
|
+
seen(sessionId: string): Set<string>;
|
|
18
|
+
/** 标记本轮实际注入的记录 id(写穿;调用方保证只传模型真实看到的条目)。 */
|
|
19
|
+
mark(sessionId: string, recordIds: string[]): void;
|
|
20
|
+
/** 清空该会话的记录(compact/clear 后上下文已丢失,记忆需可重新注入)。 */
|
|
21
|
+
reset(sessionId: string): void;
|
|
22
|
+
/** 等待在途持久化写完成(测试/停机用)。 */
|
|
23
|
+
flush(): Promise<void>;
|
|
24
|
+
private persist;
|
|
25
|
+
private serialize;
|
|
26
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 召回去重存储:sessionId → 已注入 L1 记录 id 集合的持久化映射。
|
|
3
|
+
*
|
|
4
|
+
* 语义(2026-08-24 设计共识):
|
|
5
|
+
* - 同会话内已注入过的记忆不再重复注入(模型上下文已持有,重复注入浪费 token);
|
|
6
|
+
* - 压制粒度 = 记录 id——去重合并更新会换新 id,新内容天然解除压制重新注入;
|
|
7
|
+
* - compact/clear 事件重置(上下文被压缩/清空,注入内容已丢失);resume 不重置;
|
|
8
|
+
* - 热路径(召回 pre-step)同步内存读取,mark/reset 写穿持久化(session-modes 同款:
|
|
9
|
+
* 串行化原子写 + 失败降级内存态),任何 I/O 失败绝不抛进召回路径。
|
|
10
|
+
*/
|
|
11
|
+
import * as path from 'node:path';
|
|
12
|
+
import { errDetail } from '../util/filelog.js';
|
|
13
|
+
import { atomicWriteJson, ensureDir, readJsonIfExists } from './io.js';
|
|
14
|
+
/** 会话条目上限(按 updatedAt 淘汰最旧;防文件无限增长)。 */
|
|
15
|
+
export const RECALL_DEDUPE_SESSION_CAP = 200;
|
|
16
|
+
/** 单会话记录 id 上限(按插入序淘汰最旧;Set 迭代序即插入序)。 */
|
|
17
|
+
export const RECALL_DEDUPE_IDS_CAP = 512;
|
|
18
|
+
/** 条目过期清理(90 天未更新即丢弃,与 session-modes 同款量级)。 */
|
|
19
|
+
const PRUNE_MS = 90 * 24 * 3600_000;
|
|
20
|
+
export class RecallDedupeStore {
|
|
21
|
+
logger;
|
|
22
|
+
file;
|
|
23
|
+
entries = new Map();
|
|
24
|
+
persistFailed = false;
|
|
25
|
+
/** 串行化持久化写(避免并发原子写撞临时文件名);init 链最前(先载入再落盘,防丢更新)。 */
|
|
26
|
+
writeChain;
|
|
27
|
+
constructor(dataDir, logger) {
|
|
28
|
+
this.logger = logger;
|
|
29
|
+
this.file = path.join(dataDir, 'recall-dedupe.json');
|
|
30
|
+
this.writeChain = this.init();
|
|
31
|
+
}
|
|
32
|
+
/** 载入持久化映射(合并进内存——构造与载入之间发生的 mark 不丢);失败降级内存态。 */
|
|
33
|
+
async init() {
|
|
34
|
+
const data = await readJsonIfExists(this.file);
|
|
35
|
+
if (!data?.sessions || typeof data.sessions !== 'object')
|
|
36
|
+
return;
|
|
37
|
+
const now = Date.now();
|
|
38
|
+
let count = 0;
|
|
39
|
+
for (const [sid, entry] of Object.entries(data.sessions)) {
|
|
40
|
+
if (!Array.isArray(entry?.recordIds))
|
|
41
|
+
continue;
|
|
42
|
+
if (now - (entry.updatedAt ?? 0) > PRUNE_MS)
|
|
43
|
+
continue;
|
|
44
|
+
const existing = this.entries.get(sid);
|
|
45
|
+
if (existing) {
|
|
46
|
+
// 合并:构造后、载入完成前已发生的 mark(保留较大 updatedAt)
|
|
47
|
+
for (const id of entry.recordIds)
|
|
48
|
+
existing.ids.add(id);
|
|
49
|
+
existing.updatedAt = Math.max(existing.updatedAt, entry.updatedAt ?? 0);
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
this.entries.set(sid, { ids: new Set(entry.recordIds), updatedAt: entry.updatedAt ?? now });
|
|
53
|
+
}
|
|
54
|
+
count++;
|
|
55
|
+
}
|
|
56
|
+
if (count > 0)
|
|
57
|
+
this.logger?.info(`[memory] 召回去重记录载入 ${count} 个会话`);
|
|
58
|
+
}
|
|
59
|
+
/** 该会话的已注入集合(热路径同步读;未出现过的会话返回空集合,惰性建条)。 */
|
|
60
|
+
seen(sessionId) {
|
|
61
|
+
let entry = this.entries.get(sessionId);
|
|
62
|
+
if (!entry) {
|
|
63
|
+
entry = { ids: new Set(), updatedAt: 0 };
|
|
64
|
+
this.entries.set(sessionId, entry);
|
|
65
|
+
}
|
|
66
|
+
return entry.ids;
|
|
67
|
+
}
|
|
68
|
+
/** 标记本轮实际注入的记录 id(写穿;调用方保证只传模型真实看到的条目)。 */
|
|
69
|
+
mark(sessionId, recordIds) {
|
|
70
|
+
if (recordIds.length === 0)
|
|
71
|
+
return;
|
|
72
|
+
const ids = this.seen(sessionId);
|
|
73
|
+
for (const id of recordIds)
|
|
74
|
+
ids.add(id);
|
|
75
|
+
// 插入序淘汰最旧(Set 迭代序 = 插入序)
|
|
76
|
+
while (ids.size > RECALL_DEDUPE_IDS_CAP) {
|
|
77
|
+
const oldest = ids.values().next().value;
|
|
78
|
+
if (oldest === undefined)
|
|
79
|
+
break;
|
|
80
|
+
ids.delete(oldest);
|
|
81
|
+
}
|
|
82
|
+
const entry = this.entries.get(sessionId);
|
|
83
|
+
entry.updatedAt = Date.now();
|
|
84
|
+
this.writeChain = this.writeChain.then(() => this.persist());
|
|
85
|
+
}
|
|
86
|
+
/** 清空该会话的记录(compact/clear 后上下文已丢失,记忆需可重新注入)。 */
|
|
87
|
+
reset(sessionId) {
|
|
88
|
+
if (!this.entries.has(sessionId))
|
|
89
|
+
return;
|
|
90
|
+
this.entries.delete(sessionId);
|
|
91
|
+
this.writeChain = this.writeChain.then(() => this.persist());
|
|
92
|
+
}
|
|
93
|
+
/** 等待在途持久化写完成(测试/停机用)。 */
|
|
94
|
+
flush() {
|
|
95
|
+
return this.writeChain;
|
|
96
|
+
}
|
|
97
|
+
async persist() {
|
|
98
|
+
try {
|
|
99
|
+
await ensureDir(path.dirname(this.file));
|
|
100
|
+
await atomicWriteJson(this.file, this.serialize());
|
|
101
|
+
this.persistFailed = false;
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
if (!this.persistFailed) {
|
|
105
|
+
this.persistFailed = true;
|
|
106
|
+
this.logger?.warn(`[memory] 召回去重持久化失败(降级内存态): ${errDetail(err)}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
serialize() {
|
|
111
|
+
const now = Date.now();
|
|
112
|
+
// 超期清理 + 条数上限(按 updatedAt 淘汰最旧)
|
|
113
|
+
for (const [sid, e] of this.entries) {
|
|
114
|
+
if (now - e.updatedAt > PRUNE_MS && e.updatedAt > 0)
|
|
115
|
+
this.entries.delete(sid);
|
|
116
|
+
}
|
|
117
|
+
while (this.entries.size > RECALL_DEDUPE_SESSION_CAP) {
|
|
118
|
+
let oldest;
|
|
119
|
+
let oldestAt = Infinity;
|
|
120
|
+
for (const [sid, e] of this.entries) {
|
|
121
|
+
if (e.updatedAt > 0 && e.updatedAt < oldestAt) {
|
|
122
|
+
oldest = sid;
|
|
123
|
+
oldestAt = e.updatedAt;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (oldest === undefined)
|
|
127
|
+
break; // 只剩惰性空条目(updatedAt=0),不占文件体积可留待过期清理
|
|
128
|
+
this.entries.delete(oldest);
|
|
129
|
+
}
|
|
130
|
+
const sessions = {};
|
|
131
|
+
for (const [sid, e] of this.entries) {
|
|
132
|
+
if (e.ids.size === 0)
|
|
133
|
+
continue; // 空集合不落盘
|
|
134
|
+
sessions[sid] = { recordIds: [...e.ids], updatedAt: e.updatedAt };
|
|
135
|
+
}
|
|
136
|
+
return { version: 1, sessions };
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -60,8 +60,6 @@ export declare class RuntimeInstaller {
|
|
|
60
60
|
* runNpm 起跑前复查即不再起新进程(否则回退的 npm 会跑到自然结束且无法再取消)。
|
|
61
61
|
*/
|
|
62
62
|
cancel(): boolean;
|
|
63
|
-
/** 从 runtime 目录解析已安装的 transformers 模块(LocalEmbeddingService 用)。 */
|
|
64
|
-
resolveModule(): unknown;
|
|
65
63
|
private pushLine;
|
|
66
64
|
/** 跑一次 npm 子进程(采集尾行 + 超时 kill),返回退出码(null = 被杀死/启动失败)。 */
|
|
67
65
|
private runNpm;
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
* - 幂等:已装版本 == 目标版本直接就绪;版本漂移(插件升级换了钉死版本)重装覆盖。
|
|
15
15
|
*/
|
|
16
16
|
import { spawn } from 'node:child_process';
|
|
17
|
-
import { createRequire } from 'node:module';
|
|
18
17
|
import { promises as fs } from 'node:fs';
|
|
19
18
|
import * as path from 'node:path';
|
|
20
19
|
import { fileURLToPath } from 'node:url';
|
|
@@ -139,11 +138,6 @@ export class RuntimeInstaller {
|
|
|
139
138
|
this.child?.kill();
|
|
140
139
|
return true;
|
|
141
140
|
}
|
|
142
|
-
/** 从 runtime 目录解析已安装的 transformers 模块(LocalEmbeddingService 用)。 */
|
|
143
|
-
resolveModule() {
|
|
144
|
-
const req = createRequire(path.join(this.runtimeDir, 'package.json'));
|
|
145
|
-
return req(RuntimeInstaller.packageName);
|
|
146
|
-
}
|
|
147
141
|
pushLine(line) {
|
|
148
142
|
const lines = this.progress.lastLines;
|
|
149
143
|
lines.push(line.length > 300 ? line.slice(0, 300) + '…' : line);
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
/** 标准 RRF 常数(原论文值);k 越大越偏向低排名项(分布更平滑)。 */
|
|
2
2
|
export declare const RRF_K = 60;
|
|
3
|
+
/** 衰减地板(#29 时效加权的安全边界):老记忆最多损失一半排序分,永不沉底。
|
|
4
|
+
* 内部常量不进配置——它是安全机制不是调参旋钮。 */
|
|
5
|
+
export declare const DECAY_FLOOR = 0.5;
|
|
6
|
+
/**
|
|
7
|
+
* 时效衰减加权(#29,读路径专用):score × max(FLOOR, 0.5^(Δ天/半衰期)) 后重排序。
|
|
8
|
+
*
|
|
9
|
+
* - Δ 按 updated_at(内容版本时间)起算,缺失/非法按最老 → 地板接管(零特判分支);
|
|
10
|
+
* - 乘法保相关性主导:只在相关度相近的候选之间轮转名次(名额新鲜度),不淘汰不
|
|
11
|
+
* 硬过滤——score≈0 的新记忆乘什么都是 ≈0;hit 的原 score 字段不被改写(排序用
|
|
12
|
+
* 加权分,展示仍反映检索相关度);
|
|
13
|
+
* - halfLifeDays ≤ 0 直接原样返回(开关关闭);
|
|
14
|
+
* - 仅用于召回/工具检索;searchCandidates(去重候选)不得应用——写路径找同语义
|
|
15
|
+
* 旧记录要无视新旧,衰减会让去重漏检(同事实双记录)。
|
|
16
|
+
*/
|
|
17
|
+
export declare function applyDecayWeight<T extends {
|
|
18
|
+
score: number;
|
|
19
|
+
}>(hits: T[], halfLifeDays: number, updatedAtOf: (hit: T) => number | undefined, now?: number): T[];
|
|
3
20
|
/**
|
|
4
21
|
* RRF 融合多个已排序列表:每项得分 = 各列表 1/(k + rank + 1) 之和。
|
|
5
22
|
* 出现在多个列表的项得分累加,按得分降序返回(附 rrfScore)。
|
|
@@ -10,6 +10,35 @@
|
|
|
10
10
|
import { tokenize } from '../util/text.js';
|
|
11
11
|
/** 标准 RRF 常数(原论文值);k 越大越偏向低排名项(分布更平滑)。 */
|
|
12
12
|
export const RRF_K = 60;
|
|
13
|
+
/** 衰减地板(#29 时效加权的安全边界):老记忆最多损失一半排序分,永不沉底。
|
|
14
|
+
* 内部常量不进配置——它是安全机制不是调参旋钮。 */
|
|
15
|
+
export const DECAY_FLOOR = 0.5;
|
|
16
|
+
/**
|
|
17
|
+
* 时效衰减加权(#29,读路径专用):score × max(FLOOR, 0.5^(Δ天/半衰期)) 后重排序。
|
|
18
|
+
*
|
|
19
|
+
* - Δ 按 updated_at(内容版本时间)起算,缺失/非法按最老 → 地板接管(零特判分支);
|
|
20
|
+
* - 乘法保相关性主导:只在相关度相近的候选之间轮转名次(名额新鲜度),不淘汰不
|
|
21
|
+
* 硬过滤——score≈0 的新记忆乘什么都是 ≈0;hit 的原 score 字段不被改写(排序用
|
|
22
|
+
* 加权分,展示仍反映检索相关度);
|
|
23
|
+
* - halfLifeDays ≤ 0 直接原样返回(开关关闭);
|
|
24
|
+
* - 仅用于召回/工具检索;searchCandidates(去重候选)不得应用——写路径找同语义
|
|
25
|
+
* 旧记录要无视新旧,衰减会让去重漏检(同事实双记录)。
|
|
26
|
+
*/
|
|
27
|
+
export function applyDecayWeight(hits, halfLifeDays, updatedAtOf, now = Date.now()) {
|
|
28
|
+
if (!(halfLifeDays > 0) || hits.length === 0)
|
|
29
|
+
return hits;
|
|
30
|
+
const weight = (h) => {
|
|
31
|
+
const t = updatedAtOf(h);
|
|
32
|
+
if (t == null || !Number.isFinite(t))
|
|
33
|
+
return DECAY_FLOOR;
|
|
34
|
+
const days = Math.max(0, (now - t) / 86_400_000);
|
|
35
|
+
return Math.max(DECAY_FLOOR, 0.5 ** (days / halfLifeDays));
|
|
36
|
+
};
|
|
37
|
+
return hits
|
|
38
|
+
.map((h) => ({ h, weighted: h.score * weight(h) }))
|
|
39
|
+
.sort((a, b) => b.weighted - a.weighted)
|
|
40
|
+
.map((x) => x.h);
|
|
41
|
+
}
|
|
13
42
|
/**
|
|
14
43
|
* RRF 融合多个已排序列表:每项得分 = 各列表 1/(k + rank + 1) 之和。
|
|
15
44
|
* 出现在多个列表的项得分累加,按得分降序返回(附 rrfScore)。
|
package/dist/store/sqlite.d.ts
CHANGED
|
@@ -148,6 +148,8 @@ export declare class MemoryDb {
|
|
|
148
148
|
countL0(): number;
|
|
149
149
|
/** 统计 recorded_at >= iso 的消息数(状态面板"今日捕获"用)。 */
|
|
150
150
|
countL0Since(iso: string): number;
|
|
151
|
+
/** 统计某会话已捕获消息数(session-stats 数据源;idx_l0_session_id 索引点查)。 */
|
|
152
|
+
countL0BySession(sessionId: string): number;
|
|
151
153
|
/** 按会话取最近消息(时间升序返回;走 idx_l0_session_id 索引)。
|
|
152
154
|
* 蒸馏背景参考专用——按会话现查替代全局内存数组(ADR-0003)。 */
|
|
153
155
|
recentL0BySession(sessionId: string, limit: number): L0MessageRecord[];
|
package/dist/store/sqlite.js
CHANGED
|
@@ -670,12 +670,19 @@ export class MemoryDb {
|
|
|
670
670
|
/** 事务内的单条写入体(upsertL1 / upsertL1Batch 共用;调用方负责 BEGIN/COMMIT)。 */
|
|
671
671
|
upsertL1InTx(record, embedding) {
|
|
672
672
|
const ts = timestampsToDb(record.timestamps);
|
|
673
|
+
// 绑定层字段兜底(取 schema 列默认):旧版 JSONL 等外部数据缺字段时 undefined
|
|
674
|
+
// 无法绑定(node:sqlite 拒绝绑定),曾致旧版导入逐条全挂、每次启动无限重试(#28)。
|
|
675
|
+
// 主表与 FTS 两条语句共用同源归一化值;type 归一化后 familyForType 也不再收到 undefined。
|
|
676
|
+
const type = record.type ?? '';
|
|
677
|
+
const priority = record.priority ?? 50;
|
|
678
|
+
const sceneName = record.scene_name ?? '';
|
|
679
|
+
const family = record.family ?? familyForType(type);
|
|
673
680
|
// 防御性 FTS 删除的前置点查(主键索引,微秒级):record_id 在 FTS 表是 UNINDEXED,
|
|
674
681
|
// 按 id DELETE 是 O(N) 全表扫描——导入/重建/重嵌等"全新增"路径曾为每条记录白付一次
|
|
675
682
|
// 全扫(批量写整体 O(N²))。只有主表已有该行(覆盖/合并)才可能有旧 FTS 行需要删。
|
|
676
683
|
// 同批重复 id 也能正确处理:首条插入后,第二条的点查在同一事务内已见新行。
|
|
677
684
|
const ftsExisted = this.ftsAvailable ? this.stmtL1Exists.get(record.id) !== undefined : false;
|
|
678
|
-
this.stmtUpsertL1.run(record.id, record.content,
|
|
685
|
+
this.stmtUpsertL1.run(record.id, record.content, type, priority, sceneName, record.sessionId ?? 'default', record.version ?? 0, ts.str, ts.start, ts.end, toIso(record.createdAt), toIso(record.updatedAt), JSON.stringify(record.metadata ?? {}), family);
|
|
679
686
|
// vec0 不支持 ON CONFLICT → 先删后插;零向量跳过(cosine 未定义)
|
|
680
687
|
if (this.stmtDeleteL1Vec && this.stmtInsertL1Vec) {
|
|
681
688
|
this.stmtDeleteL1Vec.run(record.id);
|
|
@@ -688,7 +695,7 @@ export class MemoryDb {
|
|
|
688
695
|
if (this.ftsAvailable) {
|
|
689
696
|
if (ftsExisted)
|
|
690
697
|
this.stmtL1FtsDelete.run(record.id);
|
|
691
|
-
this.stmtL1FtsInsert.run(tokenizeForFts(record.content), record.content, record.id,
|
|
698
|
+
this.stmtL1FtsInsert.run(tokenizeForFts(record.content), record.content, record.id, type, priority, sceneName, record.sessionId ?? 'default', record.version ?? 0, ts.str, ts.start, ts.end, JSON.stringify(record.metadata ?? {}), family);
|
|
692
699
|
}
|
|
693
700
|
}
|
|
694
701
|
/** 批量删除 L1(元数据 + 向量 + FTS),返回删除条数。IN 按 ≤900 分块(避变量数上限)。 */
|
|
@@ -932,22 +939,29 @@ export class MemoryDb {
|
|
|
932
939
|
try {
|
|
933
940
|
this.db.exec('BEGIN');
|
|
934
941
|
for (let i = 0; i < records.length; i++) {
|
|
935
|
-
const
|
|
942
|
+
const rec = records[i];
|
|
943
|
+
// 绑定层字段兜底(取 schema 列默认):同 upsertL1InTx——外部数据缺字段时
|
|
944
|
+
// undefined 无法绑定(#28);主表/向量/FTS 共用同源归一化值
|
|
945
|
+
const sessionId = rec.sessionId ?? 'default';
|
|
946
|
+
const role = rec.role ?? '';
|
|
947
|
+
const content = rec.content ?? '';
|
|
948
|
+
const recordedAt = rec.recordedAt ?? '';
|
|
949
|
+
const timestamp = rec.timestamp ?? 0;
|
|
936
950
|
// 同 upsertL1 的点查预判:全新增路径跳过 UNINDEXED 列的 FTS 全扫删除
|
|
937
|
-
const ftsExisted = this.ftsAvailable ? this.stmtL0Exists.get(
|
|
938
|
-
this.stmtUpsertL0.run(
|
|
951
|
+
const ftsExisted = this.ftsAvailable ? this.stmtL0Exists.get(rec.id) !== undefined : false;
|
|
952
|
+
this.stmtUpsertL0.run(rec.id, sessionId, role, content, recordedAt, timestamp);
|
|
939
953
|
if (this.stmtDeleteL0Vec && this.stmtInsertL0Vec) {
|
|
940
|
-
this.stmtDeleteL0Vec.run(
|
|
954
|
+
this.stmtDeleteL0Vec.run(rec.id);
|
|
941
955
|
const vec = embeddings?.[i];
|
|
942
956
|
if (vec && !isZeroVector(vec)) {
|
|
943
|
-
this.stmtInsertL0Vec.run(
|
|
957
|
+
this.stmtInsertL0Vec.run(rec.id, vecToBuffer(vec), recordedAt);
|
|
944
958
|
}
|
|
945
959
|
}
|
|
946
960
|
if (this.ftsAvailable) {
|
|
947
961
|
// 同 upsertL1:FTS 失败冒泡触发整批回滚,禁止"删了没补"的索引空洞。
|
|
948
962
|
if (ftsExisted)
|
|
949
|
-
this.stmtL0FtsDelete.run(
|
|
950
|
-
this.stmtL0FtsInsert.run(tokenizeForFts(
|
|
963
|
+
this.stmtL0FtsDelete.run(rec.id);
|
|
964
|
+
this.stmtL0FtsInsert.run(tokenizeForFts(content), content, rec.id, sessionId, role, recordedAt, timestamp);
|
|
951
965
|
}
|
|
952
966
|
}
|
|
953
967
|
this.db.exec('COMMIT');
|
|
@@ -989,6 +1003,20 @@ export class MemoryDb {
|
|
|
989
1003
|
return 0;
|
|
990
1004
|
}
|
|
991
1005
|
}
|
|
1006
|
+
/** 统计某会话已捕获消息数(session-stats 数据源;idx_l0_session_id 索引点查)。 */
|
|
1007
|
+
countL0BySession(sessionId) {
|
|
1008
|
+
if (this.degraded)
|
|
1009
|
+
return 0;
|
|
1010
|
+
try {
|
|
1011
|
+
const row = this.db
|
|
1012
|
+
.prepare('SELECT COUNT(*) AS n FROM l0_conversations WHERE session_id = ?')
|
|
1013
|
+
.get(sessionId);
|
|
1014
|
+
return row?.n ?? 0;
|
|
1015
|
+
}
|
|
1016
|
+
catch {
|
|
1017
|
+
return 0;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
992
1020
|
/** 按会话取最近消息(时间升序返回;走 idx_l0_session_id 索引)。
|
|
993
1021
|
* 蒸馏背景参考专用——按会话现查替代全局内存数组(ADR-0003)。 */
|
|
994
1022
|
recentL0BySession(sessionId, limit) {
|