dsh-layered-memory 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +101 -81
- package/README.md +88 -80
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/dist/client.js +1100 -402
- package/dist/config.d.ts +12 -0
- package/dist/config.js +20 -17
- package/dist/hooks/capture.d.ts +17 -1
- package/dist/hooks/capture.js +45 -11
- package/dist/hooks/recall.d.ts +7 -0
- package/dist/hooks/recall.js +20 -4
- package/dist/index.d.ts +8 -0
- package/dist/index.js +107 -44
- package/dist/pipeline/rebuild.d.ts +2 -0
- package/dist/pipeline/rebuild.js +7 -0
- package/dist/pipeline/runner.d.ts +7 -1
- package/dist/pipeline/runner.js +22 -7
- package/dist/settings.js +79 -20
- package/dist/stats.d.ts +7 -1
- package/dist/stats.js +131 -14
- package/dist/store/download-queue.d.ts +71 -0
- package/dist/store/download-queue.js +313 -0
- package/dist/store/embedding-source.d.ts +160 -0
- package/dist/store/embedding-source.js +421 -0
- package/dist/store/embedding.d.ts +3 -1
- package/dist/store/embedding.js +5 -0
- package/dist/store/l0.d.ts +13 -5
- package/dist/store/l0.js +33 -6
- package/dist/store/l1.d.ts +15 -5
- package/dist/store/l1.js +40 -15
- package/dist/store/local-embedding.d.ts +64 -0
- package/dist/store/local-embedding.js +120 -0
- package/dist/store/model-catalog.d.ts +45 -0
- package/dist/store/model-catalog.js +78 -0
- package/dist/store/runtime-installer.d.ts +60 -0
- package/dist/store/runtime-installer.js +181 -0
- package/dist/store/sqlite.d.ts +53 -7
- package/dist/store/sqlite.js +339 -72
- package/dist/tools/index.js +6 -4
- package/dist/util/filelog.d.ts +2 -0
- package/dist/util/filelog.js +20 -3
- package/package.json +1 -1
- package/assets/readme/hero.svg +0 -58
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 嵌入源状态层 + 活切换管理器(D4/D5/D6 决策落地)。
|
|
3
|
+
*
|
|
4
|
+
* - 状态文件 embedding-source.json(写穿持久化,session-modes 同款:内存态 + 原子写
|
|
5
|
+
* + 写队列串行化);无文件 = remote(与历史行为完全一致,老用户无感);
|
|
6
|
+
* - 生效 = 部署上限 AND 运行时选择(仓库铁律):远程档要求静态四件套配齐,
|
|
7
|
+
* 本地档受 embedding.allowLocalModels 上限约束;
|
|
8
|
+
* - 活切换链(后台执行,RPC 立即返回 accepted,进度靠轮询):
|
|
9
|
+
* 安装运行时(首次本地)→ 预热加载 → 换服务 + swapProvider(维度变化 drop 向量表)
|
|
10
|
+
* → 后台全量重嵌(L1/L0 计数进度,可取消)→ 持久化状态。
|
|
11
|
+
* 失败语义:异常(安装失败/模型加载失败/db 拒绝)→ 状态不持久化,重启回到旧源;
|
|
12
|
+
* 重嵌取消/部分失败 → 已切换(物理表即新维度,meta 已同步),缺失向量由周期
|
|
13
|
+
* backfill 补齐——不回滚(回滚需要再 drop 一次表,得不偿失)。
|
|
14
|
+
*/
|
|
15
|
+
import { promises as fs } from 'node:fs';
|
|
16
|
+
import * as path from 'node:path';
|
|
17
|
+
import { NoopEmbeddingService, RemoteEmbeddingService } from './embedding.js';
|
|
18
|
+
import { catalogById, MODEL_CATALOG } from './model-catalog.js';
|
|
19
|
+
import { LocalEmbeddingService } from './local-embedding.js';
|
|
20
|
+
// ── 状态存储(写穿持久化) ──
|
|
21
|
+
export class EmbeddingSourceStore {
|
|
22
|
+
state = { source: 'remote', activeModel: null };
|
|
23
|
+
file;
|
|
24
|
+
writeQueue = Promise.resolve();
|
|
25
|
+
logger;
|
|
26
|
+
constructor(dataDir, logger) {
|
|
27
|
+
this.file = path.join(dataDir, 'embedding-source.json');
|
|
28
|
+
this.logger = logger;
|
|
29
|
+
}
|
|
30
|
+
get() {
|
|
31
|
+
return { ...this.state };
|
|
32
|
+
}
|
|
33
|
+
async init() {
|
|
34
|
+
try {
|
|
35
|
+
const raw = await fs.readFile(this.file, 'utf8');
|
|
36
|
+
const parsed = JSON.parse(raw);
|
|
37
|
+
if ((parsed.source === 'remote' || parsed.source === 'local' || parsed.source === 'off') &&
|
|
38
|
+
(parsed.activeModel === null || typeof parsed.activeModel === 'string')) {
|
|
39
|
+
this.state = { source: parsed.source, activeModel: parsed.activeModel };
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
this.logger?.warn('[memory] 嵌入源状态文件损坏,按默认 remote 起步');
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// 无文件 = 历史行为(跟随部署配置的远程嵌入)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async set(next) {
|
|
50
|
+
this.state = { source: next.source, activeModel: next.activeModel };
|
|
51
|
+
this.writeQueue = this.writeQueue.then(() => this.persist()).catch(() => { });
|
|
52
|
+
await this.writeQueue;
|
|
53
|
+
}
|
|
54
|
+
async persist() {
|
|
55
|
+
const tmp = this.file + '.tmp';
|
|
56
|
+
await fs.writeFile(tmp, JSON.stringify(this.state, null, 2), 'utf8');
|
|
57
|
+
await fs.rename(tmp, this.file);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** 远程档部署上限:静态四件套 + enabled。 */
|
|
61
|
+
export function remoteCeiling(cfg) {
|
|
62
|
+
const e = cfg.embedding;
|
|
63
|
+
return e.enabled && !!e.baseUrl && !!e.apiKey && !!e.model && e.dimensions > 0;
|
|
64
|
+
}
|
|
65
|
+
export async function resolveInitialEmbedding(cfg, sourceStore, downloader, makeLocal, logger) {
|
|
66
|
+
const state = sourceStore.get();
|
|
67
|
+
if (state.source === 'off') {
|
|
68
|
+
return { svc: new NoopEmbeddingService(), dims: 0 };
|
|
69
|
+
}
|
|
70
|
+
if (state.source === 'local') {
|
|
71
|
+
if (!cfg.embedding.allowLocalModels) {
|
|
72
|
+
logger?.warn('[memory] 嵌入源为 local 但部署已禁用本地模型(allowLocalModels=false),本次运行纯 FTS');
|
|
73
|
+
return { svc: new NoopEmbeddingService(), dims: 0, note: '部署配置已禁用本地嵌入模型' };
|
|
74
|
+
}
|
|
75
|
+
const entry = state.activeModel ? catalogById(state.activeModel) : undefined;
|
|
76
|
+
if (!entry) {
|
|
77
|
+
logger?.warn(`[memory] 嵌入源 local 的模型 ${state.activeModel} 不在目录,本次运行纯 FTS`);
|
|
78
|
+
return { svc: new NoopEmbeddingService(), dims: 0, note: '启用的模型不在内置目录' };
|
|
79
|
+
}
|
|
80
|
+
if (!(await downloader.isDownloaded(entry.id))) {
|
|
81
|
+
logger?.warn(`[memory] 本地模型 ${entry.id} 文件缺失(可能被清理),本次运行纯 FTS`);
|
|
82
|
+
return { svc: new NoopEmbeddingService(), dims: 0, note: '模型文件缺失,请重新下载' };
|
|
83
|
+
}
|
|
84
|
+
const svc = makeLocal(entry.id);
|
|
85
|
+
if (!svc)
|
|
86
|
+
return { svc: new NoopEmbeddingService(), dims: 0, note: '本地服务构造失败' };
|
|
87
|
+
return { svc, dims: entry.dims, providerInfo: { provider: 'local', model: entry.id, dimensions: entry.dims } };
|
|
88
|
+
}
|
|
89
|
+
// remote(默认)
|
|
90
|
+
if (!remoteCeiling(cfg)) {
|
|
91
|
+
return { svc: new NoopEmbeddingService(), dims: 0 };
|
|
92
|
+
}
|
|
93
|
+
const svc = new RemoteEmbeddingService({
|
|
94
|
+
baseUrl: cfg.embedding.baseUrl,
|
|
95
|
+
apiKey: cfg.embedding.apiKey,
|
|
96
|
+
model: cfg.embedding.model,
|
|
97
|
+
dimensions: cfg.embedding.dimensions,
|
|
98
|
+
maxInputChars: cfg.embedding.maxInputChars,
|
|
99
|
+
timeoutMs: cfg.embedding.timeoutMs,
|
|
100
|
+
logger,
|
|
101
|
+
});
|
|
102
|
+
return { svc, dims: cfg.embedding.dimensions, providerInfo: svc.getProviderInfo() };
|
|
103
|
+
}
|
|
104
|
+
/** 本地服务构造工厂(index.ts 的初始解析与 Manager 共用一份实现,防漂移)。 */
|
|
105
|
+
export function makeLocalServiceFactory(installer, downloader, logger) {
|
|
106
|
+
return (modelId) => {
|
|
107
|
+
const entry = catalogById(modelId);
|
|
108
|
+
if (!entry)
|
|
109
|
+
return null;
|
|
110
|
+
return new LocalEmbeddingService(entry, downloader.modelsDir(entry.id), () => Promise.resolve(installer.resolveModule()), logger);
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
export class EmbeddingManager {
|
|
114
|
+
sourceStore;
|
|
115
|
+
installer;
|
|
116
|
+
downloader;
|
|
117
|
+
deps;
|
|
118
|
+
current;
|
|
119
|
+
localSvc = null;
|
|
120
|
+
applyPhase = 'idle';
|
|
121
|
+
applyMessage = '';
|
|
122
|
+
applyStartedAt = 0;
|
|
123
|
+
applyBusy = false;
|
|
124
|
+
reindex = {
|
|
125
|
+
running: false,
|
|
126
|
+
l1Done: 0,
|
|
127
|
+
l1Total: 0,
|
|
128
|
+
l0Done: 0,
|
|
129
|
+
l0Total: 0,
|
|
130
|
+
startedAt: 0,
|
|
131
|
+
cancelled: false,
|
|
132
|
+
};
|
|
133
|
+
reindexCancel = false;
|
|
134
|
+
/** 停机标志:dispose 后应用链不再推进(防卸载后的孤儿重嵌/安装)。 */
|
|
135
|
+
disposedFlag = false;
|
|
136
|
+
/** 当前生效目标的 providerInfo(backfill/启动链的 meta 写入用——杜绝陈旧闭包)。 */
|
|
137
|
+
currentInfo;
|
|
138
|
+
/** 初始解析的降级说明(活切换成功后清空,防过期提示常驻)。 */
|
|
139
|
+
activeNote;
|
|
140
|
+
constructor(deps) {
|
|
141
|
+
this.deps = deps;
|
|
142
|
+
this.sourceStore = deps.sourceStore;
|
|
143
|
+
this.installer = deps.installer;
|
|
144
|
+
this.downloader = deps.downloader;
|
|
145
|
+
this.current = deps.initial.svc;
|
|
146
|
+
this.currentInfo = deps.initial.providerInfo;
|
|
147
|
+
this.activeNote = deps.initial.note;
|
|
148
|
+
// 启动即本地档:initial.svc 已是绑定真实运行时 loader 的 LocalEmbeddingService。
|
|
149
|
+
// 立即后台预热(不阻塞启动)——否则 L1/L0.reindex 与 EmbedHelper.batch 的
|
|
150
|
+
// vectorReady 短路都不会触发懒加载,缺失向量要等到首次召回才补(review A)
|
|
151
|
+
if (deps.initial.providerInfo?.provider === 'local') {
|
|
152
|
+
this.localSvc = this.current;
|
|
153
|
+
this.localSvc.startWarmup();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** 当前目标的 providerInfo(index.ts 的启动重嵌链/周期 backfill 写 meta 用)。 */
|
|
157
|
+
currentProviderInfo() {
|
|
158
|
+
return this.currentInfo;
|
|
159
|
+
}
|
|
160
|
+
/** 取消运行时安装(RPC:npm 卡死/用户主动放弃)。 */
|
|
161
|
+
cancelRuntimeInstall() {
|
|
162
|
+
return this.installer.cancel();
|
|
163
|
+
}
|
|
164
|
+
/** 当前生效服务(index.ts 初始建 store 用)。 */
|
|
165
|
+
getService() {
|
|
166
|
+
return this.current;
|
|
167
|
+
}
|
|
168
|
+
/** 构造绑定真实运行时 loader 的本地服务(deps.makeLocal 可注入,测试替换)。 */
|
|
169
|
+
makeLocalService(modelId) {
|
|
170
|
+
const factory = this.deps.makeLocal ?? makeLocalServiceFactory(this.installer, this.downloader, this.deps.logger);
|
|
171
|
+
return factory(modelId);
|
|
172
|
+
}
|
|
173
|
+
/** 活切换请求:验证通过即接受,后台执行应用链(进度轮询可见)。 */
|
|
174
|
+
requestSource(next) {
|
|
175
|
+
if (this.applyBusy)
|
|
176
|
+
return { accepted: false, error: '切换进行中,请等待完成' };
|
|
177
|
+
if (next.source === 'remote' && !remoteCeiling(this.deps.cfg)) {
|
|
178
|
+
return { accepted: false, error: '部署未配置远程嵌入(baseUrl/apiKey/model/dimensions 或 enabled),远程档不可选' };
|
|
179
|
+
}
|
|
180
|
+
if (next.source === 'local') {
|
|
181
|
+
if (!this.deps.cfg.embedding.allowLocalModels) {
|
|
182
|
+
return { accepted: false, error: '部署已禁用本地嵌入模型(allowLocalModels=false)' };
|
|
183
|
+
}
|
|
184
|
+
if (!next.activeModel || !catalogById(next.activeModel)) {
|
|
185
|
+
return { accepted: false, error: '请选择内置目录中的模型' };
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const state = {
|
|
189
|
+
source: next.source,
|
|
190
|
+
activeModel: next.source === 'local' ? next.activeModel : null,
|
|
191
|
+
};
|
|
192
|
+
this.applyBusy = true;
|
|
193
|
+
this.applyStartedAt = Date.now();
|
|
194
|
+
this.applyMessage = '';
|
|
195
|
+
void this.applyChain(state).finally(() => {
|
|
196
|
+
this.applyBusy = false;
|
|
197
|
+
});
|
|
198
|
+
return { accepted: true };
|
|
199
|
+
}
|
|
200
|
+
/** 下载启动(串行队列忙时拒绝);完成后自动做一次可加载性预热验证(D6)。 */
|
|
201
|
+
startDownload(modelId) {
|
|
202
|
+
if (!this.deps.cfg.embedding.allowLocalModels) {
|
|
203
|
+
return { ok: false, error: '部署已禁用本地嵌入模型' };
|
|
204
|
+
}
|
|
205
|
+
if (!catalogById(modelId))
|
|
206
|
+
return { ok: false, error: '未知模型' };
|
|
207
|
+
if (this.downloader.isBusy())
|
|
208
|
+
return { ok: false, error: '已有下载任务进行中' };
|
|
209
|
+
void this.downloader
|
|
210
|
+
.start(modelId)
|
|
211
|
+
.then(async (p) => {
|
|
212
|
+
if (p.phase !== 'done')
|
|
213
|
+
return;
|
|
214
|
+
// 下载完成自动预热验证:当前启用模型 → 直接 warmup;未启用的临时模型验完即释放
|
|
215
|
+
if (this.sourceStore.get().source === 'local' && this.sourceStore.get().activeModel === modelId) {
|
|
216
|
+
this.localSvc?.startWarmup();
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const scratch = this.makeLocalService(modelId);
|
|
220
|
+
if (!scratch)
|
|
221
|
+
return;
|
|
222
|
+
if (await this.installer.isReady()) {
|
|
223
|
+
// 预热只为验证可加载性:完成后必须释放(bge-m3 ~550MB,不关就常驻泄漏,
|
|
224
|
+
// 且 onnxruntime 持文件句柄会卡住该模型的删除)
|
|
225
|
+
scratch.startWarmup();
|
|
226
|
+
void scratch.waitForReady().then(() => scratch.close(), () => scratch.close());
|
|
227
|
+
}
|
|
228
|
+
})
|
|
229
|
+
.catch(() => {
|
|
230
|
+
/* 失败态在 downloader 进度里 */
|
|
231
|
+
});
|
|
232
|
+
return { ok: true };
|
|
233
|
+
}
|
|
234
|
+
cancelDownload() {
|
|
235
|
+
return this.downloader.cancel();
|
|
236
|
+
}
|
|
237
|
+
async deleteModel(modelId) {
|
|
238
|
+
const state = this.sourceStore.get();
|
|
239
|
+
if (state.source === 'local' && state.activeModel === modelId) {
|
|
240
|
+
return { ok: false, error: '该模型正在使用中,请先切换嵌入源' };
|
|
241
|
+
}
|
|
242
|
+
return this.downloader.deleteModel(modelId);
|
|
243
|
+
}
|
|
244
|
+
cancelReindex() {
|
|
245
|
+
if (!this.reindex.running)
|
|
246
|
+
return false;
|
|
247
|
+
this.reindexCancel = true;
|
|
248
|
+
return true;
|
|
249
|
+
}
|
|
250
|
+
/** 应用链/后台任务是否在跑(backfill 并发门禁用)。 */
|
|
251
|
+
isBusy() {
|
|
252
|
+
return this.applyBusy || this.reindex.running || this.downloader.isBusy();
|
|
253
|
+
}
|
|
254
|
+
/** 停机钩子(插件 dispose):取消 npm 安装、下载与重嵌——不留后台孤儿任务。 */
|
|
255
|
+
dispose() {
|
|
256
|
+
this.disposedFlag = true;
|
|
257
|
+
this.installer.cancel();
|
|
258
|
+
this.downloader.cancel();
|
|
259
|
+
this.cancelReindex();
|
|
260
|
+
this.localSvc?.close();
|
|
261
|
+
}
|
|
262
|
+
async applyChain(next) {
|
|
263
|
+
try {
|
|
264
|
+
let svc;
|
|
265
|
+
let providerInfo;
|
|
266
|
+
if (next.source === 'off') {
|
|
267
|
+
this.applyPhase = 'switching';
|
|
268
|
+
svc = new NoopEmbeddingService();
|
|
269
|
+
providerInfo = undefined;
|
|
270
|
+
this.currentInfo = undefined;
|
|
271
|
+
}
|
|
272
|
+
else if (next.source === 'remote') {
|
|
273
|
+
this.applyPhase = 'switching';
|
|
274
|
+
svc = new RemoteEmbeddingService({
|
|
275
|
+
baseUrl: this.deps.cfg.embedding.baseUrl,
|
|
276
|
+
apiKey: this.deps.cfg.embedding.apiKey,
|
|
277
|
+
model: this.deps.cfg.embedding.model,
|
|
278
|
+
dimensions: this.deps.cfg.embedding.dimensions,
|
|
279
|
+
maxInputChars: this.deps.cfg.embedding.maxInputChars,
|
|
280
|
+
timeoutMs: this.deps.cfg.embedding.timeoutMs,
|
|
281
|
+
logger: this.deps.logger,
|
|
282
|
+
});
|
|
283
|
+
providerInfo = svc.getProviderInfo();
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
// local:下载完整性前置校验 → 运行时 → 预热 → 切换
|
|
287
|
+
if (!(await this.downloader.isDownloaded(next.activeModel))) {
|
|
288
|
+
throw new Error('模型文件不完整(未下载或已损坏),请先完成下载');
|
|
289
|
+
}
|
|
290
|
+
if (!(await this.installer.isReady())) {
|
|
291
|
+
this.applyPhase = 'installing-runtime';
|
|
292
|
+
const ok = await this.installer.ensure();
|
|
293
|
+
if (!ok) {
|
|
294
|
+
throw new Error(`运行时安装失败: ${this.installer.getProgress().error ?? '未知原因'}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
if (this.disposedFlag)
|
|
298
|
+
throw new Error('插件已卸载,切换中止');
|
|
299
|
+
this.applyPhase = 'warming';
|
|
300
|
+
const local = this.makeLocalService(next.activeModel);
|
|
301
|
+
if (!local)
|
|
302
|
+
throw new Error('模型不在目录');
|
|
303
|
+
await local.waitForReady();
|
|
304
|
+
svc = local;
|
|
305
|
+
providerInfo = local.getProviderInfo();
|
|
306
|
+
this.applyPhase = 'switching';
|
|
307
|
+
}
|
|
308
|
+
// 换服务 + 换表(providerInfo 变化 → drop 向量表按新维度重建)
|
|
309
|
+
let needsReindex = false;
|
|
310
|
+
if (providerInfo) {
|
|
311
|
+
const swap = this.deps.db.swapProvider(providerInfo);
|
|
312
|
+
if (!swap.ok)
|
|
313
|
+
throw new Error(swap.error ?? '切换向量引擎失败');
|
|
314
|
+
needsReindex = swap.needsReindex;
|
|
315
|
+
// 物理表在 swap 成功那一刻已是新维度——meta 立即跟上(即使后续重嵌被取消/
|
|
316
|
+
// 部分失败):meta 的语义是"物理表现状",缺失行由 backfill 按 missing 计数补,
|
|
317
|
+
// 不依赖 meta。拖着不写会把"meta=旧 provider"留给下次比对埋雷。
|
|
318
|
+
this.deps.db.markEmbeddingSynced(providerInfo);
|
|
319
|
+
this.currentInfo = providerInfo;
|
|
320
|
+
}
|
|
321
|
+
const oldLocal = this.localSvc;
|
|
322
|
+
this.localSvc = next.source === 'local' ? svc : null;
|
|
323
|
+
this.current = svc;
|
|
324
|
+
this.deps.l0.setEmbeddingService(svc);
|
|
325
|
+
this.deps.l1.setEmbeddingService(svc);
|
|
326
|
+
oldLocal?.close();
|
|
327
|
+
let pendingNote = '';
|
|
328
|
+
if (needsReindex) {
|
|
329
|
+
if (this.disposedFlag)
|
|
330
|
+
throw new Error('插件已卸载,重嵌入中止');
|
|
331
|
+
this.applyPhase = 'reindexing';
|
|
332
|
+
const result = await this.reindexNow();
|
|
333
|
+
if (result.cancelled) {
|
|
334
|
+
pendingNote = ';重嵌入已取消,缺失向量由周期任务补齐';
|
|
335
|
+
this.deps.logger.warn('[memory] 重嵌入已取消,缺失向量将由周期补齐(检索暂按关键词降级)');
|
|
336
|
+
}
|
|
337
|
+
else if (result.failedTotal > 0) {
|
|
338
|
+
pendingNote = `;${result.failedTotal} 条向量待补齐(周期任务会补)`;
|
|
339
|
+
}
|
|
340
|
+
if (result.error)
|
|
341
|
+
throw new Error(result.error);
|
|
342
|
+
}
|
|
343
|
+
await this.sourceStore.set(next);
|
|
344
|
+
this.activeNote = undefined;
|
|
345
|
+
this.applyPhase = 'done';
|
|
346
|
+
this.applyMessage =
|
|
347
|
+
next.source === 'off'
|
|
348
|
+
? '已切换为关键词检索'
|
|
349
|
+
: '切换完成' + pendingNote;
|
|
350
|
+
}
|
|
351
|
+
catch (err) {
|
|
352
|
+
this.applyPhase = 'error';
|
|
353
|
+
this.applyMessage = err instanceof Error ? err.message : String(err);
|
|
354
|
+
this.deps.logger.warn(`[memory] 嵌入源切换失败(状态保持不变): ${this.applyMessage}`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
async reindexNow() {
|
|
358
|
+
this.reindexCancel = false;
|
|
359
|
+
this.reindex = { running: true, l1Done: 0, l1Total: 0, l0Done: 0, l0Total: 0, startedAt: Date.now(), cancelled: false };
|
|
360
|
+
let failedTotal = 0;
|
|
361
|
+
try {
|
|
362
|
+
const r1 = await this.deps.l1.reindex({
|
|
363
|
+
onProgress: (done, total) => {
|
|
364
|
+
this.reindex.l1Done = done;
|
|
365
|
+
this.reindex.l1Total = total;
|
|
366
|
+
},
|
|
367
|
+
shouldCancel: () => this.reindexCancel,
|
|
368
|
+
});
|
|
369
|
+
failedTotal += r1.failed;
|
|
370
|
+
const r0 = await this.deps.l0.reindex({
|
|
371
|
+
onProgress: (done, total) => {
|
|
372
|
+
this.reindex.l0Done = done;
|
|
373
|
+
this.reindex.l0Total = total;
|
|
374
|
+
},
|
|
375
|
+
shouldCancel: () => this.reindexCancel,
|
|
376
|
+
});
|
|
377
|
+
failedTotal += r0.failed;
|
|
378
|
+
const cancelled = !!(r1.cancelled || r0.cancelled);
|
|
379
|
+
this.reindex.cancelled = cancelled;
|
|
380
|
+
this.reindex.running = false;
|
|
381
|
+
return { cancelled, failedTotal };
|
|
382
|
+
}
|
|
383
|
+
catch (err) {
|
|
384
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
385
|
+
this.reindex.running = false;
|
|
386
|
+
this.reindex.error = message;
|
|
387
|
+
return { cancelled: false, failedTotal, error: `重嵌入失败: ${message}` };
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
/** RPC 快照(设置页嵌入区块数据源;client 忙时 1s 轮询)。 */
|
|
391
|
+
async snapshot() {
|
|
392
|
+
const status = await this.downloader.listStatus();
|
|
393
|
+
const models = MODEL_CATALOG.map((entry) => {
|
|
394
|
+
const s = status.find((x) => x.id === entry.id);
|
|
395
|
+
return {
|
|
396
|
+
id: entry.id,
|
|
397
|
+
name: entry.name,
|
|
398
|
+
dims: entry.dims,
|
|
399
|
+
contextTokens: entry.contextTokens,
|
|
400
|
+
tags: entry.tags,
|
|
401
|
+
description: entry.description,
|
|
402
|
+
totalBytes: s?.totalBytes ?? 0,
|
|
403
|
+
bytesOnDisk: s?.bytesOnDisk ?? 0,
|
|
404
|
+
state: s?.state ?? 'none',
|
|
405
|
+
};
|
|
406
|
+
});
|
|
407
|
+
const state = this.sourceStore.get();
|
|
408
|
+
return {
|
|
409
|
+
source: state.source,
|
|
410
|
+
activeModel: state.activeModel,
|
|
411
|
+
ceilings: { remote: remoteCeiling(this.deps.cfg), local: this.deps.cfg.embedding.allowLocalModels },
|
|
412
|
+
runtime: this.installer.getProgress(),
|
|
413
|
+
models,
|
|
414
|
+
download: this.downloader.getProgress(),
|
|
415
|
+
apply: { phase: this.applyPhase, message: this.applyMessage, startedAt: this.applyStartedAt, busy: this.applyBusy },
|
|
416
|
+
local: this.localSvc ? { state: this.localSvc.getState(), error: this.localSvc.getLoadError() } : null,
|
|
417
|
+
reindex: { ...this.reindex },
|
|
418
|
+
activeNote: this.activeNote,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
}
|
|
@@ -55,10 +55,12 @@ export declare class RemoteEmbeddingService implements EmbeddingService {
|
|
|
55
55
|
* 同类失败只告警一次,避免刷屏。
|
|
56
56
|
*/
|
|
57
57
|
export declare class EmbedHelper {
|
|
58
|
-
private
|
|
58
|
+
private embed;
|
|
59
59
|
private readonly logger?;
|
|
60
60
|
private warned;
|
|
61
61
|
constructor(embed: EmbeddingService, logger?: MemoryLogger | undefined);
|
|
62
|
+
/** 活切换嵌入源(D4/D5):换掉底层服务并复位一次性告警(新服务重新获得告警机会)。 */
|
|
63
|
+
setService(svc: EmbeddingService): void;
|
|
62
64
|
vectorReady(): boolean;
|
|
63
65
|
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。 */
|
|
64
66
|
query(text: string): Promise<Float32Array | undefined>;
|
package/dist/store/embedding.js
CHANGED
|
@@ -95,6 +95,11 @@ export class EmbedHelper {
|
|
|
95
95
|
this.embed = embed;
|
|
96
96
|
this.logger = logger;
|
|
97
97
|
}
|
|
98
|
+
/** 活切换嵌入源(D4/D5):换掉底层服务并复位一次性告警(新服务重新获得告警机会)。 */
|
|
99
|
+
setService(svc) {
|
|
100
|
+
this.embed = svc;
|
|
101
|
+
this.warned = false;
|
|
102
|
+
}
|
|
98
103
|
vectorReady() {
|
|
99
104
|
return this.embed.isReady();
|
|
100
105
|
}
|
package/dist/store/l0.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { ConversationMessage, L0MessageRecord, MemoryLogger } from '../types.js';
|
|
2
2
|
import { type EmbeddingService } from './embedding.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type MemoryDb } from './sqlite.js';
|
|
4
4
|
export declare class L0Store {
|
|
5
5
|
private readonly db;
|
|
6
6
|
private readonly dir;
|
|
7
7
|
private readonly legacyDir;
|
|
8
8
|
private readonly helper;
|
|
9
|
-
private
|
|
9
|
+
private embedSvc;
|
|
10
10
|
private readonly logger?;
|
|
11
11
|
constructor(dataDir: string, db: MemoryDb, embed?: EmbeddingService, logger?: MemoryLogger);
|
|
12
12
|
init(): Promise<void>;
|
|
@@ -17,12 +17,20 @@ export declare class L0Store {
|
|
|
17
17
|
countToday(): Promise<number>;
|
|
18
18
|
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
19
19
|
search(query: string, limit: number): Promise<L0MessageRecord[]>;
|
|
20
|
+
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
|
21
|
+
setEmbeddingService(svc: EmbeddingService): void;
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
+
* 增量重嵌入(同 L1Store.reindex:只补缺失向量,零向量记 skipped 并入 skip 集,
|
|
24
|
+
* 不算失败、不阻塞同步标记——保证补齐判据收敛)。onProgress/shouldCancel
|
|
25
|
+
* 供活切换(D5)的进度展示与取消。
|
|
23
26
|
*/
|
|
24
|
-
reindex(
|
|
27
|
+
reindex(opts?: {
|
|
28
|
+
onProgress?: (done: number, total: number) => void;
|
|
29
|
+
shouldCancel?: () => boolean;
|
|
30
|
+
}): Promise<{
|
|
25
31
|
written: number;
|
|
26
32
|
failed: number;
|
|
33
|
+
skipped: number;
|
|
34
|
+
cancelled?: boolean;
|
|
27
35
|
}>;
|
|
28
36
|
}
|
package/dist/store/l0.js
CHANGED
|
@@ -8,6 +8,7 @@ import * as path from 'node:path';
|
|
|
8
8
|
import { EmbedHelper, NoopEmbeddingService } from './embedding.js';
|
|
9
9
|
import { appendJsonl, dayKey, ensureDir, nowIso, readJsonl } from './io.js';
|
|
10
10
|
import { rrfMerge } from './search-utils.js';
|
|
11
|
+
import { isZeroVector } from './sqlite.js';
|
|
11
12
|
/** 官方过度召回倍数(conversation-search:limit × 3)。 */
|
|
12
13
|
const CANDIDATE_MULTIPLIER = 3;
|
|
13
14
|
export class L0Store {
|
|
@@ -115,18 +116,33 @@ export class L0Store {
|
|
|
115
116
|
}
|
|
116
117
|
return this.db.searchL0Fts(query, limit).map(({ score: _score, ...r }) => r);
|
|
117
118
|
}
|
|
119
|
+
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
|
120
|
+
setEmbeddingService(svc) {
|
|
121
|
+
this.embedSvc = svc;
|
|
122
|
+
this.helper.setService(svc);
|
|
123
|
+
}
|
|
118
124
|
/**
|
|
119
|
-
*
|
|
120
|
-
*
|
|
125
|
+
* 增量重嵌入(同 L1Store.reindex:只补缺失向量,零向量记 skipped 并入 skip 集,
|
|
126
|
+
* 不算失败、不阻塞同步标记——保证补齐判据收敛)。onProgress/shouldCancel
|
|
127
|
+
* 供活切换(D5)的进度展示与取消。
|
|
121
128
|
*/
|
|
122
|
-
async reindex() {
|
|
129
|
+
async reindex(opts) {
|
|
123
130
|
if (!this.helper.vectorReady())
|
|
124
|
-
return { written: 0, failed: 0 };
|
|
125
|
-
const items = this.db.getL0ForReindex();
|
|
131
|
+
return { written: 0, failed: 0, skipped: 0 };
|
|
132
|
+
const items = this.db.getL0ForReindex(this.db.getVecSkipSet('l0'));
|
|
133
|
+
const total = items.length;
|
|
134
|
+
let done = 0;
|
|
126
135
|
let written = 0;
|
|
127
136
|
let failed = 0;
|
|
137
|
+
let skipped = 0;
|
|
138
|
+
let cancelled = false;
|
|
139
|
+
const skippedNow = [];
|
|
128
140
|
const CHUNK = 32;
|
|
129
141
|
for (let i = 0; i < items.length; i += CHUNK) {
|
|
142
|
+
if (opts?.shouldCancel?.()) {
|
|
143
|
+
cancelled = true;
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
130
146
|
const chunk = items.slice(i, i + CHUNK);
|
|
131
147
|
let vecs;
|
|
132
148
|
try {
|
|
@@ -134,15 +150,26 @@ export class L0Store {
|
|
|
134
150
|
}
|
|
135
151
|
catch {
|
|
136
152
|
failed += chunk.length;
|
|
153
|
+
done += chunk.length;
|
|
154
|
+
opts?.onProgress?.(done, total);
|
|
137
155
|
continue;
|
|
138
156
|
}
|
|
139
157
|
chunk.forEach((c, j) => {
|
|
158
|
+
if (isZeroVector(vecs[j])) {
|
|
159
|
+
skipped++;
|
|
160
|
+
skippedNow.push(c.id);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
140
163
|
if (this.db.updateL0Vec(c.id, vecs[j], ''))
|
|
141
164
|
written++;
|
|
142
165
|
else
|
|
143
166
|
failed++;
|
|
144
167
|
});
|
|
168
|
+
done += chunk.length;
|
|
169
|
+
opts?.onProgress?.(done, total);
|
|
145
170
|
}
|
|
146
|
-
|
|
171
|
+
if (skippedNow.length > 0)
|
|
172
|
+
this.db.addVecSkippedIds('l0', skippedNow);
|
|
173
|
+
return { written, failed, skipped, cancelled };
|
|
147
174
|
}
|
|
148
175
|
}
|
package/dist/store/l1.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { L1Hit, MemoryFamily, MemoryLogger, MemoryRecord } from '../types.js';
|
|
2
2
|
import { type EmbeddingService } from './embedding.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type MemoryDb } from './sqlite.js';
|
|
4
4
|
export type RecallStrategy = 'keyword' | 'embedding' | 'hybrid';
|
|
5
5
|
export interface L1SearchOptions {
|
|
6
6
|
/** 按记忆类型精确过滤(后置过滤,官方做法)。 */
|
|
@@ -17,7 +17,7 @@ export declare class L1Store {
|
|
|
17
17
|
private readonly recordsDir;
|
|
18
18
|
private readonly legacyFile;
|
|
19
19
|
private readonly helper;
|
|
20
|
-
private
|
|
20
|
+
private embedSvc;
|
|
21
21
|
private readonly logger?;
|
|
22
22
|
constructor(dataDir: string, db: MemoryDb, embed?: EmbeddingService, strategy?: RecallStrategy, logger?: MemoryLogger);
|
|
23
23
|
init(): Promise<void>;
|
|
@@ -32,6 +32,8 @@ export declare class L1Store {
|
|
|
32
32
|
appendNew(records: MemoryRecord[]): Promise<void>;
|
|
33
33
|
/** 去重 update/merge 产出的记录:只更新检索库(JSONL 事实源不改写,官方语义)。 */
|
|
34
34
|
upsert(record: MemoryRecord): Promise<void>;
|
|
35
|
+
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
|
36
|
+
setEmbeddingService(svc: EmbeddingService): void;
|
|
35
37
|
deleteBatch(ids: string[]): Promise<void>;
|
|
36
38
|
/**
|
|
37
39
|
* 三策略检索(自动召回与 memory_search 工具共用接缝)。
|
|
@@ -59,12 +61,20 @@ export declare class L1Store {
|
|
|
59
61
|
*/
|
|
60
62
|
searchCandidates(query: string, limit: number, family?: MemoryFamily): Promise<MemoryRecord[]>;
|
|
61
63
|
/**
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
+
* 增量重嵌入(embedding 配置变化 / 周期性补齐用):只处理缺失向量的记录,
|
|
65
|
+
* 排除已判定"当前 provider 不可嵌入"的 skip 集。返回写入/失败/跳过数——
|
|
66
|
+
* failed > 0 时调用方不应标记 meta 同步完成;skipped(零向量)不算失败、
|
|
67
|
+
* 不阻塞同步标记(否则补齐判据永不收敛,每 30 分钟全量重嵌死循环)。
|
|
68
|
+
* onProgress/shouldCancel 供活切换(D5)的进度展示与取消。
|
|
64
69
|
*/
|
|
65
|
-
reindex(
|
|
70
|
+
reindex(opts?: {
|
|
71
|
+
onProgress?: (done: number, total: number) => void;
|
|
72
|
+
shouldCancel?: () => boolean;
|
|
73
|
+
}): Promise<{
|
|
66
74
|
written: number;
|
|
67
75
|
failed: number;
|
|
76
|
+
skipped: number;
|
|
77
|
+
cancelled?: boolean;
|
|
68
78
|
}>;
|
|
69
79
|
private postProcess;
|
|
70
80
|
}
|