@modusensus/dsh-mneme 0.8.12 → 0.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -5
- package/bin/dsh-mneme-serve.mjs +151 -0
- package/lib/api-standalone.js +110 -9
- package/lib/api.js +2 -0
- package/lib/client.js +12 -1
- package/lib/config.js +26 -0
- package/lib/guide.js +13 -1
- package/lib/index.js +31 -191
- package/lib/semantic.js +229 -0
- package/lib/sensitive-scan.js +209 -0
- package/lib/serve.js +215 -0
- package/lib/service.js +54 -18
- package/lib/settings.js +7 -1
- package/package.json +6 -2
- package/scripts/changelog-prep.mjs +36 -0
- package/src/api-standalone.js +110 -9
- package/src/api.js +2 -0
- package/src/config.js +26 -0
- package/src/guide.js +13 -1
- package/src/index.js +31 -191
- package/src/semantic.js +229 -0
- package/src/sensitive-scan.js +209 -0
- package/src/serve.js +215 -0
- package/src/service.js +54 -18
- package/src/settings.js +7 -1
- package/test/api.test.js +12 -2
- package/test/changelog-prep.test.js +60 -0
- package/test/inject-heat.test.js +31 -15
- package/test/inject-pools.test.js +19 -0
- package/test/scope-strict.test.js +136 -0
- package/test/sensitive-scan.test.js +207 -0
- package/test/serve-bin.test.js +121 -0
- package/test/serve.test.js +196 -0
- package/test/standalone-api.test.js +212 -5
- package/test/write-admission.test.js +115 -0
package/lib/index.js
CHANGED
|
@@ -5,6 +5,8 @@ import { resolveDocumentDir } from "./document.js";
|
|
|
5
5
|
import { createService } from "./service.js";
|
|
6
6
|
// #254 写入准入(第一阶段只计量,不拦截):见 src/write-admission.js 的文件头。
|
|
7
7
|
import { createWriteAdmission } from "./write-admission.js";
|
|
8
|
+
// #164 A2:写入边界的密钥 / PII 判据,注入给上面的写入准入。
|
|
9
|
+
import { createSensitiveScan } from "./sensitive-scan.js";
|
|
8
10
|
import { createTools } from "./tools.js";
|
|
9
11
|
import { createInjector } from "./inject.js";
|
|
10
12
|
import { createContinuityRescue } from "./continuity.js";
|
|
@@ -17,10 +19,10 @@ import { createStandaloneApi } from "./api-standalone.js";
|
|
|
17
19
|
import { createMaintenance } from "./maintenance.js";
|
|
18
20
|
import { createSettings } from "./settings.js";
|
|
19
21
|
import { createCommandManager } from "./commands.js";
|
|
20
|
-
import { createEmbedder } from "./embedding.js";
|
|
21
|
-
import { createEmbedderByProvider } from "./local-embedder.js";
|
|
22
|
-
import { LocalReranker } from "./reranker.js";
|
|
23
22
|
import { createVectorIndex } from "./vector-index.js";
|
|
23
|
+
// semantic(embedder/reranker/boot 回填)纯搬移至 src/semantic.js(PR2):宿主与
|
|
24
|
+
// daemon(dsh-mneme-serve,#363)共用同一套装配,原文件保留 barrel 出口。
|
|
25
|
+
import { createSemantic } from "./semantic.js";
|
|
24
26
|
import { Config, applyLightModePreset, injectChildEnabled } from "./config.js";
|
|
25
27
|
import { langOf } from "./lang.js";
|
|
26
28
|
import { extractEntities } from "./entities/extractor.js";
|
|
@@ -147,43 +149,9 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, serv
|
|
|
147
149
|
};
|
|
148
150
|
}
|
|
149
151
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
*
|
|
154
|
-
* Runs regardless of the model fingerprint: the old call-site gate returned
|
|
155
|
-
* early when vector_meta already held the embedder's hash, permanently
|
|
156
|
-
* orphaning rows whose embed failed at write time (embedder not ready /
|
|
157
|
-
* provider rate limit) — one successful embed was enough to never backfill
|
|
158
|
-
* again. markModel is idempotent when the fingerprint already matches, so
|
|
159
|
-
* re-running costs nothing beyond the actually-missing rows.
|
|
160
|
-
*/
|
|
161
|
-
export async function backfillMissingEmbeddings({
|
|
162
|
-
store, embedder, vectorIndex, logger,
|
|
163
|
-
maxTotal = 500, batchSize = 10, rateLimitMs = 200
|
|
164
|
-
}) {
|
|
165
|
-
let indexed = 0;
|
|
166
|
-
for (let done = 0; done < maxTotal;) {
|
|
167
|
-
const rows = store.needsEmbedding(batchSize);
|
|
168
|
-
if (!rows.length) break;
|
|
169
|
-
for (const row of rows) {
|
|
170
|
-
try {
|
|
171
|
-
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
172
|
-
const vector = await embedder.embedSingle(text);
|
|
173
|
-
if (vector?.length) {
|
|
174
|
-
store.setEmbedding(row.id, vector);
|
|
175
|
-
indexed++;
|
|
176
|
-
}
|
|
177
|
-
} catch { /* skip the bad row */ }
|
|
178
|
-
}
|
|
179
|
-
done += rows.length;
|
|
180
|
-
// Rate limit: space out batches so the provider is not hammered.
|
|
181
|
-
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
|
|
182
|
-
}
|
|
183
|
-
if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
184
|
-
logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
|
|
185
|
-
return indexed;
|
|
186
|
-
}
|
|
152
|
+
// backfillMissingEmbeddings 已随语义装配整体搬至 src/semantic.js(纯搬移);
|
|
153
|
+
// 保留 barrel 再出口 —— test/reindex-backfill.test.js 仍从本模块 import,调用方零改动。
|
|
154
|
+
export { backfillMissingEmbeddings } from "./semantic.js";
|
|
187
155
|
|
|
188
156
|
export const apply = (ctx, config) => {
|
|
189
157
|
const rawCfg = Config(config);
|
|
@@ -278,11 +246,17 @@ export const apply = (ctx, config) => {
|
|
|
278
246
|
// 关掉,在无保留期的表里按写入频次增长是不能接受的)。
|
|
279
247
|
//
|
|
280
248
|
// sensitiveScan 是密钥 / PII 那一类判据的注入点。按 #254 验收第 4 条它是 #164 A2
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
|
|
249
|
+
// 的判据来源,现在由 src/sensitive-scan.js 实现(维护者 09-28 把 A2 的认领转给
|
|
250
|
+
// 本侧)。这里只做接线:判据开不开由它自己的键 sensitiveScanEnabled 决定,工厂在
|
|
251
|
+
// 关时返回 null,闸门的行为就与 #332 合并时逐字段一致(那一版根本没有这个函数);
|
|
252
|
+
// 命中之后是仅告警还是真拦,仍是 writeAdmission.enforce 的事(#164 口径:
|
|
253
|
+
// 默认仅告警、拦截 opt-in),判据不碰决策。
|
|
254
|
+
const writeAdmission = createWriteAdmission({
|
|
255
|
+
store,
|
|
256
|
+
config: cfg,
|
|
257
|
+
logger: ctx.logger,
|
|
258
|
+
sensitiveScan: createSensitiveScan({ config: cfg })
|
|
259
|
+
});
|
|
286
260
|
const service = createService({ store, mirror, config: cfg, logger: ctx.logger, documentIndex, writeAdmission });
|
|
287
261
|
|
|
288
262
|
// F-NEW-03: if the mirror sync failed last run (persisted dirty state), retry
|
|
@@ -331,150 +305,14 @@ export const apply = (ctx, config) => {
|
|
|
331
305
|
}
|
|
332
306
|
};
|
|
333
307
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
//
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
// human mirror edits still merge on boot.
|
|
343
|
-
applyHumanEdits();
|
|
344
|
-
} else if (cfg.embedProvider === "openai") {
|
|
345
|
-
// vectorIndex is passed so the legacy OpenAI embedder records the producing
|
|
346
|
-
// model fingerprint after each successful embed (Bug3).
|
|
347
|
-
embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
|
|
348
|
-
service.setEmbedder(embedder);
|
|
349
|
-
// issue #135: 未配置时明确告警一次。此前 legacy OpenAI embedder 恒报
|
|
350
|
-
// ready=true,向量层「绿的但全哑」可以静默存在很久(本机持续了数周)。
|
|
351
|
-
// 只记日志、不阻断启动:轻量模式与「先跑起来再补配置」都是正当用法。
|
|
352
|
-
if (embedder.configured === false) {
|
|
353
|
-
ctx.logger?.warn?.(
|
|
354
|
-
"[dsh-mneme] 向量层未配置(vector-config 的 enabled/baseUrl/apiKey/model 有缺):"
|
|
355
|
-
+ "语义召回、语义去重、rerank、sleep 冲突检测将静默失效,"
|
|
356
|
-
+ "dream 的语义聚类会退化为全量窗口兜底。"
|
|
357
|
-
+ "请在设置面板补全 embedding 端点与模型,或把 embedProvider 改为 local/ollama。"
|
|
358
|
-
);
|
|
359
|
-
}
|
|
360
|
-
// legacy OpenAI embedder needs no async init → human edits apply right away
|
|
361
|
-
applyHumanEdits();
|
|
362
|
-
} else {
|
|
363
|
-
try {
|
|
364
|
-
embedder = createEmbedderByProvider(cfg.embedProvider, {
|
|
365
|
-
model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
|
|
366
|
-
dimension: cfg.localEmbedDimension,
|
|
367
|
-
device: cfg.localEmbedDevice,
|
|
368
|
-
batchSize: cfg.localEmbedBatchSize,
|
|
369
|
-
// 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
|
|
370
|
-
// 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
|
|
371
|
-
pooling: cfg.localEmbedPooling,
|
|
372
|
-
cacheDir: cfg.embedModelCacheDir,
|
|
373
|
-
runtimeDir: cfg.runtimeDir,
|
|
374
|
-
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
375
|
-
remoteHost: cfg.embedModelMirror,
|
|
376
|
-
resilientModelDownload: cfg.resilientModelDownload,
|
|
377
|
-
baseUrl: cfg.ollamaBaseUrl,
|
|
378
|
-
logger: ctx.logger
|
|
379
|
-
});
|
|
380
|
-
service.setEmbedder(embedder);
|
|
381
|
-
// issue #6: wait for extractor init before applying human edits, so
|
|
382
|
-
// scheduled embeddings see a ready embedder.
|
|
383
|
-
const bootEmbedder = () => embedder.init()
|
|
384
|
-
.then(() => { applyHumanEdits(); return true; })
|
|
385
|
-
.catch(() => false);
|
|
386
|
-
// #118: the old one-shot probe permanently degraded search to keyword
|
|
387
|
-
// when Ollama was briefly unreachable at boot (recoverable only by
|
|
388
|
-
// restart). Retry briefly (5 attempts total: 1 initial + 4 × 15s);
|
|
389
|
-
// search degrades to keyword meanwhile because per-query embed failures
|
|
390
|
-
// are swallowed.
|
|
391
|
-
bootEmbedder().then((ok) => {
|
|
392
|
-
if (ok) return;
|
|
393
|
-
let tries = 4;
|
|
394
|
-
const retry = () => {
|
|
395
|
-
if (tries-- <= 0) {
|
|
396
|
-
ctx.logger?.warn?.("[dsh-mneme] embedder init retries exhausted, search degrades to keyword");
|
|
397
|
-
service.setEmbedder(null);
|
|
398
|
-
applyHumanEdits();
|
|
399
|
-
return;
|
|
400
|
-
}
|
|
401
|
-
embedRetryTimer = setTimeout(async () => {
|
|
402
|
-
if (await bootEmbedder()) return;
|
|
403
|
-
retry();
|
|
404
|
-
}, 15_000);
|
|
405
|
-
};
|
|
406
|
-
ctx.logger?.warn?.("[dsh-mneme] embedder init failed, retrying");
|
|
407
|
-
retry();
|
|
408
|
-
});
|
|
409
|
-
} catch (error) {
|
|
410
|
-
ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
411
|
-
applyHumanEdits();
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
// Cross-encoder rerank over recall candidates. Best-effort: a failed model
|
|
416
|
-
// load only disables reranking, never search itself. Explicit opt-in only
|
|
417
|
-
// (rerankEnabled defaults to false): constructing LocalReranker is what pulls
|
|
418
|
-
// in onnxruntime, so the default config never loads it (item ⑥).
|
|
419
|
-
if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
|
|
420
|
-
try {
|
|
421
|
-
reranker = new LocalReranker({
|
|
422
|
-
model: cfg.rerankModel,
|
|
423
|
-
batchSize: cfg.rerankBatchSize,
|
|
424
|
-
maxCandidates: cfg.rerankMaxCandidates,
|
|
425
|
-
scoreThreshold: cfg.rerankScoreThreshold,
|
|
426
|
-
device: cfg.localEmbedDevice,
|
|
427
|
-
cacheDir: cfg.embedModelCacheDir,
|
|
428
|
-
runtimeDir: cfg.runtimeDir,
|
|
429
|
-
// #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
|
|
430
|
-
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
431
|
-
useDtype: cfg.rerankDtype,
|
|
432
|
-
remoteHost: cfg.embedModelMirror,
|
|
433
|
-
resilientModelDownload: cfg.resilientModelDownload,
|
|
434
|
-
logger: ctx.logger
|
|
435
|
-
});
|
|
436
|
-
service.setReranker(reranker);
|
|
437
|
-
reranker.init().catch((error) => {
|
|
438
|
-
ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
|
|
439
|
-
service.setReranker(null);
|
|
440
|
-
});
|
|
441
|
-
} catch (error) {
|
|
442
|
-
ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
|
|
447
|
-
// is configured and rows still lack an embedding (e.g. written before vector
|
|
448
|
-
// search was enabled), the backfill runs in the background after a short
|
|
449
|
-
// delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
|
|
450
|
-
// large backlog never floods the provider. Failures degrade silently —
|
|
451
|
-
// search stays keyword.
|
|
452
|
-
function scheduleAutoReindex() {
|
|
453
|
-
if (cfg.autoReindexOnBoot === false) return;
|
|
454
|
-
const attempt = (tries) => {
|
|
455
|
-
try {
|
|
456
|
-
if (!embedder || typeof embedder.embedSingle !== "function") return;
|
|
457
|
-
if ("ready" in embedder && embedder.ready !== true) {
|
|
458
|
-
// Local/ollama embedders init asynchronously; give them a moment
|
|
459
|
-
// before giving up on this boot (next boot retries).
|
|
460
|
-
if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
|
|
461
|
-
return;
|
|
462
|
-
}
|
|
463
|
-
if (!store.needsEmbedding(1).length) return; // nothing to backfill
|
|
464
|
-
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
|
|
465
|
-
// used to return early and permanently orphan rows whose embed failed
|
|
466
|
-
// at write time. See backfillMissingEmbeddings().
|
|
467
|
-
backfillMissingEmbeddings({ store, embedder, vectorIndex, logger: ctx.logger })
|
|
468
|
-
.catch((error) => {
|
|
469
|
-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
470
|
-
});
|
|
471
|
-
} catch (error) {
|
|
472
|
-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
473
|
-
}
|
|
474
|
-
};
|
|
475
|
-
setTimeout(() => attempt(5), 5000);
|
|
476
|
-
}
|
|
477
|
-
scheduleAutoReindex();
|
|
308
|
+
// embedder/reranker/boot 回填装配已整体搬至 src/semantic.js(纯搬移,宿主与 daemon
|
|
309
|
+
// 共用):调用时序(applyHumanEdits 在各分支的触发点、#118 重试、autoReindexOnBoot)
|
|
310
|
+
// 原样保留在 createSemantic 内部,这里只拿引用。init 全失败的 embedder 引用仍会进入
|
|
311
|
+
// dream/sleep 的 semantic 面 —— 与搬移前一致:检索侧 setEmbedder(null) 降级关键词。
|
|
312
|
+
const semantic = createSemantic({
|
|
313
|
+
store, service, settings, cfg, logger: ctx.logger, vectorIndex, applyHumanEdits, lightMode
|
|
314
|
+
});
|
|
315
|
+
const { embedder, reranker } = semantic;
|
|
478
316
|
|
|
479
317
|
// Custom commands: register persisted commands into the DSH command registry
|
|
480
318
|
// on boot; add/remove re-register live through the API.
|
|
@@ -593,7 +431,7 @@ export const apply = (ctx, config) => {
|
|
|
593
431
|
|
|
594
432
|
// #118: never let a pending embedder init retry fire after unload and touch
|
|
595
433
|
// a torn-down context.
|
|
596
|
-
disposers.push(() =>
|
|
434
|
+
disposers.push(() => semantic.dispose()); // #118 重试计时器 + boot 回填计时器(搬入 semantic.js 后由它自持)
|
|
597
435
|
|
|
598
436
|
ctx.inject(["systemPrompt"], (promptCtx) => {
|
|
599
437
|
if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
|
|
@@ -641,7 +479,9 @@ export const apply = (ctx, config) => {
|
|
|
641
479
|
// kv and is auto-generated on first boot by createStandaloneApi. Binding a
|
|
642
480
|
// non-loopback host is the operator's documented responsibility.
|
|
643
481
|
if ((settings.getExternalApi?.()?.enabled ?? cfg.externalApiEnabled) === true) {
|
|
644
|
-
|
|
482
|
+
// embedder 交给 /context 的查询嵌入(issue #370):与 service.setEmbedder 同一
|
|
483
|
+
// 实例(semantic.js 装配产物),lightMode 下为 null,路由侧降级规则档。
|
|
484
|
+
const standalone = createStandaloneApi({ service, store, config: cfg, logger: ctx.logger, settings, maintenance, embedder });
|
|
645
485
|
disposers.push(() => standalone.server.close());
|
|
646
486
|
standalone.ready.catch((error) => {
|
|
647
487
|
ctx.logger?.warn?.(`[dsh-mneme] standalone API failed to start: ${String(error)}`);
|
package/lib/semantic.js
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// src/semantic.js —— embedder/reranker 装配与 boot 自动回填。
|
|
2
|
+
// 纯搬移自 src/index.js(2026-10,PR2):backfillMissingEmbeddings(原 :150-186)与
|
|
3
|
+
// 装配段(原 :311-477),行为逐字节对齐,仅两处已注记的机械差异(ctx.logger → 注入
|
|
4
|
+
// logger;boot 回填的首查计时器纳入 dispose)。搬移原因:daemon(dsh-mneme-serve,
|
|
5
|
+
// #363)与宿主要共用同一套语义装配——「向量检索开箱即用」的承诺落在两侧同一份
|
|
6
|
+
// 代码上,而不是 daemon 复刻一份会漂移的副本。拆法遵循 AGENTS.md 尺寸约定:
|
|
7
|
+
// 纯搬移独立 PR、原文件调用方零改动(backfill 经 index.js barrel 再出口,测试照旧)。
|
|
8
|
+
//
|
|
9
|
+
// 时序契约(搬移前即如此,由 index.js 全量测试与 reindex-backfill.test.js 锁):
|
|
10
|
+
// lightMode / openai / 同步构造失败 → applyHumanEdits 立即;
|
|
11
|
+
// local|ollama → init 成功后 applyHumanEdits;#118 重试(1 + 4×15s)耗尽 →
|
|
12
|
+
// setEmbedder(null) 检索降级关键词,随后仍 applyHumanEdits;
|
|
13
|
+
// reranker 异步 init 失败只降级 rerank 自身,绝不影响 search。
|
|
14
|
+
import { createEmbedder } from "./embedding.js";
|
|
15
|
+
import { createEmbedderByProvider } from "./local-embedder.js";
|
|
16
|
+
import { LocalReranker } from "./reranker.js";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Issue #128: bounded backfill of rows still missing an embedding (active rows
|
|
20
|
+
* only — needsEmbedding filters archived/forgotten). Exported for tests.
|
|
21
|
+
*
|
|
22
|
+
* Runs regardless of the model fingerprint: the old call-site gate returned
|
|
23
|
+
* early when vector_meta already held the embedder's hash, permanently
|
|
24
|
+
* orphaning rows whose embed failed at write time (embedder not ready /
|
|
25
|
+
* provider rate limit) — one successful embed was enough to never backfill
|
|
26
|
+
* again. markModel is idempotent when the fingerprint already matches, so
|
|
27
|
+
* re-running costs nothing beyond the actually-missing rows.
|
|
28
|
+
*/
|
|
29
|
+
export async function backfillMissingEmbeddings({
|
|
30
|
+
store, embedder, vectorIndex, logger,
|
|
31
|
+
maxTotal = 500, batchSize = 10, rateLimitMs = 200
|
|
32
|
+
}) {
|
|
33
|
+
let indexed = 0;
|
|
34
|
+
for (let done = 0; done < maxTotal;) {
|
|
35
|
+
const rows = store.needsEmbedding(batchSize);
|
|
36
|
+
if (!rows.length) break;
|
|
37
|
+
for (const row of rows) {
|
|
38
|
+
try {
|
|
39
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
40
|
+
const vector = await embedder.embedSingle(text);
|
|
41
|
+
if (vector?.length) {
|
|
42
|
+
store.setEmbedding(row.id, vector);
|
|
43
|
+
indexed++;
|
|
44
|
+
}
|
|
45
|
+
} catch { /* skip the bad row */ }
|
|
46
|
+
}
|
|
47
|
+
done += rows.length;
|
|
48
|
+
// Rate limit: space out batches so the provider is not hammered.
|
|
49
|
+
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
|
|
50
|
+
}
|
|
51
|
+
if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
52
|
+
logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
|
|
53
|
+
return indexed;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* 组装语义管线(embedder + reranker)并挂到 service 上,随后调度 boot 自动回填。
|
|
58
|
+
* @param {object} opts
|
|
59
|
+
* store/service/settings/vectorIndex — 宿主与 daemon 同形传入;
|
|
60
|
+
* cfg — 宿主传合并后的完整配置;daemon 传语义子集(默认值锚定 config.js);
|
|
61
|
+
* logger — console 形状(原代码读 ctx.logger,搬移后注入);
|
|
62
|
+
* applyHumanEdits— 人改镜像合并回调(index.js 闭包,读 mirror.readHumanEdits);
|
|
63
|
+
* 在哪个分支何时被调是时序契约的一部分,见文件头;
|
|
64
|
+
* lightMode — 轻量档:整条向量管线关闭,无 embedder/reranker、不回填。
|
|
65
|
+
* @returns {{embedder, reranker, dispose}} embedder 构造失败(同步抛)时为 null;
|
|
66
|
+
* init 异步失败经 #118 重试后 service 侧降级,此处引用仍在(dream/sleep 语义面
|
|
67
|
+
* 与搬移前一致)。dispose 清两个引导期计时器。
|
|
68
|
+
*/
|
|
69
|
+
export function createSemantic({ store, service, settings, cfg, logger, vectorIndex, applyHumanEdits, lightMode = false }) {
|
|
70
|
+
let embedder = null;
|
|
71
|
+
let reranker = null;
|
|
72
|
+
// #118: pending embedder-init retry timer, cleared on unload.
|
|
73
|
+
let embedRetryTimer = null;
|
|
74
|
+
// boot 回填首查计时器。搬移前在 index.js 是裸 setTimeout(不参与卸载清理);
|
|
75
|
+
// 纳入 dispose 是修悬挂,不改变启动行为。
|
|
76
|
+
let reindexTimer = null;
|
|
77
|
+
|
|
78
|
+
if (lightMode) {
|
|
79
|
+
// Light mode: the whole vector pipeline stays off — no embedder (nothing
|
|
80
|
+
// pulls in ONNX/transformers), no reranker, no boot backfill (the preset
|
|
81
|
+
// also cleared autoReindexOnBoot). Recall degrades to keyword search and
|
|
82
|
+
// human mirror edits still merge on boot.
|
|
83
|
+
applyHumanEdits();
|
|
84
|
+
} else if (cfg.embedProvider === "openai") {
|
|
85
|
+
// vectorIndex is passed so the legacy OpenAI embedder records the producing
|
|
86
|
+
// model fingerprint after each successful embed (Bug3).
|
|
87
|
+
embedder = createEmbedder({ store, settings, logger, vectorIndex });
|
|
88
|
+
service.setEmbedder(embedder);
|
|
89
|
+
// issue #135: 未配置时明确告警一次。此前 legacy OpenAI embedder 恒报
|
|
90
|
+
// ready=true,向量层「绿的但全哑」可以静默存在很久(本机持续了数周)。
|
|
91
|
+
// 只记日志、不阻断启动:轻量模式与「先跑起来再补配置」都是正当用法。
|
|
92
|
+
if (embedder.configured === false) {
|
|
93
|
+
logger?.warn?.(
|
|
94
|
+
"[dsh-mneme] 向量层未配置(vector-config 的 enabled/baseUrl/apiKey/model 有缺):"
|
|
95
|
+
+ "语义召回、语义去重、rerank、sleep 冲突检测将静默失效,"
|
|
96
|
+
+ "dream 的语义聚类会退化为全量窗口兜底。"
|
|
97
|
+
+ "请在设置面板补全 embedding 端点与模型,或把 embedProvider 改为 local/ollama。"
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
// legacy OpenAI embedder needs no async init → human edits apply right away
|
|
101
|
+
applyHumanEdits();
|
|
102
|
+
} else {
|
|
103
|
+
try {
|
|
104
|
+
embedder = createEmbedderByProvider(cfg.embedProvider, {
|
|
105
|
+
model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
|
|
106
|
+
dimension: cfg.localEmbedDimension,
|
|
107
|
+
device: cfg.localEmbedDevice,
|
|
108
|
+
batchSize: cfg.localEmbedBatchSize,
|
|
109
|
+
// 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
|
|
110
|
+
// 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
|
|
111
|
+
pooling: cfg.localEmbedPooling,
|
|
112
|
+
cacheDir: cfg.embedModelCacheDir,
|
|
113
|
+
runtimeDir: cfg.runtimeDir,
|
|
114
|
+
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
115
|
+
remoteHost: cfg.embedModelMirror,
|
|
116
|
+
resilientModelDownload: cfg.resilientModelDownload,
|
|
117
|
+
baseUrl: cfg.ollamaBaseUrl,
|
|
118
|
+
logger
|
|
119
|
+
});
|
|
120
|
+
service.setEmbedder(embedder);
|
|
121
|
+
// issue #6: wait for extractor init before applying human edits, so
|
|
122
|
+
// scheduled embeddings see a ready embedder.
|
|
123
|
+
const bootEmbedder = () => embedder.init()
|
|
124
|
+
.then(() => { applyHumanEdits(); return true; })
|
|
125
|
+
.catch(() => false);
|
|
126
|
+
// #118: the old one-shot probe permanently degraded search to keyword
|
|
127
|
+
// when Ollama was briefly unreachable at boot (recoverable only by
|
|
128
|
+
// restart). Retry briefly (5 attempts total: 1 initial + 4 × 15s);
|
|
129
|
+
// search degrades to keyword meanwhile because per-query embed failures
|
|
130
|
+
// are swallowed.
|
|
131
|
+
bootEmbedder().then((ok) => {
|
|
132
|
+
if (ok) return;
|
|
133
|
+
let tries = 4;
|
|
134
|
+
const retry = () => {
|
|
135
|
+
if (tries-- <= 0) {
|
|
136
|
+
logger?.warn?.("[dsh-mneme] embedder init retries exhausted, search degrades to keyword");
|
|
137
|
+
service.setEmbedder(null);
|
|
138
|
+
applyHumanEdits();
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
embedRetryTimer = setTimeout(async () => {
|
|
142
|
+
if (await bootEmbedder()) return;
|
|
143
|
+
retry();
|
|
144
|
+
}, 15_000);
|
|
145
|
+
};
|
|
146
|
+
logger?.warn?.("[dsh-mneme] embedder init failed, retrying");
|
|
147
|
+
retry();
|
|
148
|
+
});
|
|
149
|
+
} catch (error) {
|
|
150
|
+
logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
151
|
+
applyHumanEdits();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Cross-encoder rerank over recall candidates. Best-effort: a failed model
|
|
156
|
+
// load only disables reranking, never search itself. Explicit opt-in only
|
|
157
|
+
// (rerankEnabled defaults to false): constructing LocalReranker is what pulls
|
|
158
|
+
// in onnxruntime, so the default config never loads it (item ⑥).
|
|
159
|
+
if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
|
|
160
|
+
try {
|
|
161
|
+
reranker = new LocalReranker({
|
|
162
|
+
model: cfg.rerankModel,
|
|
163
|
+
batchSize: cfg.rerankBatchSize,
|
|
164
|
+
maxCandidates: cfg.rerankMaxCandidates,
|
|
165
|
+
scoreThreshold: cfg.rerankScoreThreshold,
|
|
166
|
+
device: cfg.localEmbedDevice,
|
|
167
|
+
cacheDir: cfg.embedModelCacheDir,
|
|
168
|
+
runtimeDir: cfg.runtimeDir,
|
|
169
|
+
// #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
|
|
170
|
+
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
171
|
+
useDtype: cfg.rerankDtype,
|
|
172
|
+
remoteHost: cfg.embedModelMirror,
|
|
173
|
+
resilientModelDownload: cfg.resilientModelDownload,
|
|
174
|
+
logger
|
|
175
|
+
});
|
|
176
|
+
service.setReranker(reranker);
|
|
177
|
+
reranker.init().catch((error) => {
|
|
178
|
+
logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
|
|
179
|
+
service.setReranker(null);
|
|
180
|
+
});
|
|
181
|
+
} catch (error) {
|
|
182
|
+
logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
|
|
187
|
+
// is configured and rows still lack an embedding (e.g. written before vector
|
|
188
|
+
// search was enabled), the backfill runs in the background after a short
|
|
189
|
+
// delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
|
|
190
|
+
// large backlog never floods the provider. Failures degrade silently —
|
|
191
|
+
// search stays keyword.
|
|
192
|
+
function scheduleAutoReindex() {
|
|
193
|
+
if (cfg.autoReindexOnBoot === false) return;
|
|
194
|
+
const attempt = (tries) => {
|
|
195
|
+
try {
|
|
196
|
+
if (!embedder || typeof embedder.embedSingle !== "function") return;
|
|
197
|
+
if ("ready" in embedder && embedder.ready !== true) {
|
|
198
|
+
// Local/ollama embedders init asynchronously; give them a moment
|
|
199
|
+
// before giving up on this boot (next boot retries).
|
|
200
|
+
// CodeRabbit on #365:嵌套重试计时器同样入册,否则 dispose 后仍可能
|
|
201
|
+
// 对已关库跑 needsEmbedding(有 try/catch 兜底只是日志噪声,但状态要收干净)。
|
|
202
|
+
if (tries > 0) reindexTimer = setTimeout(() => attempt(tries - 1), 2000);
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
if (!store.needsEmbedding(1).length) return; // nothing to backfill
|
|
206
|
+
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
|
|
207
|
+
// used to return early and permanently orphan rows whose embed failed
|
|
208
|
+
// at write time. See backfillMissingEmbeddings().
|
|
209
|
+
backfillMissingEmbeddings({ store, embedder, vectorIndex, logger })
|
|
210
|
+
.catch((error) => {
|
|
211
|
+
logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
212
|
+
});
|
|
213
|
+
} catch (error) {
|
|
214
|
+
logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
reindexTimer = setTimeout(() => attempt(5), 5000);
|
|
218
|
+
}
|
|
219
|
+
scheduleAutoReindex();
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
embedder,
|
|
223
|
+
reranker,
|
|
224
|
+
dispose() {
|
|
225
|
+
if (embedRetryTimer !== null) { clearTimeout(embedRetryTimer); embedRetryTimer = null; }
|
|
226
|
+
if (reindexTimer !== null) { clearTimeout(reindexTimer); reindexTimer = null; }
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
}
|