@modusensus/dsh-mneme 0.8.13 → 0.8.15

package/lib/semantic.js

@@ -0,0 +1,229 @@
+// src/semantic.js —— embedder/reranker 装配与 boot 自动回填。
+// 纯搬移自 src/index.js(2026-10,PR2):backfillMissingEmbeddings(原 :150-186)与
+// 装配段(原 :311-477),行为逐字节对齐,仅两处已注记的机械差异(ctx.logger → 注入
+// logger;boot 回填的首查计时器纳入 dispose)。搬移原因:daemon(dsh-mneme-serve,
+// #363)与宿主要共用同一套语义装配——「向量检索开箱即用」的承诺落在两侧同一份
+// 代码上,而不是 daemon 复刻一份会漂移的副本。拆法遵循 AGENTS.md 尺寸约定:
+// 纯搬移独立 PR、原文件调用方零改动(backfill 经 index.js barrel 再出口,测试照旧)。
+//
+// 时序契约(搬移前即如此,由 index.js 全量测试与 reindex-backfill.test.js 锁):
+//   lightMode / openai / 同步构造失败 → applyHumanEdits 立即;
+//   local|ollama → init 成功后 applyHumanEdits;#118 重试(1 + 4×15s)耗尽 →
+//   setEmbedder(null) 检索降级关键词,随后仍 applyHumanEdits;
+//   reranker 异步 init 失败只降级 rerank 自身,绝不影响 search。
+import { createEmbedder } from "./embedding.js";
+import { createEmbedderByProvider } from "./local-embedder.js";
+import { LocalReranker } from "./reranker.js";
+
+/**
+ * Issue #128: bounded backfill of rows still missing an embedding (active rows
+ * only — needsEmbedding filters archived/forgotten). Exported for tests.
+ *
+ * Runs regardless of the model fingerprint: the old call-site gate returned
+ * early when vector_meta already held the embedder's hash, permanently
+ * orphaning rows whose embed failed at write time (embedder not ready /
+ * provider rate limit) — one successful embed was enough to never backfill
+ * again. markModel is idempotent when the fingerprint already matches, so
+ * re-running costs nothing beyond the actually-missing rows.
+ */
+export async function backfillMissingEmbeddings({
+  store, embedder, vectorIndex, logger,
+  maxTotal = 500, batchSize = 10, rateLimitMs = 200
+}) {
+  let indexed = 0;
+  for (let done = 0; done < maxTotal;) {
+    const rows = store.needsEmbedding(batchSize);
+    if (!rows.length) break;
+    for (const row of rows) {
+      try {
+        const text = [row.title, row.content].filter(Boolean).join("\n");
+        const vector = await embedder.embedSingle(text);
+        if (vector?.length) {
+          store.setEmbedding(row.id, vector);
+          indexed++;
+        }
+      } catch { /* skip the bad row */ }
+    }
+    done += rows.length;
+    // Rate limit: space out batches so the provider is not hammered.
+    if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
+  }
+  if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
+  logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
+  return indexed;
+}
+
+/**
+ * 组装语义管线(embedder + reranker)并挂到 service 上,随后调度 boot 自动回填。
+ * @param {object} opts
+ *   store/service/settings/vectorIndex — 宿主与 daemon 同形传入;
+ *   cfg            — 宿主传合并后的完整配置;daemon 传语义子集(默认值锚定 config.js);
+ *   logger         — console 形状(原代码读 ctx.logger,搬移后注入);
+ *   applyHumanEdits— 人改镜像合并回调(index.js 闭包,读 mirror.readHumanEdits);
+ *                    在哪个分支何时被调是时序契约的一部分,见文件头;
+ *   lightMode      — 轻量档:整条向量管线关闭,无 embedder/reranker、不回填。
+ * @returns {{embedder, reranker, dispose}} embedder 构造失败(同步抛)时为 null;
+ *   init 异步失败经 #118 重试后 service 侧降级,此处引用仍在(dream/sleep 语义面
+ *   与搬移前一致)。dispose 清两个引导期计时器。
+ */
+export function createSemantic({ store, service, settings, cfg, logger, vectorIndex, applyHumanEdits, lightMode = false }) {
+  let embedder = null;
+  let reranker = null;
+  // #118: pending embedder-init retry timer, cleared on unload.
+  let embedRetryTimer = null;
+  // boot 回填首查计时器。搬移前在 index.js 是裸 setTimeout(不参与卸载清理);
+  // 纳入 dispose 是修悬挂,不改变启动行为。
+  let reindexTimer = null;
+
+  if (lightMode) {
+    // Light mode: the whole vector pipeline stays off — no embedder (nothing
+    // pulls in ONNX/transformers), no reranker, no boot backfill (the preset
+    // also cleared autoReindexOnBoot). Recall degrades to keyword search and
+    // human mirror edits still merge on boot.
+    applyHumanEdits();
+  } else if (cfg.embedProvider === "openai") {
+    // vectorIndex is passed so the legacy OpenAI embedder records the producing
+    // model fingerprint after each successful embed (Bug3).
+    embedder = createEmbedder({ store, settings, logger, vectorIndex });
+    service.setEmbedder(embedder);
+    // issue #135: 未配置时明确告警一次。此前 legacy OpenAI embedder 恒报
+    // ready=true，向量层「绿的但全哑」可以静默存在很久（本机持续了数周）。
+    // 只记日志、不阻断启动：轻量模式与「先跑起来再补配置」都是正当用法。
+    if (embedder.configured === false) {
+      logger?.warn?.(
+        "[dsh-mneme] 向量层未配置（vector-config 的 enabled/baseUrl/apiKey/model 有缺）："
+        + "语义召回、语义去重、rerank、sleep 冲突检测将静默失效，"
+        + "dream 的语义聚类会退化为全量窗口兜底。"
+        + "请在设置面板补全 embedding 端点与模型，或把 embedProvider 改为 local/ollama。"
+      );
+    }
+    // legacy OpenAI embedder needs no async init → human edits apply right away
+    applyHumanEdits();
+  } else {
+    try {
+      embedder = createEmbedderByProvider(cfg.embedProvider, {
+        model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
+        dimension: cfg.localEmbedDimension,
+        device: cfg.localEmbedDevice,
+        batchSize: cfg.localEmbedBatchSize,
+        // 池化方式必须与模型的训练口径一致（BGE 系 = CLS）。它既进 embed() 的调用，
+        // 也进 modelHash —— 池化改了就是换向量空间，既有索引会被判失配并重建。
+        pooling: cfg.localEmbedPooling,
+        cacheDir: cfg.embedModelCacheDir,
+        runtimeDir: cfg.runtimeDir,
+        // #188：embedModelMirror 接成 transformers 的下载镜像（此前死配置）。
+        remoteHost: cfg.embedModelMirror,
+        resilientModelDownload: cfg.resilientModelDownload,
+        baseUrl: cfg.ollamaBaseUrl,
+        logger
+      });
+      service.setEmbedder(embedder);
+      // issue #6: wait for extractor init before applying human edits, so
+      // scheduled embeddings see a ready embedder.
+      const bootEmbedder = () => embedder.init()
+        .then(() => { applyHumanEdits(); return true; })
+        .catch(() => false);
+      // #118: the old one-shot probe permanently degraded search to keyword
+      // when Ollama was briefly unreachable at boot (recoverable only by
+      // restart). Retry briefly (5 attempts total: 1 initial + 4 × 15s);
+      // search degrades to keyword meanwhile because per-query embed failures
+      // are swallowed.
+      bootEmbedder().then((ok) => {
+        if (ok) return;
+        let tries = 4;
+        const retry = () => {
+          if (tries-- <= 0) {
+            logger?.warn?.("[dsh-mneme] embedder init retries exhausted, search degrades to keyword");
+            service.setEmbedder(null);
+            applyHumanEdits();
+            return;
+          }
+          embedRetryTimer = setTimeout(async () => {
+            if (await bootEmbedder()) return;
+            retry();
+          }, 15_000);
+        };
+        logger?.warn?.("[dsh-mneme] embedder init failed, retrying");
+        retry();
+      });
+    } catch (error) {
+      logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
+      applyHumanEdits();
+    }
+  }
+
+  // Cross-encoder rerank over recall candidates. Best-effort: a failed model
+  // load only disables reranking, never search itself. Explicit opt-in only
+  // (rerankEnabled defaults to false): constructing LocalReranker is what pulls
+  // in onnxruntime, so the default config never loads it (item ⑥).
+  if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
+    try {
+      reranker = new LocalReranker({
+        model: cfg.rerankModel,
+        batchSize: cfg.rerankBatchSize,
+        maxCandidates: cfg.rerankMaxCandidates,
+        scoreThreshold: cfg.rerankScoreThreshold,
+        device: cfg.localEmbedDevice,
+        cacheDir: cfg.embedModelCacheDir,
+        runtimeDir: cfg.runtimeDir,
+        // #188：量化档默认 q8（此前不传 dtype 会去要 1GB 级 fp32 模型）；
+        // embedModelMirror 此前是死配置，现接成 transformers 的下载镜像。
+        useDtype: cfg.rerankDtype,
+        remoteHost: cfg.embedModelMirror,
+        resilientModelDownload: cfg.resilientModelDownload,
+        logger
+      });
+      service.setReranker(reranker);
+      reranker.init().catch((error) => {
+        logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
+        service.setReranker(null);
+      });
+    } catch (error) {
+      logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
+    }
+  }
+
+  // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
+  // is configured and rows still lack an embedding (e.g. written before vector
+  // search was enabled), the backfill runs in the background after a short
+  // delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
+  // large backlog never floods the provider. Failures degrade silently —
+  // search stays keyword.
+  function scheduleAutoReindex() {
+    if (cfg.autoReindexOnBoot === false) return;
+    const attempt = (tries) => {
+      try {
+        if (!embedder || typeof embedder.embedSingle !== "function") return;
+        if ("ready" in embedder && embedder.ready !== true) {
+          // Local/ollama embedders init asynchronously; give them a moment
+          // before giving up on this boot (next boot retries).
+          // CodeRabbit on #365:嵌套重试计时器同样入册,否则 dispose 后仍可能
+          // 对已关库跑 needsEmbedding(有 try/catch 兜底只是日志噪声,但状态要收干净)。
+          if (tries > 0) reindexTimer = setTimeout(() => attempt(tries - 1), 2000);
+          return;
+        }
+        if (!store.needsEmbedding(1).length) return; // nothing to backfill
+        // Issue #128: no fingerprint gate here anymore — a matching fingerprint
+        // used to return early and permanently orphan rows whose embed failed
+        // at write time. See backfillMissingEmbeddings().
+        backfillMissingEmbeddings({ store, embedder, vectorIndex, logger })
+          .catch((error) => {
+            logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
+          });
+      } catch (error) {
+        logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
+      }
+    };
+    reindexTimer = setTimeout(() => attempt(5), 5000);
+  }
+  scheduleAutoReindex();
+
+  return {
+    embedder,
+    reranker,
+    dispose() {
+      if (embedRetryTimer !== null) { clearTimeout(embedRetryTimer); embedRetryTimer = null; }
+      if (reindexTimer !== null) { clearTimeout(reindexTimer); reindexTimer = null; }
+    }
+  };
+}