@modusensus/dsh-mneme 0.3.5 → 0.3.7

package/README.md

@@ -5,7 +5,7 @@
 [![npm version](https://img.shields.io/npm/v/@modusensus/dsh-mneme?color=blue&label=npm)](https://www.npmjs.com/package/@modusensus/dsh-mneme)
 [![license](https://img.shields.io/badge/license-MIT-green)](LICENSE)
 [![Awesome](https://awesome-dsh-plugin.com/badge.svg)](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
-[![tests](https://img.shields.io/badge/tests-429%20passed-success)](https://github.com/modusensus/dsh-mneme)
+[![tests](https://img.shields.io/badge/tests-443%20passed-success)](https://github.com/modusensus/dsh-mneme)
 
 > 给 DeepSeek Harness 的跨会话记忆插件：让 Agent 记住你、记住项目、自动整理记忆。**Mneme**（Μνήμη）——希腊记忆女神 Mnemosyne 之名，掌管记忆与梦境，正如 autoDream 在后台巩固记忆。
 
@@ -243,7 +243,7 @@ src/
 lib/
 ├── client.js         # Web 面板（手写 ModuleLoader bundle）
 └── *.js              # src 的同步分发产物
-test/                 # 429 个 node:test 测试（含审计与三轴线压测不变量）
+test/                 # 443 个 node:test 测试（含审计与三轴线压测不变量）
 scripts/              # e2e-dsh.js 端到端演示 · stress-dsh.js 三轴线压测 · sync-lib.js 同步
 ```
 
@@ -252,7 +252,7 @@ scripts/              # e2e-dsh.js 端到端演示 · stress-dsh.js 三轴线压
 ```bash
 cd dsh-mneme
 npm install        # 安装 peer 依赖（以 devDependencies 形式，用于本地测试）
-npm test           # 运行 429 个测试
+npm test           # 运行 443 个测试
 npm run stress     # 三轴线压测：长会话检索 / 冲突仲裁 / 多 Agent 并发（离线 mock LLM）
 npm run sync       # 把 src/ 同步到 lib/（发布时由 prepack 钩子自动执行）
 ```
@@ -265,9 +265,6 @@ npm run sync       # 把 src/ 同步到 lib/（发布时由 prepack 钩子自动
 
 > 设计文档位于仓库根 `docs/`，链接以 `../docs/` 相对路径指向（GitHub 上从本目录打开可正常跳转）。
 
-- [记忆库设计](../docs/superpowers/specs/2026-08-13-dsh-mneme-design.md)
-- [autoDream 设计](../docs/superpowers/specs/2026-08-13-dsh-mneme-autodream-design.md)
-- [实施计划](../docs/superpowers/plans/2026-08-13-dsh-memory-autodream.md)
 - [实体结构化记忆设计](docs/ENTITIES.md)
 - [语义增强架构](docs/SEMANTIC.md)
 - [本地模型部署指南](docs/LOCAL_MODEL.md)

package/lib/api.js

@@ -268,21 +268,43 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
     }
   });
 
-  // --- health: mirror sync state (F-NEW-03) ---
+  // --- health: mirror sync state (F-NEW-03 / v0.3.6) ---
+  // Auth-gated; only returns a sanitized error code (never raw last_error which
+  // may leak paths/token-like strings/internal hosts). On state read failure it
+  // reports unknown/degraded (fail-closed) instead of a false dirty=false.
   register({
     kind: "exact",
     path: "/api/dsh-mneme/health",
     handler(req, res) {
+      if (!requireAuth(req, res, apiToken)) return;
+      let state = null;
       try {
-        const state = service.getMirrorHealth?.() ?? null;
-        sendJson(res, 200, {
-          mirror: state
-            ? { dirty: state.dirty === true, last_error: state.last_error ?? null, last_attempt: state.last_attempt ?? null, success_at: state.success_at ?? null }
-            : null
-        });
+        state = service.getMirrorHealth?.() ?? null;
       } catch {
-        sendJson(res, 500, { error: "internal" });
+        // read failure is itself a health signal: do not report a false clean
+        sendJson(res, 200, { mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null } });
+        return;
       }
+      if (!state) {
+        sendJson(res, 200, { mirror: { dirty: null, status: "unknown", last_error: null, last_attempt: null, success_at: null } });
+        return;
+      }
+      // Sanitized: boolean dirty + coarse status only; error string is mapped to
+      // a bounded code, never echoed verbatim.
+      let code = null;
+      if (state.last_error) {
+        const e = String(state.last_error);
+        code = /enospc|no space/i.test(e) ? "no-space" : /permission|eacces/i.test(e) ? "permission" : "sync-failed";
+      }
+      sendJson(res, 200, {
+        mirror: {
+          dirty: state.dirty === true,
+          status: state.dirty === true ? "degraded" : (code ? "degraded" : "ok"),
+          last_error: code,
+          last_attempt: state.last_attempt ?? null,
+          success_at: state.success_at ?? null
+        }
+      });
     }
   });
 

package/lib/index.js

@@ -80,11 +80,28 @@ export const apply = (ctx, config) => {
   const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
   service.setVectorIndex(vectorIndex);
 
+  // Human edits in mirror files win on every sync; merge them back first.
+  // TYPE_FILE maps each memory type to its mirror filename. Read every type's
+  // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
+  // a per-type read-then-merge loop would overwrite edits in files not yet read
+  // (e.g. preferences.md merging would clobber unsynced projects.md edits).
+  const humanEdits = new Map();
+  for (const type of Object.keys(TYPE_FILE)) {
+    humanEdits.set(type, mirror.readHumanEdits(type));
+  }
+  const applyHumanEdits = () => {
+    for (const [type, edits] of humanEdits) {
+      if (edits.length) service.mergeHumanEdits(type, edits);
+    }
+  };
+
   let embedder = null;
   let reranker = null;
   if (cfg.embedProvider === "openai") {
     embedder = createEmbedder({ store, settings, logger: ctx.logger });
     service.setEmbedder(embedder);
+    // legacy OpenAI embedder is immediately usable
+    applyHumanEdits();
   } else {
     try {
       embedder = createEmbedderByProvider(cfg.embedProvider, {
@@ -97,12 +114,18 @@ export const apply = (ctx, config) => {
         logger: ctx.logger
       });
       service.setEmbedder(embedder);
-      embedder.init().catch((error) => {
-        ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
-        service.setEmbedder(null);
-      });
+      // issue #6: wait for extractor init before applying human edits, so
+      // scheduled embeddings see a ready embedder.
+      embedder.init()
+        .then(() => applyHumanEdits())
+        .catch((error) => {
+          ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
+          service.setEmbedder(null);
+          applyHumanEdits();
+        });
     } catch (error) {
       ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
+      applyHumanEdits();
     }
   }
 
@@ -139,19 +162,6 @@ export const apply = (ctx, config) => {
     commands.sync();
   }
 
-  // Human edits in mirror files win on every sync; merge them back first.
-  // TYPE_FILE maps each memory type to its mirror filename. Read every type's
-  // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
-  // a per-type read-then-merge loop would overwrite edits in files not yet read
-  // (e.g. preferences.md merging would clobber unsynced projects.md edits).
-  const humanEdits = new Map();
-  for (const type of Object.keys(TYPE_FILE)) {
-    humanEdits.set(type, mirror.readHumanEdits(type));
-  }
-  for (const [type, edits] of humanEdits) {
-    if (edits.length) service.mergeHumanEdits(type, edits);
-  }
-
   // Dream scheduler: automatic consolidation + summary runs, triggered by
   // store growth. Writes through the service fire the dream hook, which asks
   // the scheduler to (re)schedule a run once absolute and since-last-run

package/lib/local-embedder.js

@@ -58,16 +58,21 @@ export class LocalEmbedder {
     // Test hook: replace the pipeline factory without touching modules.
     this.engineFactory = opts.engineFactory || defaultPipelineLoader;
     this.extractor = null;
+    // issue #6: readiness flag for the service's scheduleEmbed gate. False until
+    // init() succeeds, so "ready" in embedder is observable even pre-init.
+    this.ready = false;
   }
 
-  /** Load the model; throws when it cannot be loaded. */
+  /** Load the model; throws when it cannot be loaded. Idempotent. */
   async init() {
+    if (this.extractor) return this; // already initialized: no-op
     const options = {
       dtype: this.useDtype,
       device: this.device
     };
     if (this.cacheDir) options.cache_dir = this.cacheDir;
     this.extractor = await this.engineFactory("feature-extraction", this.model, options);
+    this.ready = true; // service reads this to flush queued re-embeds
     this.logger?.info?.(
       `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
     );
@@ -107,6 +112,7 @@ export class LocalEmbedder {
       // best-effort: some engines free resources on GC
     }
     this.extractor = null;
+    this.ready = false;
   }
 }
 

package/lib/service.js

@@ -37,11 +37,69 @@ export function createService({ store, mirror, config, onWrite, logger }) {
   // replays them exactly once against the committed state.
   let txDepth = 0;
 
+  // issue #6 (part 2): startup race defense. A local embedder (LocalEmbedder /
+  // Ollama) exposes an async init(), so between `setEmbedder` and init()
+  // resolving there is a window where embedSingle would throw "not initialized"
+  // and the re-embed would be silently dropped. When the embedder carries a
+  // `ready` flag we queue writes in embedPending until init sets ready=true,
+  // then flush them through the embedder's real interface. Embedders without a
+  // `ready` flag (legacy OpenAI, instantly usable) keep their old behavior.
+  let embedPending = [];
+  let embedReadyTimer = null;
+  const EMBED_PENDING_MAX = 100; // bound the queue; drop oldest beyond this
+  const EMBED_READY_POLL_MS = 100;
+  const EMBED_READY_POLL_LIMIT = 30; // ~3s ceiling; never poll forever
+
+  /** Flush the queued re-embeds once the embedder is ready. Fail-safe. */
+  function flushEmbedPending() {
+    if (!embedder || embedPending.length === 0) return;
+    const batch = embedPending.splice(0, embedPending.length);
+    for (const memory of batch) {
+      try {
+        if (!memory?.id) continue;
+        if (typeof embedder.schedule === "function") {
+          embedder.schedule(memory);
+        } else if (typeof embedder.embedSingle === "function") {
+          const text = [memory.title, memory.content].filter(Boolean).join("\n");
+          if (!text) continue;
+          embedder
+            .embedSingle(text)
+            .then((vec) => {
+              if (Array.isArray(vec) && vec.length) {
+                store.setEmbedding(memory.id, vec);
+              }
+            })
+            .catch((err) => {
+              logger?.warn?.("flushEmbedPending embedSingle failed:", err);
+            });
+        }
+      } catch (err) {
+        logger?.warn?.("flushEmbedPending failed:", err);
+      }
+    }
+  }
+
+  function stopEmbedReadyPolling() {
+    if (embedReadyTimer) {
+      clearInterval(embedReadyTimer);
+      embedReadyTimer = null;
+    }
+  }
+
   function scheduleEmbed(memory) {
     try {
       if (txDepth > 0) return; // deferred to the transaction's commit
       if (!embedder || !memory?.id) return;
 
+      // Readiness gate: embedder exposes `ready` (async init) and is not ready
+      // yet — queue instead of firing embedSingle into a half-built extractor.
+      const hasReady = "ready" in embedder;
+      if (hasReady && embedder.ready !== true) {
+        if (embedPending.length >= EMBED_PENDING_MAX) embedPending.shift();
+        embedPending.push(memory);
+        return;
+      }
+
       if (typeof embedder.schedule === "function") {
         embedder.schedule(memory);
         return;
@@ -110,7 +168,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
 
   /**
    * Search for memories attached to a named entity (v0.3.0 Phase 3).
-   * 合并优先级（桉桉确认）：entity_attrs.memory_id 精确关联 = 1.0 > 关键词提及 = 0.7；
+   * 合并优先级：entity_attrs.memory_id 精确关联 = 1.0 > 关键词提及 = 0.7；
    * attr 命中不覆盖，keyword 只补充召回，最后按 _score 降序取 topK。
    * @param {string} entityName
    * @param {object} [options]
@@ -511,38 +569,82 @@ export function createService({ store, mirror, config, onWrite, logger }) {
    * human-editable file (a human "edit" could otherwise resurrect them).
    */
   // syncMirror: 同步 mirror，并在失败/成功时持久记录 dirty 状态；保证自身不抛出。
-  // 失败写 store.markMirrorDirty（dirty=1 + last_error + last_attempt），成功写
-  // store.markMirrorClean（dirty=0 + last_error=null + success_at）。状态写入用
-  // try/catch 包住，避免 db 关闭等场景下 syncMirror 自身再抛出（F-NEW-03）。
+  // v0.3.6（audit peer 4 阻断）：
+  //   - 开始时 incrementGeneration 绑定本次期望轮次 gen；成功用
+  //     markMirrorCleanForGeneration(gen, now) CAS/fence 清 dirty——旧 worker
+  //     （gen 已过期）不会误清另一 worker 未恢复的故障债务；
+  //   - 失败写 markMirrorDirty（递增 desired 绑定新债务），下次 recover 恢复；
+  //   - 逐 type 用 setTypeStatus 记录部分成功/失败（type_status JSON）；
+  //   - 所有 store 状态写入各自 try/catch，失败只 warn，绝不向外抛（F-NEW-03）。
   function syncMirror() {
     if (txDepth > 0 || !mirror) return; // deferred to the transaction's commit
     const now = new Date().toISOString();
+    let gen;
     try {
-      mirror.sync(reconcileHumanEdits(store.list({ limit: 500, includeForgotten: false })));
-      // 成功：写 clean 状态
+      // 绑定本次期望轮次，必须在任何渲染之前，避免制造幽灵债务
+      const state = store.incrementGeneration();
+      gen = state.generation;
+    } catch (stateError) {
+      logger?.warn?.("syncMirror: incrementGeneration failed:", stateError);
+      return;
+    }
+    // coveredTypes 提到 try 外初始化：即使 store.list 先抛错，catch 分支也有
+    // 合法的空 Set 可迭代，保证 syncMirror 自身绝不抛（fail-safe）。
+    const coveredTypes = new Set();
+    try {
+      // 预先获取本次要覆盖的 type 集合（只调一次 store.list）
+      const list = store.list({ limit: 500, includeForgotten: false });
+      for (const memory of list) {
+        if (memory?.type && TYPE_FILE[memory.type]) {
+          coveredTypes.add(memory.type);
+        }
+      }
+
+      // 全量渲染
+      mirror.sync(reconcileHumanEdits(list));
+
+      // 成功：CAS/fence 绑定到本地 gen，旧 worker（gen 已过期）会被拦截
       try {
-        store.markMirrorClean(now);
+        store.markMirrorCleanForGeneration(gen, now);
       } catch (stateError) {
-        // markMirrorClean 失败不能影响主要同步逻辑，仅记录日志
-        logger?.warn?.("syncMirror: markMirrorClean failed:", stateError);
+        logger?.warn?.("syncMirror: markMirrorCleanForGeneration failed:", stateError);
+      }
+      // 逐 type 标记为 clean
+      for (const type of coveredTypes) {
+        try {
+          store.setTypeStatus(type, { dirty: false, applied_gen: gen, last_error: null });
+        } catch (stateError) {
+          logger?.warn?.(`syncMirror: setTypeStatus(${type}) clean failed:`, stateError);
+        }
       }
     } catch (error) {
-      // 同步失败：写 dirty 状态
       const errMsg = error?.message ?? String(error);
       logger?.warn?.("syncMirror failed:", error);
       try {
+        // 债务绑定到新的一轮（desired generation +1）
         store.markMirrorDirty(errMsg, now);
       } catch (stateError) {
-        // markMirrorDirty 失败同样不能向外抛出
         logger?.warn?.("syncMirror: markMirrorDirty failed:", stateError);
       }
+      // 逐 type 标记为 dirty（applied_gen 不动）
+      for (const type of coveredTypes) {
+        try {
+          store.setTypeStatus(type, { dirty: true, last_error: errMsg });
+        } catch (stateError) {
+          logger?.warn?.(`syncMirror: setTypeStatus(${type}) dirty failed:`, stateError);
+        }
+      }
     }
   }
 
   // recoverMirror: 启动/手动 reconcile 时根据持久 dirty 状态决定是否恢复同步
-  // （F-NEW-03）。dirty=true → 有界重试（最多 3 次，立即重试）重跑 syncMirror
-  // 收敛镜像；某次成功（dirty 变 0）立即停止。返回 { recovered, error } 供
-  // index.js 启动 / api.js health 判断。一切 fail-safe，绝不向外抛。
+  // （F-NEW-03 + v0.3.6）。触发条件不只是 dirty——还检查
+  // generation > applied_generation（有未应用的债务），这样 COMMIT→dirty 崩溃
+  // 窗口（DB 提交后、markMirrorDirty/clean 前进程退出 → dirty=false 但
+  // generation 不一致）也能被捕获。有界重试（最多 3 次）重跑 syncMirror 收敛；
+  // 某次成功后 dirty=false 且无更新债务（generation <= applied_generation）
+  // 立即停止。返回 { recovered, error } 供 index.js 启动 / api.js health 判断。
+  // 一切 fail-safe，绝不向外抛。
   function recoverMirror() {
     const MAX_ATTEMPTS = 3;
     let lastError = null;
@@ -550,24 +652,29 @@ export function createService({ store, mirror, config, onWrite, logger }) {
 
     try {
       const state = store.getMirrorState();
-      if (!state?.dirty) {
+      // 崩溃窗口检测：dirty 或 generation > applied_generation（COMMIT→dirty 窗口）
+      if (!state?.dirty && !(state.generation > state.applied_generation)) {
         // 本来就干净：无需恢复，视为成功
         return { recovered: true, error: null };
       }
 
-      // dirty 状态：最多尝试 3 次 sync
+      // 有 dirty 或有未应用债务：最多尝试 3 次 sync
       for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         try {
           syncMirror(); // syncMirror 内部已 catch，不会向外抛
           const currentState = store.getMirrorState();
-          if (!currentState?.dirty) {
-            // 成功：镜像已收敛为 clean
+          // 成功条件：dirty 为 false 且没有更新一轮的债务
+          // （generation <= applied_generation，恢复后由 syncMirror 里
+          //   markMirrorCleanForGeneration 自动把 applied 跟上）
+          if (!currentState?.dirty && currentState.generation <= currentState.applied_generation) {
             recovered = true;
             lastError = null;
             break;
           }
-          // 仍 dirty：记录最后一次错误供重试耗尽后上报
-          lastError = currentState.last_error || `Sync attempt ${attempt + 1} left mirror dirty`;
+          // 仍 dirty 或仍有更新债务：记录最后一次错误供重试耗尽后上报。
+          // 注意：若别的 worker 又失败产生新债务（dirty 仍 true），这是"新债务"
+          // 不是本次失败，继续重试直到耗尽次数。
+          lastError = currentState.last_error || `Sync attempt ${attempt + 1} left mirror dirty or has pending debt`;
         } catch (syncError) {
           // syncMirror 理论不抛，fail-safe 兜底
           const errMsg = syncError?.message ?? String(syncError);
@@ -579,7 +686,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
       if (!recovered) {
         logger?.warn?.("dsh-mneme mirror: recover failed after", MAX_ATTEMPTS, "attempts");
       } else {
-        logger?.warn?.("dsh-mneme mirror: recovered from dirty state");
+        logger?.warn?.("dsh-mneme mirror: recovered from dirty/pending state");
       }
     } catch (error) {
       // fail-safe：任何意外异常不向外抛
@@ -632,7 +739,32 @@ export function createService({ store, mirror, config, onWrite, logger }) {
     toApiList,
     transaction,
     setDreamHook(fn) { dreamHook = fn; },
-    setEmbedder(emb) { embedder = emb; },
+    setEmbedder(emb) {
+      embedder = emb;
+      if (!emb) {
+        // embedder removed (init failed in index.js): stop polling and drop
+        // queued re-embeds — search just degrades to keyword.
+        stopEmbedReadyPolling();
+        embedPending = [];
+        return;
+      }
+      if (emb.ready === true) {
+        flushEmbedPending();
+        return;
+      }
+      // Async-initializing embedder: poll `ready` until it flips, then flush.
+      if ("ready" in emb && embedReadyTimer === null) {
+        let attempts = 0;
+        embedReadyTimer = setInterval(() => {
+          attempts++;
+          if (emb.ready === true || attempts >= EMBED_READY_POLL_LIMIT) {
+            stopEmbedReadyPolling();
+            if (emb.ready === true) flushEmbedPending();
+            else embedPending = []; // init never landed: drop the queue
+          }
+        }, EMBED_READY_POLL_MS);
+      }
+    },
     setEntityExtractor(fn) { entityExtractor = fn; },
     setVectorIndex(vi) { vectorIndex = vi; },
     setReranker(rn) { reranker = rn; },