@modusensus/dsh-mneme 0.1.5 → 0.2.0

package/src/embedding.js

@@ -1,97 +1,97 @@
-// OpenAI-compatible embedding client for vector search. DSH's LLM service is
-// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
-// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
-// proxy) and any provider exposing the standard embeddings API.
-const DEFAULT_TIMEOUT_MS = 15000;
-
-/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
-function embeddingsUrl(baseUrl) {
-  const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
-  if (!base) return "";
-  // Accept both "https://host/v1" and a full path ending in /embeddings.
-  if (/\/embeddings$/i.test(base)) return base;
-  return `${base}/embeddings`;
-}
-
-/**
- * Call the embeddings API for one text. Resolves to a Float64 array, or null
- * when the provider is not configured, the call fails, or the response is
- * unusable. Never throws: failures degrade to keyword search.
- */
-export async function embedText({ baseUrl, apiKey, model }, text) {
-  const url = embeddingsUrl(baseUrl);
-  if (!url || !apiKey || !model || !text) return null;
-  let res;
-  try {
-    res = await fetch(url, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "Authorization": `Bearer ${apiKey}`
-      },
-      body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
-      signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
-    });
-  } catch {
-    return null;
-  }
-  if (!res.ok) return null;
-  let body;
-  try {
-    body = await res.json();
-  } catch {
-    return null;
-  }
-  const vec = body?.data?.[0]?.embedding;
-  return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
-}
-
-/**
- * Embedder bound to the current settings + store: on each write it re-embeds
- * the row's title+content and stores the vector. Failures are swallowed so a
- * flaky embedding endpoint never breaks memory writes.
- */
-export function createEmbedder({ store, settings, logger }) {
-  async function embedFor(id, title, content) {
-    const cfg = settings.getVectorConfig();
-    if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
-    const text = [title, content].filter(Boolean).join("\n");
-    const vector = await embedText(cfg, text);
-    if (vector) {
-      store.setEmbedding(id, vector);
-      logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
-    }
-  }
-
-  return {
-    /** Fire-and-forget re-embed of a memory after any write. */
-    schedule(memory) {
-      if (!memory?.id) return;
-      embedFor(memory.id, memory.title, memory.content).catch(() => {});
-    },
-
-    /** Embed one text and return its vector (null on failure/disabled). */
-    async embed(query) {
-      const cfg = settings.getVectorConfig();
-      if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
-      return embedText(cfg, query);
-    },
-
-    /** Batch re-index rows still missing an embedding. */
-    async reindexMissing(limit = 50) {
-      const cfg = settings.getVectorConfig();
-      if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
-      const rows = store.needsEmbedding(limit);
-      let indexed = 0;
-      for (const row of rows) {
-        const text = [row.title, row.content].filter(Boolean).join("\n");
-        const vector = await embedText(cfg, text);
-        if (vector) {
-          store.setEmbedding(row.id, vector);
-          indexed++;
-        }
-      }
-      return { indexed, skipped: rows.length - indexed };
-    }
-  };
-}
+// OpenAI-compatible embedding client for vector search. DSH's LLM service is
+// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
+// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
+// proxy) and any provider exposing the standard embeddings API.
+const DEFAULT_TIMEOUT_MS = 15000;
+
+/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
+function embeddingsUrl(baseUrl) {
+  const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
+  if (!base) return "";
+  // Accept both "https://host/v1" and a full path ending in /embeddings.
+  if (/\/embeddings$/i.test(base)) return base;
+  return `${base}/embeddings`;
+}
+
+/**
+ * Call the embeddings API for one text. Resolves to a Float64 array, or null
+ * when the provider is not configured, the call fails, or the response is
+ * unusable. Never throws: failures degrade to keyword search.
+ */
+export async function embedText({ baseUrl, apiKey, model }, text) {
+  const url = embeddingsUrl(baseUrl);
+  if (!url || !apiKey || !model || !text) return null;
+  let res;
+  try {
+    res = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
+      signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
+    });
+  } catch {
+    return null;
+  }
+  if (!res.ok) return null;
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    return null;
+  }
+  const vec = body?.data?.[0]?.embedding;
+  return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
+}
+
+/**
+ * Embedder bound to the current settings + store: on each write it re-embeds
+ * the row's title+content and stores the vector. Failures are swallowed so a
+ * flaky embedding endpoint never breaks memory writes.
+ */
+export function createEmbedder({ store, settings, logger }) {
+  async function embedFor(id, title, content) {
+    const cfg = settings.getVectorConfig();
+    if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
+    const text = [title, content].filter(Boolean).join("\n");
+    const vector = await embedText(cfg, text);
+    if (vector) {
+      store.setEmbedding(id, vector);
+      logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
+    }
+  }
+
+  return {
+    /** Fire-and-forget re-embed of a memory after any write. */
+    schedule(memory) {
+      if (!memory?.id) return;
+      embedFor(memory.id, memory.title, memory.content).catch(() => {});
+    },
+
+    /** Embed one text and return its vector (null on failure/disabled). */
+    async embed(query) {
+      const cfg = settings.getVectorConfig();
+      if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
+      return embedText(cfg, query);
+    },
+
+    /** Batch re-index rows still missing an embedding. */
+    async reindexMissing(limit = 50) {
+      const cfg = settings.getVectorConfig();
+      if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
+      const rows = store.needsEmbedding(limit);
+      let indexed = 0;
+      for (const row of rows) {
+        const text = [row.title, row.content].filter(Boolean).join("\n");
+        const vector = await embedText(cfg, text);
+        if (vector) {
+          store.setEmbedding(row.id, vector);
+          indexed++;
+        }
+      }
+      return { indexed, skipped: rows.length - indexed };
+    }
+  };
+}

package/src/index.js

@@ -1,120 +1,176 @@
-import { createStore } from "./store.js";
-import { createMirror, TYPE_FILE } from "./mirror.js";
-import { createService } from "./service.js";
-import { createTools } from "./tools.js";
-import { createInjector } from "./inject.js";
-import { createSummarizer } from "./summarize.js";
-import { createDreamScheduler } from "./dream.js";
-import { createApi } from "./api.js";
-import { createSettings } from "./settings.js";
-import { createCommandManager } from "./commands.js";
-import { createEmbedder } from "./embedding.js";
-import { Config } from "./config.js";
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { homedir } from "node:os";
-
-export const name = "dsh-mneme";
-export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
-export { Config };
-
-// Arrow (not function declaration): cordis 4 treats any apply with a
-// prototype as a class constructor (`new apply(...)`) and discards its return
-// value, so a `function apply` disposer would never run on unload. An arrow
-// has no prototype, is called normally, and its returned disposer is collected
-// and run by the fiber on unload.
-export const apply = (ctx, config) => {
-  const cfg = Config(config);
-
-  // Resolve memoryDir: expand leading "~"
-  const memoryDir = cfg.memoryDir.startsWith("~")
-    ? join(homedir(), cfg.memoryDir.slice(1))
-    : cfg.memoryDir;
-  mkdirSync(memoryDir, { recursive: true });
-
-  const store = createStore(join(memoryDir, "memory.db"));
-  const mirror = createMirror(memoryDir);
-  const service = createService({ store, mirror, config: cfg });
-
-  // User-configurable settings (profile, rules) and custom commands share the
-  // same SQLite file but live in dedicated tables, isolated from memories.
-  const settings = createSettings(store.db);
-
-  // Vector search: embedder calls the configured OpenAI-compatible embeddings
-  // endpoint on writes and for queries. service re-embeds after each write.
-  const embedder = createEmbedder({ store, settings, logger: ctx.logger });
-  service.setEmbedder(embedder);
-
-  // Custom commands: register persisted commands into the DSH command registry
-  // on boot; add/remove re-register live through the API.
-  let commands = null;
-  if (ctx.commands) {
-    commands = createCommandManager({ ctx, settings, logger: ctx.logger });
-    commands.sync();
-  }
-
-  // Human edits in mirror files win on every sync; merge them back first.
-  // TYPE_FILE maps each memory type to its mirror filename. Read every type's
-  // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
-  // a per-type read-then-merge loop would overwrite edits in files not yet read
-  // (e.g. preferences.md merging would clobber unsynced projects.md edits).
-  const humanEdits = new Map();
-  for (const type of Object.keys(TYPE_FILE)) {
-    humanEdits.set(type, mirror.readHumanEdits(type));
-  }
-  for (const [type, edits] of humanEdits) {
-    if (edits.length) service.mergeHumanEdits(type, edits);
-  }
-
-  // Dream scheduler: automatic consolidation + summary runs, triggered by
-  // store growth. Writes through the service fire the dream hook, which asks
-  // the scheduler to (re)schedule a run once absolute and since-last-run
-  // thresholds are both exceeded. onRun is deferred through `dream` so the
-  // closure sees the assigned scheduler; the null guard keeps a run safe even
-  // if the hook fires before assignment or after dispose.
-  let dream = null;
-  if (cfg.autoDream) {
-    dream = createDreamScheduler({
-      thresholdCount: cfg.dreamThresholdCount,
-      thresholdChars: cfg.dreamThresholdChars,
-      delayMs: cfg.dreamDelayMs,
-      logger: ctx.logger,
-      onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
-    });
-    service.setDreamHook(() => dream.maybeSchedule(service));
-  }
-
-  const disposers = [];
-
-  ctx.inject(["systemPrompt"], (promptCtx) => {
-    if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
-  });
-
-  ctx.inject(["tools"], (toolsCtx) => {
-    disposers.push(createTools(toolsCtx, service, cfg));
-  });
-
-  const summarizer = createSummarizer(ctx, service, cfg);
-  disposers.push(summarizer.dispose);
-
-  if (ctx.webServer) {
-    const api = createApi(ctx, service, settings, commands ?? {
-      add: () => { throw new Error("commands unavailable"); },
-      remove: () => false,
-      list: () => []
-    }, embedder);
-    disposers.push(api.dispose);
-  }
-
-  // Async disposer: cordis awaits the returned promise on unload (runDisposable),
-  // so an in-flight dream run is allowed to finish before the SQLite store is
-  // closed — dream.dispose() resolves only after its current run settles.
-  return async () => {
-    for (const dispose of disposers) {
-      if (typeof dispose === "function") dispose();
-    }
-    commands?.dispose();
-    if (dream) await dream.dispose();
-    store.close();
-  };
-};
+import { createStore } from "./store.js";
+import { createMirror, TYPE_FILE } from "./mirror.js";
+import { createService } from "./service.js";
+import { createTools } from "./tools.js";
+import { createInjector } from "./inject.js";
+import { createSummarizer } from "./summarize.js";
+import { createDreamScheduler } from "./dream.js";
+import { createApi } from "./api.js";
+import { createSettings } from "./settings.js";
+import { createCommandManager } from "./commands.js";
+import { createEmbedder } from "./embedding.js";
+import { createEmbedderByProvider } from "./local-embedder.js";
+import { LocalReranker } from "./reranker.js";
+import { createVectorIndex } from "./vector-index.js";
+import { Config } from "./config.js";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+export const name = "dsh-mneme";
+export const inject = ["tools", "systemPrompt", "webServer", "llm", "agentDefaultModel", "commands"];
+export { Config };
+
+// Arrow (not function declaration): cordis 4 treats any apply with a
+// prototype as a class constructor (`new apply(...)`) and discards its return
+// value, so a `function apply` disposer would never run on unload. An arrow
+// has no prototype, is called normally, and its returned disposer is collected
+// and run by the fiber on unload.
+export const apply = (ctx, config) => {
+  const cfg = Config(config);
+
+  // Resolve memoryDir: expand leading "~"
+  const memoryDir = cfg.memoryDir.startsWith("~")
+    ? join(homedir(), cfg.memoryDir.slice(1))
+    : cfg.memoryDir;
+  mkdirSync(memoryDir, { recursive: true });
+
+  const store = createStore(join(memoryDir, "memory.db"));
+  const mirror = createMirror(memoryDir);
+  const service = createService({ store, mirror, config: cfg });
+
+  // User-configurable settings (profile, rules) and custom commands share the
+  // same SQLite file but live in dedicated tables, isolated from memories.
+  const settings = createSettings(store.db);
+
+  // Semantic pipeline: a local/ollama embedder when configured, otherwise the
+  // legacy OpenAI-compatible embedder (settings-driven). The vector index wraps
+  // the store's embedding column and tracks the active model fingerprint. A
+  // slow embedder init (model download) never blocks plugin boot — failures
+  // degrade to keyword search.
+  const vectorIndex = createVectorIndex({ store, logger: ctx.logger });
+  service.setVectorIndex(vectorIndex);
+
+  let embedder = null;
+  let reranker = null;
+  if (cfg.embedProvider === "openai") {
+    embedder = createEmbedder({ store, settings, logger: ctx.logger });
+    service.setEmbedder(embedder);
+  } else {
+    try {
+      embedder = createEmbedderByProvider(cfg.embedProvider, {
+        model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
+        dimension: cfg.localEmbedDimension,
+        device: cfg.localEmbedDevice,
+        batchSize: cfg.localEmbedBatchSize,
+        cacheDir: cfg.embedModelCacheDir,
+        baseUrl: cfg.ollamaBaseUrl,
+        logger: ctx.logger
+      });
+      service.setEmbedder(embedder);
+      embedder.init().catch((error) => {
+        ctx.logger?.warn?.(`[dsh-mneme] embedder init failed, search degrades to keyword: ${String(error)}`);
+        service.setEmbedder(null);
+      });
+    } catch (error) {
+      ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
+    }
+  }
+
+  // Cross-encoder rerank over recall candidates. Best-effort: a failed model
+  // load only disables reranking, never search itself.
+  if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
+    try {
+      reranker = new LocalReranker({
+        model: cfg.rerankModel,
+        batchSize: cfg.rerankBatchSize,
+        maxCandidates: cfg.rerankMaxCandidates,
+        scoreThreshold: cfg.rerankScoreThreshold,
+        device: cfg.localEmbedDevice,
+        cacheDir: cfg.embedModelCacheDir,
+        logger: ctx.logger
+      });
+      service.setReranker(reranker);
+      reranker.init().catch((error) => {
+        ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
+        service.setReranker(null);
+      });
+    } catch (error) {
+      ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
+    }
+  }
+
+  // Custom commands: register persisted commands into the DSH command registry
+  // on boot; add/remove re-register live through the API.
+  let commands = null;
+  if (ctx.commands) {
+    commands = createCommandManager({ ctx, settings, logger: ctx.logger });
+    commands.sync();
+  }
+
+  // Human edits in mirror files win on every sync; merge them back first.
+  // TYPE_FILE maps each memory type to its mirror filename. Read every type's
+  // edits up front: mergeHumanEdits re-renders ALL mirror files on success, so
+  // a per-type read-then-merge loop would overwrite edits in files not yet read
+  // (e.g. preferences.md merging would clobber unsynced projects.md edits).
+  const humanEdits = new Map();
+  for (const type of Object.keys(TYPE_FILE)) {
+    humanEdits.set(type, mirror.readHumanEdits(type));
+  }
+  for (const [type, edits] of humanEdits) {
+    if (edits.length) service.mergeHumanEdits(type, edits);
+  }
+
+  // Dream scheduler: automatic consolidation + summary runs, triggered by
+  // store growth. Writes through the service fire the dream hook, which asks
+  // the scheduler to (re)schedule a run once absolute and since-last-run
+  // thresholds are both exceeded. onRun is deferred through `dream` so the
+  // closure sees the assigned scheduler; the null guard keeps a run safe even
+  // if the hook fires before assignment or after dispose.
+  let dream = null;
+  if (cfg.autoDream) {
+    dream = createDreamScheduler({
+      thresholdCount: cfg.dreamThresholdCount,
+      thresholdChars: cfg.dreamThresholdChars,
+      delayMs: cfg.dreamDelayMs,
+      logger: ctx.logger,
+      semantic: { embedder, vectorIndex },
+      onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
+    });
+    service.setDreamHook(() => dream.maybeSchedule(service));
+  }
+
+  const disposers = [];
+
+  ctx.inject(["systemPrompt"], (promptCtx) => {
+    if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
+  });
+
+  ctx.inject(["tools"], (toolsCtx) => {
+    disposers.push(createTools(toolsCtx, service, cfg, embedder));
+  });
+
+  const summarizer = createSummarizer(ctx, service, cfg);
+  disposers.push(summarizer.dispose);
+
+  if (ctx.webServer) {
+    const api = createApi(ctx, service, settings, commands ?? {
+      add: () => { throw new Error("commands unavailable"); },
+      remove: () => false,
+      list: () => []
+    }, embedder, { vectorIndex, reranker });
+    disposers.push(api.dispose);
+  }
+
+  // Async disposer: cordis awaits the returned promise on unload (runDisposable),
+  // so an in-flight dream run is allowed to finish before the SQLite store is
+  // closed — dream.dispose() resolves only after its current run settles.
+  return async () => {
+    for (const dispose of disposers) {
+      if (typeof dispose === "function") dispose();
+    }
+    commands?.dispose();
+    if (dream) await dream.dispose();
+    store.close();
+  };
+};

package/src/inject.js

@@ -1,47 +1,47 @@
-export function createInjector(ctx, service, settings, config) {
-  const maxItems = config.maxInjectedItems ?? 5;
-  const threshold = config.importanceThreshold ?? 3;
-
-  function render(candidates) {
-    if (!candidates.length) return "";
-    const lines = ["[记忆库] 来自 dsh-mneme 的跨会话记忆（用户偏好与高优先级项目/决策）："];
-    for (const m of candidates) {
-      lines.push(`- [${m.type}] ${m.title}（重要性 ${m.importance}）：${m.content}`);
-    }
-    return lines.join("\n");
-  }
-
-  // User profile + rules: injected ahead of the memory block because they are
-  // always-relevant instructions the agent should follow every turn.
-  function renderUserSettings() {
-    const profile = settings.getProfile().trim();
-    const rules = settings.getRules();
-    if (!profile && !rules.length) return "";
-    const lines = ["[用户设置] 来自 dsh-mneme 的用户画像与规则："];
-    if (profile) lines.push(`- 用户画像：${profile}`);
-    for (const rule of rules) lines.push(`- 规则：${rule}`);
-    return lines.join("\n");
-  }
-
-  const disposers = [
-    ctx.systemPrompt.context({
-      name: "memory",
-      order: 90,
-      text: () => {
-        const candidates = service.injectCandidates({ maxItems, threshold });
-        return render(candidates);
-      }
-    }),
-    ctx.systemPrompt.context({
-      name: "user-settings",
-      order: 85,
-      text: renderUserSettings
-    })
-  ];
-
-  return () => {
-    for (const dispose of disposers) {
-      if (typeof dispose === "function") dispose();
-    }
-  };
-}
+export function createInjector(ctx, service, settings, config) {
+  const maxItems = config.maxInjectedItems ?? 5;
+  const threshold = config.importanceThreshold ?? 3;
+
+  function render(candidates) {
+    if (!candidates.length) return "";
+    const lines = ["[记忆库] 来自 dsh-mneme 的跨会话记忆（用户偏好与高优先级项目/决策）："];
+    for (const m of candidates) {
+      lines.push(`- [${m.type}] ${m.title}（重要性 ${m.importance}）：${m.content}`);
+    }
+    return lines.join("\n");
+  }
+
+  // User profile + rules: injected ahead of the memory block because they are
+  // always-relevant instructions the agent should follow every turn.
+  function renderUserSettings() {
+    const profile = settings.getProfile().trim();
+    const rules = settings.getRules();
+    if (!profile && !rules.length) return "";
+    const lines = ["[用户设置] 来自 dsh-mneme 的用户画像与规则："];
+    if (profile) lines.push(`- 用户画像：${profile}`);
+    for (const rule of rules) lines.push(`- 规则：${rule}`);
+    return lines.join("\n");
+  }
+
+  const disposers = [
+    ctx.systemPrompt.context({
+      name: "memory",
+      order: 90,
+      text: () => {
+        const candidates = service.injectCandidates({ maxItems, threshold });
+        return render(candidates);
+      }
+    }),
+    ctx.systemPrompt.context({
+      name: "user-settings",
+      order: 85,
+      text: renderUserSettings
+    })
+  ];
+
+  return () => {
+    for (const dispose of disposers) {
+      if (typeof dispose === "function") dispose();
+    }
+  };
+}