@modusensus/dsh-mneme 0.1.6 → 0.2.0

package/src/local-embedder.js

@@ -0,0 +1,265 @@
+// Fully-local embedding backends for dsh-mneme: ONNX via transformers.js,
+// Ollama's HTTP API, and the OpenAI-compatible HTTP API (extracted from the
+// old embedding.js logic). All classes share one interface so the orchestrator
+// can pick a backend by provider name and degrade gracefully on failure.
+// Methods throw on error — the caller decides the fallback chain.
+const DEFAULT_TIMEOUT_MS = 15000;
+
+/** djb2 — stable, fast fingerprint for a provider/model string. */
+function hashString(s) {
+  let h = 5381;
+  for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
+  return h.toString(16);
+}
+
+/** Full provider+model fingerprint used for index-consistency checks. */
+function modelHash(model) {
+  return `${model}#${hashString(model)}`;
+}
+
+/** Lazy default loader: dynamic import keeps module load cheap. */
+async function defaultPipelineLoader(task, model, options) {
+  const { pipeline } = await import("@huggingface/transformers");
+  return pipeline(task, model, options);
+}
+
+/** Flatten a transformers.js Tensor [batch, dim] into number[][]. */
+function tensorToRows(tensor) {
+  const { data, dims } = tensor;
+  const rowLen = dims[dims.length - 1] || 0;
+  const rows = [];
+  for (let i = 0; i < data.length; i += rowLen) {
+    rows.push(Array.from(data.subarray(i, i + rowLen)));
+  }
+  // Single-text input may come back without the batch axis.
+  if (rows.length === 0 && rowLen > 0) rows.push(Array.from(data));
+  return rows;
+}
+
+/**
+ * ONNX text embedder backed by transformers.js (onnxruntime-node underneath).
+ * Runs fully offline with mean pooling + L2 normalization for BERT-style
+ * models like bge-small-zh. `engineFactory` is injectable for tests.
+ */
+export class LocalEmbedder {
+  constructor(opts = {}) {
+    this.model = opts.model || "Xenova/bge-small-zh-v1.5";
+    this._dimension = opts.dimension || 512;
+    this.device = opts.device || "cpu";
+    this.batchSize = opts.batchSize || 8;
+    this.cacheDir = String(opts.cacheDir ?? "").trim();
+    this.useDtype = opts.useDtype || "q8";
+    this.logger = opts.logger ?? null;
+    // Test hook: replace the pipeline factory without touching modules.
+    this.engineFactory = opts.engineFactory || defaultPipelineLoader;
+    this.extractor = null;
+  }
+
+  /** Load the model; throws when it cannot be loaded. */
+  async init() {
+    const options = {
+      dtype: this.useDtype,
+      device: this.device
+    };
+    if (this.cacheDir) options.cache_dir = this.cacheDir;
+    this.extractor = await this.engineFactory("feature-extraction", this.model, options);
+    this.logger?.info?.(
+      `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
+    );
+    return this;
+  }
+
+  /** Embed many texts with mean pooling; chunks at batchSize. */
+  async embed(texts) {
+    if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
+    if (!this.extractor) throw new Error("LocalEmbedder not initialized");
+    const out = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const chunk = texts.slice(i, i + this.batchSize);
+      const tensor = await this.extractor(chunk, { pooling: "mean", normalize: true });
+      out.push(...tensorToRows(tensor));
+    }
+    return out;
+  }
+
+  async embedSingle(text) {
+    const rows = await this.embed([String(text)]);
+    return rows[0];
+  }
+
+  get dimension() {
+    return this._dimension;
+  }
+
+  get modelHash() {
+    return modelHash(this.model);
+  }
+
+  dispose() {
+    try {
+      this.extractor?.dispose?.();
+    } catch {
+      // best-effort: some engines free resources on GC
+    }
+    this.extractor = null;
+  }
+}
+
+/** Chunk texts into batches of at most `size`. */
+function chunk(texts, size) {
+  const out = [];
+  for (let i = 0; i < texts.length; i += size) out.push(texts.slice(i, i + size));
+  return out;
+}
+
+/**
+ * Ollama embedder over its native HTTP API. `dimension` is inferred from the
+ * first response. init() verifies reachability and that the model exists.
+ */
+export class OllamaEmbedder {
+  constructor(opts = {}) {
+    this.baseUrl = String(opts.baseUrl ?? "http://localhost:11434").trim().replace(/\/+$/, "");
+    this.model = String(opts.model ?? "nomic-embed-text").trim();
+    this.logger = opts.logger ?? null;
+    this._dimension = null;
+  }
+
+  async _post(body) {
+    return fetch(`${this.baseUrl}/api/embeddings`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
+    });
+  }
+
+  /** Probe the server with a tiny prompt; throws when unreachable/missing. */
+  async init() {
+    const res = await this._post({ model: this.model, prompt: "ping" });
+    if (!res.ok) throw new Error(`Ollama ${this.model} unavailable: HTTP ${res.status}`);
+    const body = await res.json();
+    if (!Array.isArray(body?.embedding)) throw new Error(`Ollama ${this.model} returned no embedding`);
+    this._dimension = body.embedding.length;
+    this.logger?.info?.(
+      `[dsh-mneme] ollama embedder ready: ${this.model} (dim=${this._dimension})`
+    );
+    return this;
+  }
+
+  async embed(texts) {
+    if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
+    const out = [];
+    for (const text of texts) out.push(await this.embedSingle(text));
+    return out;
+  }
+
+  async embedSingle(text) {
+    const res = await this._post({ model: this.model, prompt: String(text).slice(0, 8000) });
+    if (!res.ok) throw new Error(`Ollama embed failed: HTTP ${res.status}`);
+    const body = await res.json();
+    const vec = body?.embedding;
+    if (!Array.isArray(vec) || !vec.length) throw new Error("Ollama returned no embedding");
+    if (this._dimension == null) this._dimension = vec.length;
+    return Array.from(vec);
+  }
+
+  get dimension() {
+    return this._dimension ?? 0;
+  }
+
+  get modelHash() {
+    return modelHash(this.model);
+  }
+
+  dispose() {
+    this._dimension = null;
+  }
+}
+
+/**
+ * OpenAI-compatible embedder (OpenAI, SiliconFlow, Zhipu, local proxies).
+ * Backward-compatible behavior lifted from embedding.js, but batchable and
+ * throwing on failure instead of returning null.
+ */
+export class OpenAIEmbedder {
+  constructor(opts = {}) {
+    this.baseUrl = String(opts.baseUrl ?? "").trim().replace(/\/+$/, "");
+    this.apiKey = String(opts.apiKey ?? "").trim();
+    this.model = String(opts.model ?? "").trim();
+    this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    this.logger = opts.logger ?? null;
+    this._dimension = null;
+    // Accept both "https://host/v1" and a full path ending in /embeddings.
+    this._url = /\/embeddings$/i.test(this.baseUrl)
+      ? this.baseUrl
+      : this.baseUrl ? `${this.baseUrl}/embeddings` : "";
+  }
+
+  async init() {
+    if (!this._url || !this.apiKey || !this.model) {
+      throw new Error("OpenAI embedder requires baseUrl, apiKey and model");
+    }
+    this.logger?.info?.(`[dsh-mneme] openai embedder ready: ${this.model}`);
+    return this;
+  }
+
+  async embed(texts) {
+    if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
+    const out = [];
+    for (const batch of chunk(texts, 32)) {
+      const res = await fetch(this._url, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({ model: this.model, input: batch.map((t) => String(t).slice(0, 8000)) }),
+        signal: AbortSignal.timeout(this.timeoutMs)
+      });
+      if (!res.ok) throw new Error(`Embedding API failed: HTTP ${res.status}`);
+      const body = await res.json();
+      const list = body?.data;
+      if (!Array.isArray(list) || list.length !== batch.length) {
+        throw new Error("Embedding API returned unexpected payload");
+      }
+      for (const item of list) {
+        const vec = item?.embedding;
+        if (!Array.isArray(vec) || !vec.length) throw new Error("Embedding API returned empty vector");
+        if (this._dimension == null) this._dimension = vec.length;
+        out.push(Array.from(vec));
+      }
+    }
+    return out;
+  }
+
+  async embedSingle(text) {
+    const rows = await this.embed([String(text)]);
+    return rows[0];
+  }
+
+  get dimension() {
+    return this._dimension ?? 0;
+  }
+
+  get modelHash() {
+    return modelHash(this.model);
+  }
+
+  dispose() {
+    this._dimension = null;
+  }
+}
+
+/** Pick a backend instance by provider name. Throws on unknown providers. */
+export function createEmbedderByProvider(provider, opts) {
+  switch (String(provider ?? "").toLowerCase()) {
+    case "local":
+      return new LocalEmbedder(opts);
+    case "ollama":
+      return new OllamaEmbedder(opts);
+    case "openai":
+      return new OpenAIEmbedder(opts);
+    default:
+      throw new Error(`Unknown embedding provider: ${provider}`);
+  }
+}

package/src/mirror.js

@@ -1,131 +1,131 @@
-import { mkdirSync, readFileSync, writeFileSync, existsSync, rmSync } from "node:fs";
-import { join } from "node:path";
-
-export const TYPE_FILE = {
-  preference: "preferences.md",
-  project: "projects.md",
-  decision: "decisions.md",
-  history: "history.md",
-  summary: "summary.md"
-};
-
-const ESCAPE = /([\\`*_[\]{}()#+.!|>~-])/g;
-const UNESCAPE = new RegExp("\\\\" + ESCAPE.source, "g");
-
-function esc(text) {
-  return String(text).replace(ESCAPE, "\\$1");
-}
-
-function unescape(text) {
-  return String(text).replace(UNESCAPE, "$1");
-}
-
-function renderMemory(m) {
-  const lines = [];
-  lines.push(`## ${esc(m.title)}`);
-  lines.push("");
-  lines.push(`- **ID**: \`${m.id}\``);
-  lines.push(`- **类型**: ${m.type}`);
-  lines.push(`- **重要性**: ${m.importance}`);
-  lines.push(`- **标签**: ${m.tags.map((t) => `\`${esc(t)}\``).join(" ")}`);
-  lines.push(`- **更新时间**: ${m.updated_at}`);
-  if (m.source) lines.push(`- **来源**: ${esc(m.source)}`);
-  lines.push("");
-  lines.push(m.content);
-  lines.push("");
-  lines.push("---");
-  lines.push("");
-  return lines.join("\n");
-}
-
-export function createMirror(dir) {
-  mkdirSync(dir, { recursive: true });
-
-  function filePath(type) {
-    const name = TYPE_FILE[type];
-    return name ? join(dir, name) : undefined;
-  }
-
-  /**
-   * Parse a mirror file back into {id, title, content} entries for human edits.
-   * Entries are anchored on "- **ID**: `...`" lines that are followed by the
-   * "- **类型**:" metadata line (structural entry head): each entry's block
-   * spans from its ID line up to the next ID line (or end of file). The block
-   * head (the ID line plus the generated metadata run) and the trailing
-   * structural "---" separator are stripped; everything in between is the entry
-   * body, so user content containing "---", metadata-like lines, or even a
-   * machine-format "- **ID**: `x`" line is preserved. The title is the "## "
-   * heading preceding the ID line.
-   */
-  function readHumanEdits(type = undefined) {
-    const types = type ? [type] : Object.keys(TYPE_FILE);
-    const edits = [];
-    for (const t of types) {
-      const file = filePath(t);
-      if (!file || !existsSync(file)) continue;
-      const text = readFileSync(file, "utf8").replace(/\r\n/g, "\n");
-      // Anchor on the ID line only when it is a structural entry head: the
-      // machine-rendered ID line is always followed by the "- **类型**:" line.
-      // A body line like "- **ID**: `x`" is not, so it never splits the block
-      // or produces a ghost entry.
-      const anchors = [...text.matchAll(/^- \*\*ID\*\*: `([^`]+)`\n- \*\*类型\*\*:/gm)];
-      let prevEnd = 0;
-      for (let i = 0; i < anchors.length; i++) {
-        const anchor = anchors[i];
-        const blockStart = anchor.index;
-        const blockEnd = i + 1 < anchors.length ? anchors[i + 1].index : text.length;
-
-        // Title: last "## " heading before this ID line (file header region /
-        // previous block tail). Body headings of earlier entries come before
-        // the structural "---" + "## " of this entry, so the last match wins.
-        const titleMatches = [...text.slice(prevEnd, blockStart).matchAll(/^## (.+)$/gm)];
-        const titleMatch = titleMatches[titleMatches.length - 1];
-
-        // Body: the ID line and the generated metadata run are structural head;
-        // everything after them up to the trailing "---" separator is the body.
-        let body = text
-          .slice(blockStart, blockEnd)
-          .replace(/^- \*\*ID\*\*: `[^`]+`\n?/, "")
-          .replace(/^(- \*\*(类型|重要性|标签|更新时间|来源)\*\*:.*\n?)+/, "");
-        const separators = [...body.matchAll(/^---\s*$/gm)];
-        const lastSep = separators[separators.length - 1];
-        if (lastSep) body = body.slice(0, lastSep.index);
-        body = body.trim();
-
-        edits.push({
-          id: anchor[1],
-          title: titleMatch ? unescape(titleMatch[1]).trim() : undefined,
-          content: body
-        });
-
-        const lineEnd = text.indexOf("\n", blockStart);
-        prevEnd = lineEnd === -1 ? text.length : lineEnd + 1;
-      }
-    }
-    return edits;
-  }
-
-  function sync(memories) {
-    const byType = {};
-    for (const m of memories) {
-      (byType[m.type] ??= []).push(m);
-    }
-    for (const type of Object.keys(TYPE_FILE)) {
-      const file = filePath(type);
-      const items = (byType[type] ?? [])
-        .slice()
-        .sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1));
-      if (items.length === 0) {
-        // no memories of this type: drop any stale mirror file so deleted
-        // memories do not "resurrect" via readHumanEdits
-        rmSync(file, { force: true });
-        continue;
-      }
-      const header = `# ${TYPE_FILE[type]} — dsh-mneme 镜像\n\n<!-- 手工编辑此文件会被合并回记忆库（人工优先）。 -->\n\n`;
-      const body = items.map(renderMemory).join("\n");
-      writeFileSync(file, header + body, "utf8");
-    }
-  }
-
-  return { filePath, sync, readHumanEdits };
-}
+import { mkdirSync, readFileSync, writeFileSync, existsSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+export const TYPE_FILE = {
+  preference: "preferences.md",
+  project: "projects.md",
+  decision: "decisions.md",
+  history: "history.md",
+  summary: "summary.md"
+};
+
+const ESCAPE = /([\\`*_[\]{}()#+.!|>~-])/g;
+const UNESCAPE = new RegExp("\\\\" + ESCAPE.source, "g");
+
+function esc(text) {
+  return String(text).replace(ESCAPE, "\\$1");
+}
+
+function unescape(text) {
+  return String(text).replace(UNESCAPE, "$1");
+}
+
+function renderMemory(m) {
+  const lines = [];
+  lines.push(`## ${esc(m.title)}`);
+  lines.push("");
+  lines.push(`- **ID**: \`${m.id}\``);
+  lines.push(`- **类型**: ${m.type}`);
+  lines.push(`- **重要性**: ${m.importance}`);
+  lines.push(`- **标签**: ${m.tags.map((t) => `\`${esc(t)}\``).join(" ")}`);
+  lines.push(`- **更新时间**: ${m.updated_at}`);
+  if (m.source) lines.push(`- **来源**: ${esc(m.source)}`);
+  lines.push("");
+  lines.push(m.content);
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+  return lines.join("\n");
+}
+
+export function createMirror(dir) {
+  mkdirSync(dir, { recursive: true });
+
+  function filePath(type) {
+    const name = TYPE_FILE[type];
+    return name ? join(dir, name) : undefined;
+  }
+
+  /**
+   * Parse a mirror file back into {id, title, content} entries for human edits.
+   * Entries are anchored on "- **ID**: `...`" lines that are followed by the
+   * "- **类型**:" metadata line (structural entry head): each entry's block
+   * spans from its ID line up to the next ID line (or end of file). The block
+   * head (the ID line plus the generated metadata run) and the trailing
+   * structural "---" separator are stripped; everything in between is the entry
+   * body, so user content containing "---", metadata-like lines, or even a
+   * machine-format "- **ID**: `x`" line is preserved. The title is the "## "
+   * heading preceding the ID line.
+   */
+  function readHumanEdits(type = undefined) {
+    const types = type ? [type] : Object.keys(TYPE_FILE);
+    const edits = [];
+    for (const t of types) {
+      const file = filePath(t);
+      if (!file || !existsSync(file)) continue;
+      const text = readFileSync(file, "utf8").replace(/\r\n/g, "\n");
+      // Anchor on the ID line only when it is a structural entry head: the
+      // machine-rendered ID line is always followed by the "- **类型**:" line.
+      // A body line like "- **ID**: `x`" is not, so it never splits the block
+      // or produces a ghost entry.
+      const anchors = [...text.matchAll(/^- \*\*ID\*\*: `([^`]+)`\n- \*\*类型\*\*:/gm)];
+      let prevEnd = 0;
+      for (let i = 0; i < anchors.length; i++) {
+        const anchor = anchors[i];
+        const blockStart = anchor.index;
+        const blockEnd = i + 1 < anchors.length ? anchors[i + 1].index : text.length;
+
+        // Title: last "## " heading before this ID line (file header region /
+        // previous block tail). Body headings of earlier entries come before
+        // the structural "---" + "## " of this entry, so the last match wins.
+        const titleMatches = [...text.slice(prevEnd, blockStart).matchAll(/^## (.+)$/gm)];
+        const titleMatch = titleMatches[titleMatches.length - 1];
+
+        // Body: the ID line and the generated metadata run are structural head;
+        // everything after them up to the trailing "---" separator is the body.
+        let body = text
+          .slice(blockStart, blockEnd)
+          .replace(/^- \*\*ID\*\*: `[^`]+`\n?/, "")
+          .replace(/^(- \*\*(类型|重要性|标签|更新时间|来源)\*\*:.*\n?)+/, "");
+        const separators = [...body.matchAll(/^---\s*$/gm)];
+        const lastSep = separators[separators.length - 1];
+        if (lastSep) body = body.slice(0, lastSep.index);
+        body = body.trim();
+
+        edits.push({
+          id: anchor[1],
+          title: titleMatch ? unescape(titleMatch[1]).trim() : undefined,
+          content: body
+        });
+
+        const lineEnd = text.indexOf("\n", blockStart);
+        prevEnd = lineEnd === -1 ? text.length : lineEnd + 1;
+      }
+    }
+    return edits;
+  }
+
+  function sync(memories) {
+    const byType = {};
+    for (const m of memories) {
+      (byType[m.type] ??= []).push(m);
+    }
+    for (const type of Object.keys(TYPE_FILE)) {
+      const file = filePath(type);
+      const items = (byType[type] ?? [])
+        .slice()
+        .sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1));
+      if (items.length === 0) {
+        // no memories of this type: drop any stale mirror file so deleted
+        // memories do not "resurrect" via readHumanEdits
+        rmSync(file, { force: true });
+        continue;
+      }
+      const header = `# ${TYPE_FILE[type]} — dsh-mneme 镜像\n\n<!-- 手工编辑此文件会被合并回记忆库（人工优先）。 -->\n\n`;
+      const body = items.map(renderMemory).join("\n");
+      writeFileSync(file, header + body, "utf8");
+    }
+  }
+
+  return { filePath, sync, readHumanEdits };
+}