mumpix 1.0.20 → 1.1.3

package/src/core/semantic-sidecar.js

@@ -0,0 +1,312 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+function vectorPathForDb(dbPath) {
+  return `${dbPath}.vectors.json`;
+}
+
+function vectorLogPathForSnapshot(vectorPath) {
+  return `${vectorPath}l`;
+}
+
+function emptySidecar(embedder) {
+  return {
+    version: 1,
+    embedder_id: embedder.id,
+    dim: embedder.dim,
+    entries: [],
+  };
+}
+
+function normalizeVector(vector, dim) {
+  if (!vector || typeof vector.length !== "number") return null;
+  if (Number.isFinite(dim) && vector.length !== dim) return null;
+  const out = Array.from(vector, Number);
+  if (out.some((value) => !Number.isFinite(value))) return null;
+  return out;
+}
+
+function dot(left, right) {
+  const length = Math.min(left.length, right.length);
+  let score = 0;
+  for (let i = 0; i < length; i += 1) score += left[i] * right[i];
+  return score;
+}
+
+function validateEmbedder(embedder) {
+  if (!embedder || typeof embedder !== "object") {
+    throw new Error(
+      'Mumpix semantic recall requires an embedder. Pass { embedder } or install optional package "mumpix-embed".'
+    );
+  }
+  if (typeof embedder.id !== "string" || !embedder.id.trim()) {
+    throw new Error("Mumpix semantic recall embedder requires a stable string id.");
+  }
+  if (!Number.isInteger(embedder.dim) || embedder.dim <= 0) {
+    throw new Error("Mumpix semantic recall embedder requires a positive integer dim.");
+  }
+  if (typeof embedder.embed !== "function") {
+    throw new Error("Mumpix semantic recall embedder requires embed(texts) => Promise<vector[]>.");
+  }
+}
+
+function loadDefaultEmbedder() {
+  try {
+    const mod = require("mumpix-embed");
+    const candidate =
+      typeof mod.createEmbedder === "function"
+        ? mod.createEmbedder()
+        : mod.default || mod.embedder || mod;
+    return candidate;
+  } catch (err) {
+    const code = err && err.code;
+    if (code === "MODULE_NOT_FOUND") {
+      throw new Error(
+        'Mumpix semantic recall requires optional package "mumpix-embed". Install it with: npm i mumpix-embed'
+      );
+    }
+    throw err;
+  }
+}
+
+class SemanticSidecar {
+  constructor(dbPath, embedder, opts = {}) {
+    validateEmbedder(embedder);
+    this.dbPath = dbPath;
+    this.embedder = embedder;
+    this.vectorPath = opts.vectorPath || vectorPathForDb(dbPath);
+    this.vectorLogPath = opts.vectorLogPath || vectorLogPathForSnapshot(this.vectorPath);
+    this._needsPersist = false;
+    this._bySeq = new Map();
+    this.data = this._load();
+    if (this._needsPersist) this.persist();
+  }
+
+  _load() {
+    if (!fs.existsSync(this.vectorPath)) {
+      this._needsPersist = true;
+      const data = emptySidecar(this.embedder);
+      this._hydrateMap(data.entries);
+      this._loadJournal(data);
+      return data;
+    }
+    try {
+      const loaded = JSON.parse(fs.readFileSync(this.vectorPath, "utf8"));
+      if (
+        !loaded ||
+        loaded.embedder_id !== this.embedder.id ||
+        loaded.dim !== this.embedder.dim ||
+        !Array.isArray(loaded.entries)
+      ) {
+        this._needsPersist = true;
+        this._discardJournal();
+        return emptySidecar(this.embedder);
+      }
+      const data = {
+        ...emptySidecar(this.embedder),
+        entries: loaded.entries.filter(
+          (entry) =>
+            entry &&
+            Number.isInteger(entry.seq ?? entry.id) &&
+            typeof entry.content === "string" &&
+            Array.isArray(entry.vec) &&
+            entry.vec.length === this.embedder.dim
+        ).map((entry) => this._normalizeEntry(entry)),
+      };
+      this._hydrateMap(data.entries);
+      this._loadJournal(data);
+      this._hydrateMap(data.entries);
+      return data;
+    } catch (_) {
+      this._discardJournal();
+      const data = emptySidecar(this.embedder);
+      this._hydrateMap(data.entries);
+      return data;
+    }
+  }
+
+  persist() {
+    fs.mkdirSync(path.dirname(this.vectorPath), { recursive: true });
+    const tmp = `${this.vectorPath}.tmp.${process.pid}`;
+    fs.writeFileSync(tmp, JSON.stringify(this.data, null, 2), "utf8");
+    fs.renameSync(tmp, this.vectorPath);
+  }
+
+  compact() {
+    this.persist();
+    this._discardJournal();
+  }
+
+  syncRecords(records) {
+    const bySeq = new Map(records.map((record) => [record.id, record]));
+    const before = this.data.entries.length;
+    this.data.entries = this.data.entries.filter((entry) => {
+      const record = bySeq.get(entry.seq);
+      return record && record.content === entry.content && record.ts === entry.ts;
+    });
+    this._hydrateMap(this.data.entries);
+    if (this.data.entries.length !== before) this.compact();
+  }
+
+  async reconcileRecords(records) {
+    const bySeq = new Map(records.map((record) => [record.id, record]));
+    const before = this.data.entries.length;
+    this.data.entries = this.data.entries.filter((entry) => {
+      const record = bySeq.get(entry.seq);
+      return record && record.content === entry.content && record.ts === entry.ts;
+    });
+    this._hydrateMap(this.data.entries);
+
+    const missing = records.filter((record) => !this._bySeq.has(record.id));
+    if (!missing.length) {
+      if (this.data.entries.length !== before) this.compact();
+      return { pruned: before - this.data.entries.length, embedded: 0 };
+    }
+
+    const vectors = await this.embedder.embed(missing.map((record) => record.content));
+    if (!Array.isArray(vectors) || vectors.length !== missing.length) {
+      throw new Error("Mumpix semantic recall embedder returned the wrong number of vectors.");
+    }
+
+    for (let i = 0; i < missing.length; i += 1) {
+      const record = missing[i];
+      const vec = normalizeVector(vectors[i], this.embedder.dim);
+      if (!vec) throw new Error("Mumpix semantic recall embedder returned an invalid vector.");
+      this._upsert({
+        seq: record.id,
+        record_id: record.id,
+        content: record.content,
+        ts: record.ts,
+        vec,
+      });
+    }
+
+    this.compact();
+    return { pruned: before - (this.data.entries.length - missing.length), embedded: missing.length };
+  }
+
+  async addRecord(record) {
+    const [vector] = await this.embedder.embed([record.content]);
+    const vec = normalizeVector(vector, this.embedder.dim);
+    if (!vec) throw new Error("Mumpix semantic recall embedder returned an invalid vector.");
+
+    const entry = {
+      seq: record.id,
+      id: record.id,
+      record_id: record.id,
+      content: record.content,
+      ts: record.ts,
+      vec,
+    };
+
+    this._upsert(entry);
+    this._appendJournal({ op: "upsert", entry });
+  }
+
+  clear() {
+    this.data.entries = [];
+    this._bySeq.clear();
+    this.compact();
+  }
+
+  async query(text, k, floor) {
+    if (!this.data.entries.length) return [];
+    const [queryVector] = await this.embedder.embed([text]);
+    const qVec = normalizeVector(queryVector, this.embedder.dim);
+    if (!qVec) throw new Error("Mumpix semantic recall embedder returned an invalid query vector.");
+
+    return this.data.entries
+      .map((entry) => ({ ...entry, _score: dot(qVec, entry.vec) }))
+      .filter((entry) => entry._score >= floor)
+      .sort((left, right) => {
+        if (right._score !== left._score) return right._score - left._score;
+        if (right.ts !== left.ts) return right.ts - left.ts;
+        return left.id - right.id;
+      })
+      .slice(0, k);
+  }
+
+  _normalizeEntry(entry) {
+    const seq = entry.seq ?? entry.id;
+    return {
+      seq,
+      id: seq,
+      record_id: entry.record_id ?? seq,
+      content: entry.content,
+      ts: entry.ts,
+      vec: entry.vec,
+    };
+  }
+
+  _hydrateMap(entries) {
+    this._bySeq.clear();
+    for (const entry of entries) this._bySeq.set(entry.seq, entry);
+  }
+
+  _upsert(entry) {
+    const normalized = this._normalizeEntry(entry);
+    const existing = this._bySeq.get(normalized.seq);
+    if (existing) {
+      Object.assign(existing, normalized);
+      return;
+    }
+    this._bySeq.set(normalized.seq, normalized);
+    this.data.entries.push(normalized);
+  }
+
+  _loadJournal(data) {
+    if (!fs.existsSync(this.vectorLogPath)) return;
+    const bySeq = new Map(data.entries.map((entry) => [entry.seq, entry]));
+    try {
+      const lines = fs.readFileSync(this.vectorLogPath, "utf8").split("\n");
+      for (const line of lines) {
+        if (!line.trim()) continue;
+        const event = JSON.parse(line);
+        if (event.op === "clear") {
+          data.entries = [];
+          bySeq.clear();
+        } else if (event.op === "upsert" && event.entry) {
+          const entry = this._normalizeEntry(event.entry);
+          if (
+            Number.isInteger(entry.seq) &&
+            typeof entry.content === "string" &&
+            Array.isArray(entry.vec) &&
+            entry.vec.length === this.embedder.dim
+          ) {
+            const existing = bySeq.get(entry.seq);
+            if (existing) {
+              Object.assign(existing, entry);
+            } else {
+              bySeq.set(entry.seq, entry);
+              data.entries.push(entry);
+            }
+          }
+        }
+      }
+    } catch (_) {
+      this._discardJournal();
+      data.entries = [...bySeq.values()];
+    }
+  }
+
+  _appendJournal(event) {
+    fs.mkdirSync(path.dirname(this.vectorLogPath), { recursive: true });
+    fs.appendFileSync(this.vectorLogPath, `${JSON.stringify(event)}\n`, "utf8");
+  }
+
+  _discardJournal() {
+    try {
+      fs.unlinkSync(this.vectorLogPath);
+    } catch (_) {}
+  }
+}
+
+module.exports = {
+  SemanticSidecar,
+  loadDefaultEmbedder,
+  validateEmbedder,
+  vectorLogPathForSnapshot,
+  vectorPathForDb,
+};