@modusensus/dsh-mneme 0.1.6 → 0.2.0

package/lib/commands.js

@@ -1,64 +1,64 @@
-// Custom slash-command manager: keeps the DSH command registry in sync with
-// user-defined commands persisted in SQLite. Commands are registered on boot
-// and (re)registered on add/remove through the API.
-//
-// Each custom command's handler returns the user-authored instruction as a
-// success result; the DSH UI surfaces it as a model-directed instruction.
-export function createCommandManager({ ctx, settings, logger }) {
-  const registered = new Map(); // name -> disposer
-
-  function registerOne(command) {
-    if (registered.has(command.name)) return;
-    let dispose;
-    try {
-      dispose = ctx.commands.register({
-        name: command.name,
-        description: command.description || `自定义指令 ${command.name}`,
-        handler: () => ({ kind: "success", text: command.instruction })
-      });
-    } catch (error) {
-      logger?.warn?.(`dsh-mneme: failed to register command /${command.name}: ${String(error)}`);
-      return;
-    }
-    registered.set(command.name, dispose);
-  }
-
-  function unregisterOne(name) {
-    const dispose = registered.get(name);
-    if (dispose) {
-      try {
-        dispose();
-      } catch {
-        /* ignore double-dispose */
-      }
-      registered.delete(name);
-    }
-  }
-
-  /** Register every stored command (boot-time sync). */
-  function sync() {
-    for (const command of settings.listCommands()) registerOne(command);
-  }
-
-  /** Add (or replace) a command and register it live. */
-  function add({ name, description, instruction }) {
-    const command = settings.addCommand({ name, description, instruction });
-    registerOne(command);
-    return command;
-  }
-
-  /** Remove a command by id and unregister it live. */
-  function remove(id) {
-    const existing = settings.listCommands().find((c) => c.id === id);
-    if (!existing) return false;
-    if (!settings.removeCommand(id)) return false;
-    unregisterOne(existing.name);
-    return true;
-  }
-
-  function dispose() {
-    for (const name of [...registered.keys()]) unregisterOne(name);
-  }
-
-  return { sync, add, remove, list: () => settings.listCommands(), dispose };
-}
+// Custom slash-command manager: keeps the DSH command registry in sync with
+// user-defined commands persisted in SQLite. Commands are registered on boot
+// and (re)registered on add/remove through the API.
+//
+// Each custom command's handler returns the user-authored instruction as a
+// success result; the DSH UI surfaces it as a model-directed instruction.
+export function createCommandManager({ ctx, settings, logger }) {
+  const registered = new Map(); // name -> disposer
+
+  function registerOne(command) {
+    if (registered.has(command.name)) return;
+    let dispose;
+    try {
+      dispose = ctx.commands.register({
+        name: command.name,
+        description: command.description || `自定义指令 ${command.name}`,
+        handler: () => ({ kind: "success", text: command.instruction })
+      });
+    } catch (error) {
+      logger?.warn?.(`dsh-mneme: failed to register command /${command.name}: ${String(error)}`);
+      return;
+    }
+    registered.set(command.name, dispose);
+  }
+
+  function unregisterOne(name) {
+    const dispose = registered.get(name);
+    if (dispose) {
+      try {
+        dispose();
+      } catch {
+        /* ignore double-dispose */
+      }
+      registered.delete(name);
+    }
+  }
+
+  /** Register every stored command (boot-time sync). */
+  function sync() {
+    for (const command of settings.listCommands()) registerOne(command);
+  }
+
+  /** Add (or replace) a command and register it live. */
+  function add({ name, description, instruction }) {
+    const command = settings.addCommand({ name, description, instruction });
+    registerOne(command);
+    return command;
+  }
+
+  /** Remove a command by id and unregister it live. */
+  function remove(id) {
+    const existing = settings.listCommands().find((c) => c.id === id);
+    if (!existing) return false;
+    if (!settings.removeCommand(id)) return false;
+    unregisterOne(existing.name);
+    return true;
+  }
+
+  function dispose() {
+    for (const name of [...registered.keys()]) unregisterOne(name);
+  }
+
+  return { sync, add, remove, list: () => settings.listCommands(), dispose };
+}

package/lib/config.js

@@ -1,15 +1,49 @@
-import z from "@deepseek-ai/schemastery";
-
-export const Config = z.object({
-  memoryDir: z.string().default("~/.dsh/memory"),
-  autoInject: z.boolean().default(true),
-  autoSummarize: z.boolean().default(true),
-  maxInjectedItems: z.natural().min(1).max(20).default(5),
-  importanceThreshold: z.natural().min(1).max(5).default(3),
-  autoDream: z.boolean().default(true),
-  dreamThresholdCount: z.natural().min(1).max(1000).default(10),
-  dreamThresholdChars: z.natural().min(100).max(100000).default(5000),
-  dreamDelayMs: z.natural().min(0).max(60000).default(2000),
-  dreamProvider: z.string(),
-  dreamModel: z.string()
-});
+import z from "@deepseek-ai/schemastery";
+
+export const Config = z.object({
+  memoryDir: z.string().default("~/.dsh/memory"),
+  autoInject: z.boolean().default(true),
+  autoSummarize: z.boolean().default(true),
+  maxInjectedItems: z.natural().min(1).max(20).default(5),
+  importanceThreshold: z.natural().min(1).max(5).default(3),
+  autoDream: z.boolean().default(true),
+  dreamThresholdCount: z.natural().min(1).max(1000).default(10),
+  dreamThresholdChars: z.natural().min(100).max(100000).default(5000),
+  dreamDelayMs: z.natural().min(0).max(60000).default(2000),
+  dreamProvider: z.string(),
+  dreamModel: z.string(),
+  dreamMaxTokens: z.natural().min(256).max(32768).default(4096),
+
+  // --- semantic: local embedding provider (v0.2) --------------------------
+  // "openai" keeps the legacy external-API path (settings vector config);
+  // "local" runs an ONNX model in-process; "ollama" calls a local Ollama.
+  embedProvider: z.union([z.const("openai"), z.const("local"), z.const("ollama")]).default("openai"),
+
+  // Local ONNX embedder (transformers.js / onnxruntime).
+  localEmbedModel: z.string().default("Xenova/bge-small-zh-v1.5"),
+  localEmbedDimension: z.natural().default(512),
+  localEmbedDevice: z.union([z.const("cpu"), z.const("gpu")]).default("cpu"),
+  localEmbedBatchSize: z.natural().min(1).max(64).default(8),
+
+  // Ollama embedder.
+  ollamaBaseUrl: z.string().default("http://localhost:11434"),
+  ollamaModel: z.string().default("nomic-embed-text"),
+
+  // Model download/cache.
+  embedModelCacheDir: z.string().default(""),
+  embedModelMirror: z.string().default("https://hf-mirror.com"),
+
+  // Vector search tuning.
+  vectorSearchTopK: z.natural().min(1).max(100).default(20),
+  vectorSearchThreshold: z.number().min(0).max(1).default(0.65),
+  hybridSearchVectorWeight: z.number().min(0).max(1).default(0.6),
+  hybridSearchKeywordWeight: z.number().min(0).max(1).default(0.4),
+
+  // --- semantic: rerank layer (v0.2) --------------------------------------
+  rerankEnabled: z.boolean().default(true),
+  rerankProvider: z.union([z.const("local"), z.const("none")]).default("local"),
+  rerankModel: z.string().default("Xenova/bge-reranker-base"),
+  rerankBatchSize: z.natural().min(1).max(64).default(8),
+  rerankMaxCandidates: z.natural().min(5).max(100).default(30),
+  rerankScoreThreshold: z.number().min(0).max(1).default(0.1)
+});

package/lib/dream/clustering.js

@@ -0,0 +1,118 @@
+// Cosine similarity of two equal-length numeric vectors. Returns 0 for empty
+// or mismatched-length inputs, and for zero-norm vectors (numerically stable).
+export function cosineSimilarity(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+// K-Means clustering of numeric vectors. Returns one index array per cluster.
+// Uses k-means++ seeding and repairs empty clusters mid-iteration.
+export function kMeans(vectors, k, opts = {}) {
+  const n = vectors.length;
+  if (n === 0) return [];
+  const { maxIter = 100, tol = 1e-4 } = opts ?? {};
+  if (k <= 0 || k >= n) return [vectors.map((_, i) => i)];
+
+  const dim = vectors[0].length;
+  const randomIndex = () => Math.floor(Math.random() * n);
+  const squaredDist = (v, c) => {
+    let s = 0;
+    for (let d = 0; d < dim; d++) s += (v[d] - c[d]) * (v[d] - c[d]);
+    return s;
+  };
+
+  // k-means++ seeding: first centroid uniform-random, later centroids sampled
+  // with probability proportional to squared distance from nearest centroid.
+  const centroids = [[...vectors[randomIndex()]]];
+  while (centroids.length < k) {
+    const dist = new Array(n);
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      let best = Infinity;
+      for (const c of centroids) best = Math.min(best, squaredDist(vectors[i], c));
+      dist[i] = best;
+      sum += best;
+    }
+    if (sum === 0) {
+      // All points coincide with a centroid; fall back to uniform random.
+      centroids.push([...vectors[randomIndex()]]);
+    } else {
+      let r = Math.random() * sum;
+      let pick = 0;
+      for (let i = 0; i < n; i++) {
+        r -= dist[i];
+        if (r <= 0) { pick = i; break; }
+      }
+      centroids.push([...vectors[pick]]);
+    }
+  }
+
+  const assignment = new Array(n).fill(0);
+  for (let iter = 0; iter < maxIter; iter++) {
+    // Assign each point to its nearest centroid.
+    let changed = false;
+    for (let i = 0; i < n; i++) {
+      let best = 0, bestD = Infinity;
+      for (let j = 0; j < k; j++) {
+        const d = squaredDist(vectors[i], centroids[j]);
+        if (d < bestD) { bestD = d; best = j; }
+      }
+      if (assignment[i] !== best) { assignment[i] = best; changed = true; }
+    }
+    if (!changed) break; // stable assignment => converged.
+
+    // Recompute centroids as cluster means; reseed empty clusters randomly.
+    const sums = Array.from({ length: k }, () => new Array(dim).fill(0));
+    const counts = new Array(k).fill(0);
+    for (let i = 0; i < n; i++) {
+      counts[assignment[i]]++;
+      for (let d = 0; d < dim; d++) sums[assignment[i]][d] += vectors[i][d];
+    }
+    let maxMove = 0;
+    for (let j = 0; j < k; j++) {
+      if (counts[j] === 0) {
+        centroids[j] = [...vectors[randomIndex()]];
+        maxMove = Infinity; // never call this converged this round.
+      } else {
+        for (let d = 0; d < dim; d++) {
+          const mean = sums[j][d] / counts[j];
+          const move = mean - centroids[j][d];
+          if (move * move > maxMove) maxMove = move * move;
+          centroids[j][d] = mean;
+        }
+      }
+    }
+    if (Math.sqrt(maxMove) < tol) break; // centroids moved less than tol.
+  }
+
+  // Build clusters from final assignments.
+  const clusters = Array.from({ length: k }, () => []);
+  for (let i = 0; i < n; i++) clusters[assignment[i]].push(i);
+  return clusters;
+}
+
+// Group memories by vector clustering, preserving original object references.
+export function clusterMemories(memories, vectors, k) {
+  return kMeans(vectors, k).map((indices) => indices.map((i) => memories[i]));
+}
+
+// Find memory pairs that are highly similar yet could contradict each other.
+// Only same-type pairs count; each pair reported once, lower index first.
+export function findPotentialConflicts(memories, vectors, threshold = 0.85) {
+  const pairs = [];
+  for (let i = 0; i < memories.length; i++) {
+    for (let j = i + 1; j < memories.length; j++) {
+      if (memories[i].type !== memories[j].type) continue;
+      const sim = cosineSimilarity(vectors[i], vectors[j]);
+      if (sim > threshold) pairs.push({ a: memories[i], b: memories[j], similarity: sim });
+    }
+  }
+  return pairs;
+}

package/lib/dream/decisions.js

@@ -1,121 +1,128 @@
-const ACTIONS = new Set(["keep", "merge", "archive", "conflict"]);
-
-/**
- * Validate a dream decision list against a snapshot of eligible memories.
- * @param decisions - LLM-produced decision list.
- * @param snapshot - Map<id, memory> of eligible (non-archived, non-summary) entries.
- * @returns {{ok: boolean, errors: string[]}}
- */
-export function validateDecisions(decisions, snapshot) {
-  const errors = [];
-  if (!Array.isArray(decisions) || decisions.length === 0) {
-    return { ok: false, errors: ["decision list must be a non-empty array"] };
-  }
-  const claimed = new Set();
-  for (const [index, d] of decisions.entries()) {
-    const at = `decision[${index}]`;
-    if (!d || typeof d !== "object" || !ACTIONS.has(d.action)) {
-      errors.push(`${at}: invalid action ${JSON.stringify(d?.action)}`);
-      continue;
-    }
-    const ids = d.action === "conflict" ? [d.winner, d.loser] : (d.ids ?? []);
-    if (d.action === "conflict") {
-      if (!d.winner || !d.loser || d.winner === d.loser) {
-        errors.push(`${at}: conflict needs distinct winner and loser`);
-        continue;
-      }
-    } else if (!Array.isArray(d.ids) || d.ids.length === 0) {
-      errors.push(`${at}: ${d.action} needs non-empty ids`);
-      continue;
-    }
-    for (const id of ids) {
-      const mem = snapshot.get(id);
-      if (!mem) {
-        errors.push(`${at}: unknown id ${JSON.stringify(id)}`);
-      } else if (mem.archived || mem.type === "summary") {
-        errors.push(`${at}: id ${JSON.stringify(id)} is archived or summary (not eligible)`);
-      }
-      if (claimed.has(id)) {
-        errors.push(`${at}: id ${JSON.stringify(id)} claimed by multiple decisions`);
-      }
-      claimed.add(id);
-    }
-    if (d.action === "merge") {
-      if (!d.keepSource || !d.ids.includes(d.keepSource)) {
-        errors.push(`${at}: merge keepSource must be one of ids`);
-      }
-      if (typeof d.title !== "string" || !d.title.trim() || typeof d.content !== "string" || !d.content.trim()) {
-        errors.push(`${at}: merge needs non-empty title and content`);
-      }
-      if (d.importance !== undefined && (!Number.isInteger(d.importance) || d.importance < 1 || d.importance > 5)) {
-        errors.push(`${at}: merge importance must be an integer 1-5 when provided`);
-      }
-      // Merging across types would blur preference/project/decision boundaries
-      // in the injected context; the snapshot carries each entry's type.
-      const mergeTypes = new Set(d.ids.map((id) => snapshot.get(id)?.type));
-      if (mergeTypes.size > 1) {
-        errors.push(`${at}: merge ids span multiple types (${[...mergeTypes].join(", ")})`);
-      }
-    }
-  }
-  // Every snapshot id must appear in at least one decision
-  for (const id of snapshot.keys()) {
-    if (!claimed.has(id)) errors.push(`memory ${JSON.stringify(id)} missing from decisions`);
-  }
-  return { ok: errors.length === 0, errors };
-}
-
-/**
- * Apply a validated decision list to the service. Caller must validate first.
- *
- * Note: merge is intentionally non-atomic — the keeper is updated before the
- * other sources are archived, so a failure between the two never loses content.
- *
- * @param decisions - validated decision list.
- * @param service - memory service (saveWithDedupe/getById/update/setArchived).
- * @param logger - optional logger ({ warn }); per-decision failures are logged.
- * @returns number of applied decisions (archive counts each archived memory as one).
- */
-export function applyDecisions(decisions, service, logger = null) {
-  let applied = 0;
-  for (const [i, d] of decisions.entries()) {
-    try {
-      if (d.action === "keep") continue;
-      if (d.action === "archive") {
-        for (const id of d.ids) {
-          const mem = service.getById(id);
-          if (mem && !mem.archived) { service.setArchived(id, true); applied++; }
-        }
-      } else if (d.action === "merge") {
-        const keeper = service.getById(d.keepSource);
-        if (!keeper || keeper.archived) continue;
-        service.update(d.keepSource, {
-          title: d.title,
-          content: d.content,
-          importance: d.importance ?? Math.max(keeper.importance, ...d.ids.map((id) => service.getById(id)?.importance ?? 1))
-        });
-        for (const id of d.ids) {
-          if (id !== d.keepSource) {
-            const mem = service.getById(id);
-            if (mem && !mem.archived) { service.setArchived(id, true); }
-          }
-        }
-        applied++;
-      } else if (d.action === "conflict") {
-        const winner = service.getById(d.winner);
-        const loser = service.getById(d.loser);
-        if (!winner || !loser) continue;
-        service.update(d.winner, {
-          content: `${winner.content}\n\n（已否决旧信息：${[...loser.content].slice(0, 100).join("")}）`
-        });
-        service.setArchived(d.loser, true);
-        applied++;
-      }
-    } catch (error) {
-      // Skip individual bad decision; never corrupt the store. The optional
-      // logger makes the failure visible instead of failing silently.
-      logger?.warn?.(`dsh-mneme dream: failed to apply ${d.action} at index ${i}: ${error.message}`);
-    }
-  }
-  return applied;
-}
+const ACTIONS = new Set(["keep", "merge", "archive", "conflict"]);
+
+/**
+ * Validate a dream decision list against a snapshot of eligible memories.
+ * @param decisions - LLM-produced decision list.
+ * @param snapshot - Map<id, memory> of eligible (non-archived, non-summary) entries.
+ * @returns {{ok: boolean, errors: string[]}}
+ */
+export function validateDecisions(decisions, snapshot) {
+  const errors = [];
+  if (!Array.isArray(decisions) || decisions.length === 0) {
+    return { ok: false, errors: ["decision list must be a non-empty array"] };
+  }
+  const claimed = new Set();
+  for (const [index, d] of decisions.entries()) {
+    const at = `decision[${index}]`;
+    if (!d || typeof d !== "object" || !ACTIONS.has(d.action)) {
+      errors.push(`${at}: invalid action ${JSON.stringify(d?.action)}`);
+      continue;
+    }
+    const ids = d.action === "conflict" ? [d.winner, d.loser] : (d.ids ?? []);
+    if (d.action === "conflict") {
+      if (!d.winner || !d.loser || d.winner === d.loser) {
+        errors.push(`${at}: conflict needs distinct winner and loser`);
+        continue;
+      }
+    } else if (!Array.isArray(d.ids) || d.ids.length === 0) {
+      errors.push(`${at}: ${d.action} needs non-empty ids`);
+      continue;
+    }
+    for (const id of ids) {
+      const mem = snapshot.get(id);
+      if (!mem) {
+        errors.push(`${at}: unknown id ${JSON.stringify(id)}`);
+      } else if (mem.archived || mem.type === "summary") {
+        errors.push(`${at}: id ${JSON.stringify(id)} is archived or summary (not eligible)`);
+      }
+      if (claimed.has(id)) {
+        errors.push(`${at}: id ${JSON.stringify(id)} claimed by multiple decisions`);
+      }
+      claimed.add(id);
+    }
+    if (d.action === "merge") {
+      if (!d.keepSource || !d.ids.includes(d.keepSource)) {
+        errors.push(`${at}: merge keepSource must be one of ids`);
+      }
+      if (typeof d.title !== "string" || !d.title.trim() || typeof d.content !== "string" || !d.content.trim()) {
+        errors.push(`${at}: merge needs non-empty title and content`);
+      }
+      if (d.importance !== undefined && (!Number.isInteger(d.importance) || d.importance < 1 || d.importance > 5)) {
+        errors.push(`${at}: merge importance must be an integer 1-5 when provided`);
+      }
+      // Merging across types would blur preference/project/decision boundaries
+      // in the injected context; the snapshot carries each entry's type.
+      const mergeTypes = new Set(d.ids.map((id) => snapshot.get(id)?.type));
+      if (mergeTypes.size > 1) {
+        errors.push(`${at}: merge ids span multiple types (${[...mergeTypes].join(", ")})`);
+      }
+    }
+  }
+  // Every snapshot id must appear in at least one decision
+  for (const id of snapshot.keys()) {
+    if (!claimed.has(id)) errors.push(`memory ${JSON.stringify(id)} missing from decisions`);
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/**
+ * Apply a validated decision list to the service. Caller must validate first.
+ *
+ * Note: merge is intentionally non-atomic — the keeper is updated before the
+ * other sources are archived, so a failure between the two never loses content.
+ *
+ * @param decisions - validated decision list.
+ * @param service - memory service (saveWithDedupe/getById/update/setArchived).
+ * @param logger - optional logger ({ warn }); per-decision failures are logged.
+ * @returns number of applied decisions (archive counts each archived memory as one).
+ */
+export function applyDecisions(decisions, service, logger = null) {
+  let applied = 0;
+  for (const [i, d] of decisions.entries()) {
+    try {
+      if (d.action === "keep") continue;
+      if (d.action === "archive") {
+        for (const id of d.ids) {
+          const mem = service.getById(id);
+          if (mem && !mem.archived) { service.setArchived(id, true); applied++; }
+        }
+      } else if (d.action === "merge") {
+        const keeper = service.getById(d.keepSource);
+        if (!keeper || keeper.archived) continue;
+        const sources = d.ids.filter((id) => id !== d.keepSource);
+        // Idempotent replay: if every other source is already archived, this
+        // merge already landed — skip so a replayed/concurrent decision never
+        // double-counts or re-applies (guard against duplicate merges).
+        if (sources.every((id) => service.getById(id)?.archived)) continue;
+        service.update(d.keepSource, {
+          title: d.title,
+          content: d.content,
+          importance: d.importance ?? Math.max(keeper.importance, ...d.ids.map((id) => service.getById(id)?.importance ?? 1))
+        });
+        for (const id of sources) {
+          const mem = service.getById(id);
+          if (mem && !mem.archived) { service.setArchived(id, true); }
+        }
+        applied++;
+      } else if (d.action === "conflict") {
+        const winner = service.getById(d.winner);
+        const loser = service.getById(d.loser);
+        if (!winner || !loser) continue;
+        // Idempotent replay: an already-archived loser means the conflict was
+        // already adjudicated — skip so the provenance note is never
+        // re-appended and the loser is not re-archived.
+        if (loser.archived) continue;
+        service.update(d.winner, {
+          content: `${winner.content}\n\n（已否决旧信息：${[...loser.content].slice(0, 100).join("")}）`
+        });
+        service.setArchived(d.loser, true);
+        applied++;
+      }
+    } catch (error) {
+      // Skip individual bad decision; never corrupt the store. The optional
+      // logger makes the failure visible instead of failing silently.
+      logger?.warn?.(`dsh-mneme dream: failed to apply ${d.action} at index ${i}: ${error.message}`);
+    }
+  }
+  return applied;
+}