nexusmem 0.3.0 → 0.3.1

package/dist/cli/index.js

@@ -698,10 +698,13 @@ import * as sqliteVec from "sqlite-vec";
 
 // src/store/fts.ts
 var FTS_SYNTAX = /["'()*:^{}[\]-]/g;
+var LOW_SIGNAL_TOKENS = /* @__PURE__ */ new Set(["id"]);
 function toMatchQuery(input) {
   const tokens = input.replace(FTS_SYNTAX, " ").split(/\s+/).map((t) => t.trim()).filter((t) => t.length > 0);
   if (tokens.length === 0) return null;
-  return tokens.map((t) => `"${t}"*`).join(" OR ");
+  const signal = tokens.filter((t) => !LOW_SIGNAL_TOKENS.has(t.toLowerCase()));
+  const kept = signal.length > 0 ? signal : tokens;
+  return kept.map((t) => `"${t}"*`).join(" OR ");
 }
 
 // src/store/schema.ts
@@ -848,6 +851,20 @@ var MemoryStore = class _MemoryStore {
   markSynced(projectId) {
     this.db.prepare("UPDATE projects SET last_synced_at = ? WHERE id = ?").run(Date.now(), projectId);
   }
+  /**
+   * Every other project id ever recorded in THIS repo's own database.
+   *
+   * A repo's `.nexusmem/memory.db` is never shared with another repo (each
+   * gets its own, gitignored), so any id here besides `currentProjectId` is
+   * evidence of a prior identity for this same repo -- typically its git
+   * remote URL changed since the last sync. See `reconcileProjectId` in
+   * `store/reconcile.ts`.
+   */
+  listOtherProjectIds(currentProjectId) {
+    return this.db.prepare("SELECT id FROM projects WHERE id != ?").all(currentProjectId).map(
+      (r) => r.id
+    );
+  }
   /**
    * Write a batch of nodes in one transaction.
    *
@@ -1245,6 +1262,8 @@ function approxTokens(text) {
 var DEFAULT_SUMMARY_CHARS = 320;
 var NODE_OVERHEAD_TOKENS = 8;
 var CONVERSATION_ANSWER_MARKER = "\n\nA: ";
+var MAX_PER_FAMILY = 2;
+var CHUNKED_KINDS = /* @__PURE__ */ new Set(["conversation_turn", "doc_section"]);
 var HUNK_BOUNDARY = "\n@@ ";
 var STOPWORDS = /* @__PURE__ */ new Set([
   "the",
@@ -1363,7 +1382,14 @@ function packContext(ranked, tokensBudget, opts = {}) {
   const nodes = [];
   let tokensUsed = 0;
   let droppedForBudget = 0;
+  let droppedForDiversity = 0;
+  const familyCounts = /* @__PURE__ */ new Map();
   for (const hit of ranked) {
+    const familyKey = CHUNKED_KINDS.has(hit.kind) ? `${hit.kind}:${hit.ts}` : null;
+    if (familyKey && (familyCounts.get(familyKey) ?? 0) >= MAX_PER_FAMILY) {
+      droppedForDiversity += 1;
+      continue;
+    }
     const summary = summarize(hit, summaryChars, query);
     const tokens = approxTokens(hit.title) + approxTokens(summary) + NODE_OVERHEAD_TOKENS;
     if (tokensUsed + tokens > tokensBudget) {
@@ -1382,8 +1408,9 @@ function packContext(ranked, tokensBudget, opts = {}) {
       ...hit.project ? { project: hit.project } : {}
     });
     tokensUsed += tokens;
+    if (familyKey) familyCounts.set(familyKey, (familyCounts.get(familyKey) ?? 0) + 1);
   }
-  return { nodes, tokensUsed, tokensBudget, consideredNodes: ranked.length, droppedForBudget };
+  return { nodes, tokensUsed, tokensBudget, consideredNodes: ranked.length, droppedForBudget, droppedForDiversity };
 }
 function renderContextBlock(query, result) {
   if (result.nodes.length === 0) return `No remembered context matched "${query}".`;
@@ -2425,6 +2452,7 @@ function sessionFallbackTitle(session) {
 }
 var MAX_TITLE_CHARS5 = 200;
 var GENERIC_TITLE = /^(a |the )?(session |conversation |project |work )?(summary|update|overview|recap|status)\b/i;
+var ROLE_PREAMBLE_TITLE = /^(role\s*[:：]|you\s*(?:'re|are)\s+(acting as|serving as|playing the role of|a\b)|i\s*(?:'m|am)\s+(acting as|a\b)|acting as\b|บทบาท\s*[:：]|ในฐานะ|คุณ(กำลัง)?(ทำหน้าที่เป็น|เป็น|รับบทบาทเป็น)|(ผม|ฉัน)(กำลัง)?(ทำหน้าที่เป็น|เป็น))/i;
 function cleanTitle(line) {
   return line.replace(/^[-*#>\s]+/, "").replace(/\*+/g, "").replace(/\s*:\s*$/, "").trim();
 }
@@ -2441,7 +2469,7 @@ function parseSummary(raw, fallbackTitle) {
   const candidate = cleanTitle(labelled[1]);
   const rest = lines.slice(firstIndex + 1).join("\n").trim();
   return {
-    title: candidate.length > 0 && !GENERIC_TITLE.test(candidate) ? truncate(candidate, MAX_TITLE_CHARS5) : fallback,
+    title: candidate.length > 0 && !GENERIC_TITLE.test(candidate) && !ROLE_PREAMBLE_TITLE.test(candidate) ? truncate(candidate, MAX_TITLE_CHARS5) : fallback,
     // A model that emitted only a title still gets a usable node: the title
     // doubles as the body rather than storing an empty one.
     body: rest.length > 0 ? rest : candidate
@@ -2921,6 +2949,100 @@ async function collectAvailableShellHistory(opts = {}) {
   return results;
 }
 
+// src/store/reconcile.ts
+function recomputeByNaturalKey(db, oldProjectId, newProjectId, kind, source, computeNaturalKey) {
+  const rows = source ? db.prepare("SELECT * FROM nodes WHERE project_id = ? AND kind = ? AND source = ?").all(oldProjectId, kind, source) : db.prepare("SELECT * FROM nodes WHERE project_id = ? AND kind = ?").all(oldProjectId, kind);
+  const nodeExists = db.prepare("SELECT 1 FROM nodes WHERE id = ?");
+  const insertNode = db.prepare(
+    `INSERT INTO nodes (id, kind, project_id, ts, ts_epoch, source, title, body, signal, meta, created_at)
+     VALUES (@id, @kind, @projectId, @ts, @tsEpoch, @source, @title, @body, @signal, @meta, @createdAt)`
+  );
+  const readFiles = db.prepare("SELECT path, previous_path, insertions, deletions, is_binary FROM node_files WHERE node_id = ?");
+  const insertFile = db.prepare(
+    `INSERT INTO node_files (node_id, path, previous_path, insertions, deletions, is_binary)
+     VALUES (@nodeId, @path, @previousPath, @insertions, @deletions, @isBinary)`
+  );
+  const dropEmbedding = db.prepare("DELETE FROM nodes_vec WHERE rowid = (SELECT rowid FROM nodes WHERE id = ?)");
+  const deleteNode = db.prepare("DELETE FROM nodes WHERE id = ?");
+  let migrated = 0;
+  let deduped = 0;
+  let skipped = 0;
+  for (const row of rows) {
+    let meta;
+    try {
+      meta = JSON.parse(row.meta);
+    } catch {
+      skipped += 1;
+      continue;
+    }
+    const naturalKey = computeNaturalKey(row, meta);
+    if (naturalKey === null) {
+      skipped += 1;
+      continue;
+    }
+    const newId = makeNodeId(newProjectId, kind, naturalKey);
+    if (nodeExists.get(newId)) {
+      deduped += 1;
+    } else {
+      insertNode.run({
+        id: newId,
+        kind: row.kind,
+        projectId: newProjectId,
+        ts: row.ts,
+        tsEpoch: row.ts_epoch,
+        source: row.source,
+        title: row.title,
+        body: row.body,
+        signal: row.signal,
+        meta: row.meta,
+        createdAt: row.created_at
+      });
+      for (const file of readFiles.all(row.id)) {
+        insertFile.run({
+          nodeId: newId,
+          path: file.path,
+          previousPath: file.previous_path,
+          insertions: file.insertions,
+          deletions: file.deletions,
+          isBinary: file.is_binary
+        });
+      }
+      migrated += 1;
+    }
+    dropEmbedding.run(row.id);
+    deleteNode.run(row.id);
+  }
+  return { migrated, deduped, skipped };
+}
+function reconcileProjectId(db, oldProjectId, newProjectId) {
+  return db.transaction(() => {
+    const sessions = recomputeByNaturalKey(
+      db,
+      oldProjectId,
+      newProjectId,
+      "session_summary",
+      null,
+      (_row, meta) => typeof meta.sessionKey === "string" ? meta.sessionKey : null
+    );
+    const hookShell = recomputeByNaturalKey(
+      db,
+      oldProjectId,
+      newProjectId,
+      "shell_command",
+      "shell:pwsh-hook",
+      (row, meta) => typeof meta.command === "string" ? `pwsh-hook:${row.ts}:${sha256Hex(meta.command).slice(0, 12)}` : null
+    );
+    const reassigned = db.prepare(`UPDATE nodes SET project_id = ? WHERE project_id = ? AND kind = 'conversation_turn'`).run(newProjectId, oldProjectId).changes;
+    return {
+      oldProjectId,
+      migrated: sessions.migrated + hookShell.migrated,
+      reassigned,
+      deduped: sessions.deduped + hookShell.deduped,
+      skipped: sessions.skipped + hookShell.skipped
+    };
+  })();
+}
+
 // src/vector/sync.ts
 var EMBEDDING_IDENTITY_KEY = "embedding.identity";
 var DEFAULT_BATCH_SIZE = 32;
@@ -3218,11 +3340,33 @@ async function runSync(opts) {
   const started = Date.now();
   try {
     store.upsertProject({ id: projectId, root: repo.root, originUrl: repo.originUrl });
-    await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
     if (opts.rebuild) {
       const removed = store.clearProject(projectId);
       log(`${pc4.dim("rebuild")} dropped ${removed} existing node(s)`);
     }
+    const staleProjectIds = store.listOtherProjectIds(projectId);
+    for (const staleId of staleProjectIds) {
+      const result = reconcileProjectId(store.raw, staleId, projectId);
+      const parts = [
+        result.migrated > 0 ? `${result.migrated} migrated` : null,
+        result.reassigned > 0 ? `${result.reassigned} reassigned` : null,
+        result.deduped > 0 ? `${result.deduped} already up to date` : null,
+        result.skipped > 0 ? `${result.skipped} left behind (not reconstructable)` : null
+      ].filter((part) => part !== null);
+      if (parts.length > 0) {
+        log(
+          `${pc4.yellow("reconciled")} previous project identity ${pc4.dim(staleId)} (remote URL likely changed): ${parts.join(", ")}`
+        );
+      }
+    }
+    if (staleProjectIds.length > 0) {
+      if (opts.rebuild) {
+        for (const staleId of staleProjectIds) store.clearProject(staleId);
+      }
+      await forgetProjects(staleProjectIds);
+      if (config.projectId !== projectId) await writeConfig(ws, { ...config, projectId });
+    }
+    await recordProject({ projectId, root: repo.root, dbPath: ws.dbPath, originUrl: repo.originUrl });
     const git2 = await syncGit(store, projectId, opts, repo, config, log);
     const diffs = await syncDiffs(store, projectId, opts, repo, config, log);
     const shell = await syncShell(store, projectId, opts, repo.root, config, log);
@@ -3477,7 +3621,8 @@ async function runQuery(opts) {
             packed: packed.nodes,
             tokensUsed: packed.tokensUsed,
             tokensBudget: packed.tokensBudget,
-            droppedForBudget: packed.droppedForBudget
+            droppedForBudget: packed.droppedForBudget,
+            droppedForDiversity: packed.droppedForDiversity
           },
           null,
           2
@@ -3494,7 +3639,7 @@ async function runQuery(opts) {
     process.stderr.write(
       [
         `${pc5.dim("matched")} ${bm25Count} bm25${vectorCount > 0 ? ` + ${vectorCount} vector` : ""}, packed ${pc5.bold(String(packed.nodes.length))} into budget`,
-        `${pc5.dim("tokens ")} ${packed.tokensUsed}/${packed.tokensBudget}` + (packed.droppedForBudget ? pc5.dim(`  (${packed.droppedForBudget} dropped for budget)`) : ""),
+        `${pc5.dim("tokens ")} ${packed.tokensUsed}/${packed.tokensBudget}` + (packed.droppedForBudget ? pc5.dim(`  (${packed.droppedForBudget} dropped for budget)`) : "") + (packed.droppedForDiversity ? pc5.dim(`  (${packed.droppedForDiversity} dropped for diversity)`) : ""),
         rawTokens > 0 ? `${pc5.dim("vs raw ")} ${rawTokens} tokens if these same matches were sent unpacked  ${packerEfficiency >= 0 ? pc5.green(`(${(packerEfficiency * 100).toFixed(0)}% packer efficiency)`) : pc5.yellow(`(${(-packerEfficiency * 100).toFixed(0)}% larger -- overhead dominates on small result sets)`)}` : "",
         ""
       ].filter(Boolean).join("\n")