@cognistore/mcp-server 2.1.2 → 2.2.0

package/dist/index.js

@@ -134,6 +134,12 @@ var updatePlanTaskSchema = z.object({
   notes: z.string().nullable().optional(),
   position: z.number().int().min(0).optional()
 });
+var mergeTagsBatchSchema = z.object({
+  merges: z.array(z.object({
+    from: z.string().min(1, "from is required"),
+    to: z.string().min(1, "to is required")
+  })).min(1, "at least one merge is required").max(50, "at most 50 merges per batch")
+});
 
 // ../../packages/core/dist/db/schema/index.js
 var schema_exports = {};
@@ -558,37 +564,37 @@ function createPlansEmbeddingsTable(sqlite, dimensions = DEFAULT_EMBEDDING_DIMEN
 // ../../packages/core/dist/repositories/knowledge.repository.js
 import { eq, ne, sql, and, or, isNull } from "drizzle-orm";
 var OPERATIONS_RETENTION_DAYS = 30;
-var KnowledgeRepository = class {
+var KnowledgeRepository = class _KnowledgeRepository {
   db;
   sqlite;
   constructor(db, sqlite) {
     this.db = db;
     this.sqlite = sqlite;
   }
+  /**
+   * Entry row + embedding are committed atomically: a failed embedding insert
+   * rolls the entry back instead of leaving an orphan row that semantic search
+   * can never find. Uses a RAW prepared insert (createPlan style) inside
+   * better-sqlite3's .transaction() — the body must be fully synchronous; an
+   * awaited drizzle insert opens an interleaving window where a concurrent
+   * create() issues BEGIN on the same connection ("cannot start a transaction
+   * within a transaction"). FTS stays best-effort inside the txn.
+   */
   async create(input) {
     const id = crypto.randomUUID();
     const now = (/* @__PURE__ */ new Date()).toISOString();
-    const [entry] = await this.db.insert(knowledgeEntries).values({
-      id,
-      title: input.title,
-      content: input.content,
-      tags: input.tags,
-      type: input.type,
-      scope: input.scope,
-      source: input.source,
-      confidenceScore: input.confidenceScore ?? 1,
-      expiresAt: input.expiresAt ? input.expiresAt.toISOString() : null,
-      relatedIds: input.relatedIds ?? null,
-      agentId: input.agentId ?? null,
-      createdAt: now,
-      updatedAt: now
-    }).returning();
-    insertEmbedding(this.sqlite, id, input.embedding);
-    try {
-      insertFts(this.sqlite, { id, title: input.title, content: input.content, tags: ftsTags(input.tags) });
-    } catch {
-    }
-    return entry;
+    const insertTxn = this.sqlite.transaction(() => {
+      this.sqlite.prepare(`INSERT INTO knowledge_entries
+           (id, title, content, tags, type, scope, source, version, expires_at, confidence_score, related_ids, agent_id, created_at, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?)`).run(id, input.title, input.content, JSON.stringify(input.tags ?? []), input.type, input.scope, input.source, input.expiresAt ? input.expiresAt.toISOString() : null, input.confidenceScore ?? 1, input.relatedIds ? JSON.stringify(input.relatedIds) : null, input.agentId ?? null, now, now);
+      insertEmbedding(this.sqlite, id, input.embedding);
+      try {
+        insertFts(this.sqlite, { id, title: input.title, content: input.content, tags: ftsTags(input.tags) });
+      } catch {
+      }
+    });
+    insertTxn();
+    return await this.findById(id);
   }
   async findById(id) {
     const [entry] = await this.db.select().from(knowledgeEntries).where(eq(knowledgeEntries.id, id));
@@ -719,6 +725,11 @@ var KnowledgeRepository = class {
     const result = await this.db.all(sql`SELECT value as tag, COUNT(*) as count FROM knowledge_entries, json_each(knowledge_entries.tags) WHERE knowledge_entries.type != 'system' GROUP BY value ORDER BY count DESC LIMIT ${limit}`);
     return result;
   }
+  /** Usage count for every tag (no limit/range) — feeds merge-keeper defaults. */
+  async tagCounts() {
+    const result = await this.db.all(sql`SELECT value as tag, COUNT(*) as count FROM knowledge_entries, json_each(knowledge_entries.tags) WHERE knowledge_entries.type != 'system' GROUP BY value`);
+    return result;
+  }
   /**
    * Rename/merge a tag across all entries. Per-row correlated rebuild of the JSON
    * tags array: replace `from`→`to`, and `json_group_array(DISTINCT ...)` collapses
@@ -768,14 +779,30 @@ var KnowledgeRepository = class {
   async findDuplicatePairs(opts = {}) {
     const threshold = opts.threshold ?? 0.9;
     const limit = opts.limit ?? 100;
+    const { pairs } = await this.collectDuplicatePairs(threshold, 5, limit);
+    return pairs.map((p) => ({ a: { id: p.a.id, title: p.a.title }, b: { id: p.b.id, title: p.b.title }, similarity: p.similarity })).sort((x, y) => y.similarity - x.similarity).slice(0, limit);
+  }
+  /**
+   * Shared KNN pair collection for findDuplicatePairs/findDuplicateGroups.
+   * `pairCap` exists only as a runaway safety net — group callers pass a large cap
+   * (a truncated pair graph would split/lose cluster members in union-find).
+   */
+  async collectDuplicatePairs(threshold, k, pairCap) {
     const entries = await this.listAll();
-    const meta = new Map(entries.map((e) => [e.id, e]));
+    const meta = new Map(entries.map((e) => [e.id, {
+      id: e.id,
+      title: e.title,
+      scope: e.scope,
+      type: e.type,
+      version: e.version ?? 1,
+      updatedAt: e.updatedAt
+    }]));
     const pairs = [];
     for (const entry of entries) {
       const emb = getEmbeddingById(this.sqlite, entry.id);
       if (!emb)
         continue;
-      const neighbors = searchKnn(this.sqlite, emb, 5);
+      const neighbors = searchKnn(this.sqlite, emb, k);
       for (const n of neighbors) {
         if (entry.id >= n.id)
           continue;
@@ -783,18 +810,78 @@ var KnowledgeRepository = class {
           continue;
         const similarity = 1 - n.distance;
         if (similarity >= threshold) {
-          const other = meta.get(n.id);
-          pairs.push({
-            a: { id: entry.id, title: entry.title },
-            b: { id: other.id, title: other.title },
-            similarity
-          });
+          pairs.push({ a: meta.get(entry.id), b: meta.get(n.id), similarity });
         }
       }
-      if (pairs.length >= limit)
+      if (pairs.length >= pairCap)
         break;
     }
-    return pairs.sort((x, y) => y.similarity - x.similarity).slice(0, limit);
+    return { pairs, meta };
+  }
+  // KNN width for duplicate GROUPING. Per-entry KNN keeps a cluster of up to
+  // ~DUP_KNN_K identical members fully connected; raise if users report a giant
+  // duplicate cluster rendering as two cards.
+  static DUP_KNN_K = 20;
+  // Safety net only — never intended to truncate a real pair graph (105 pairs for
+  // a 15-member cluster; 5000 covers pathological DBs without unbounded memory).
+  static DUP_PAIR_CAP = 5e3;
+  /**
+   * Cluster near-duplicate pairs into connected components (union-find) so N
+   * copies of one entry render as ONE group, not N(N-1)/2 repeated pair rows.
+   * `limit` applies to the GROUP list (size DESC, then similarity DESC) — never
+   * to the underlying pair collection.
+   */
+  async findDuplicateGroups(opts = {}) {
+    const threshold = opts.threshold ?? 0.9;
+    const limit = opts.limit ?? 100;
+    const { pairs } = await this.collectDuplicatePairs(threshold, _KnowledgeRepository.DUP_KNN_K, _KnowledgeRepository.DUP_PAIR_CAP);
+    const parent = /* @__PURE__ */ new Map();
+    const find = (x) => {
+      let root = x;
+      while (parent.get(root) !== void 0 && parent.get(root) !== root)
+        root = parent.get(root);
+      let cur = x;
+      while (cur !== root) {
+        const next = parent.get(cur);
+        parent.set(cur, root);
+        cur = next;
+      }
+      return root;
+    };
+    const union = (a, b) => {
+      if (!parent.has(a))
+        parent.set(a, a);
+      if (!parent.has(b))
+        parent.set(b, b);
+      const ra = find(a);
+      const rb = find(b);
+      if (ra !== rb)
+        parent.set(rb, ra);
+    };
+    const memberMeta = /* @__PURE__ */ new Map();
+    const maxSim = /* @__PURE__ */ new Map();
+    for (const p of pairs) {
+      union(p.a.id, p.b.id);
+      memberMeta.set(p.a.id, p.a);
+      memberMeta.set(p.b.id, p.b);
+    }
+    const grouped = /* @__PURE__ */ new Map();
+    for (const m of memberMeta.values()) {
+      const root = find(m.id);
+      const arr = grouped.get(root) ?? [];
+      arr.push(m);
+      grouped.set(root, arr);
+    }
+    for (const p of pairs) {
+      const root = find(p.a.id);
+      maxSim.set(root, Math.max(maxSim.get(root) ?? 0, p.similarity));
+    }
+    const groups = Array.from(grouped.entries()).map(([root, members]) => {
+      members.sort((a, b) => b.version - a.version || b.updatedAt.localeCompare(a.updatedAt));
+      const groupId = members.reduce((min, m) => m.id < min ? m.id : min, members[0].id);
+      return { groupId, maxSimilarity: maxSim.get(root) ?? 0, members };
+    });
+    return groups.sort((x, y) => y.members.length - x.members.length || y.maxSimilarity - x.maxSimilarity).slice(0, limit);
   }
   /**
    * (Re)populate the FTS5 index from knowledge_entries. Used at startup when the
@@ -1481,13 +1568,21 @@ var KnowledgeService = class {
    * O(n²) over the small DISTINCT tag set.
    */
   async suggestTagMerges(threshold = 0.82) {
-    const tags = await this.repository.listTags();
+    const counts = new Map((await this.repository.tagCounts()).map((r) => [r.tag, r.count]));
+    const tags = Array.from(counts.keys());
     const out = [];
     for (let i = 0; i < tags.length; i++) {
       for (let j = i + 1; j < tags.length; j++) {
         const sim = this.tagSimilarity(tags[i], tags[j]);
-        if (sim >= threshold)
-          out.push({ a: tags[i], b: tags[j], similarity: Math.round(sim * 100) / 100 });
+        if (sim >= threshold) {
+          out.push({
+            a: tags[i],
+            b: tags[j],
+            similarity: Math.round(sim * 100) / 100,
+            countA: counts.get(tags[i]) ?? 0,
+            countB: counts.get(tags[j]) ?? 0
+          });
+        }
       }
     }
     return out.sort((x, y) => y.similarity - x.similarity);
@@ -1533,12 +1628,90 @@ var KnowledgeService = class {
       this.logOp("write", affected.length);
     return { merged: affected.length };
   }
+  /**
+   * Apply several tag merges in ONE pass. All conflict detection happens BEFORE
+   * the first renameTag SQL executes, so a CONFLICT never leaves a partially
+   * merged DB:
+   *  - duplicate `from` mapped to different targets → CONFLICT error
+   *  - cycles (a→b, b→a) → CONFLICT error
+   *  - chains collapse to their terminal target (a→b, b→c ⇒ a→c, b→c), which is
+   *    equivalent to sequential application: renameTag is idempotent per terminal
+   *    target and json_group_array(DISTINCT …) collapses a pre-existing target.
+   * Affected entry ids are UNIONED so an entry touched by two merges is re-embedded
+   * exactly once, AFTER all renames (buildEmbeddingText reads the final tags).
+   * Re-embeds run through repository.update (UPDATE path: FTS resync + version
+   * bump — never the embedding-insert path) with bounded concurrency; one failed
+   * re-embed does not abort the rest.
+   */
+  async mergeTagsBatch(merges) {
+    const cleaned = merges.map((m) => ({ from: (m.from ?? "").trim(), to: (m.to ?? "").trim() })).filter((m) => m.from && m.to && m.from !== m.to);
+    if (cleaned.length === 0)
+      return { applied: [], entriesReembedded: 0 };
+    const target = /* @__PURE__ */ new Map();
+    for (const m of cleaned) {
+      const existing = target.get(m.from);
+      if (existing !== void 0 && existing !== m.to) {
+        throw new Error(`CONFLICT: tag "${m.from}" is merged into multiple targets ("${existing}" and "${m.to}")`);
+      }
+      target.set(m.from, m.to);
+    }
+    const terminal = /* @__PURE__ */ new Map();
+    for (const from of target.keys()) {
+      const visited = /* @__PURE__ */ new Set([from]);
+      let to = target.get(from);
+      while (target.has(to)) {
+        if (visited.has(to)) {
+          throw new Error(`CONFLICT: circular merge chain involving tag "${to}"`);
+        }
+        visited.add(to);
+        to = target.get(to);
+      }
+      terminal.set(from, to);
+    }
+    const applied = [];
+    const affectedIds = /* @__PURE__ */ new Set();
+    for (const [from, to] of terminal) {
+      const ids2 = this.repository.renameTag(from, to);
+      for (const id of ids2)
+        affectedIds.add(id);
+      applied.push({ from, to, count: ids2.length });
+    }
+    const ids = Array.from(affectedIds);
+    await this.mapWithConcurrency(ids, 4, async (id) => {
+      try {
+        const entry = await this.repository.findById(id);
+        if (!entry)
+          return;
+        const tags = Array.isArray(entry.tags) ? entry.tags : JSON.parse(entry.tags ?? "[]");
+        const embedding = await this.embeddingProvider.embed(this.buildEmbeddingText(entry.title, entry.content, tags));
+        await this.repository.update(id, { embedding });
+      } catch {
+      }
+    });
+    if (affectedIds.size)
+      this.logOp("write", affectedIds.size);
+    return { applied, entriesReembedded: affectedIds.size };
+  }
+  /** Minimal worker-pool: run `fn` over `items` with at most `limit` in flight. */
+  async mapWithConcurrency(items, limit, fn) {
+    let next = 0;
+    const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+      while (next < items.length) {
+        const i = next++;
+        await fn(items[i]);
+      }
+    });
+    await Promise.all(workers);
+  }
   async findStaleEntries(opts = {}) {
     return this.repository.findStaleEntries(opts);
   }
   async findDuplicatePairs(opts = {}) {
     return this.repository.findDuplicatePairs(opts);
   }
+  async findDuplicateGroups(opts = {}) {
+    return this.repository.findDuplicateGroups(opts);
+  }
   /** (Re)populate the FTS5 index if it's empty but entries exist. Returns rows indexed. */
   backfillFtsIfNeeded() {
     return this.repository.backfillFtsIfNeeded();
@@ -3310,6 +3483,16 @@ var KnowledgeSDK = class {
       throw this.wrapError(error, "Failed to merge tags");
     }
   }
+  async mergeTagsBatch(merges) {
+    this.ensureInitialized();
+    try {
+      return await this.service.mergeTagsBatch(merges);
+    } catch (error) {
+      if (error instanceof Error && error.message.startsWith("CONFLICT:"))
+        throw error;
+      throw this.wrapError(error, "Failed to merge tags");
+    }
+  }
   async findStaleEntries(opts = {}) {
     this.ensureInitialized();
     try {
@@ -3326,6 +3509,14 @@ var KnowledgeSDK = class {
       throw this.wrapError(error, "Failed to find duplicate pairs");
     }
   }
+  async findDuplicateGroups(opts = {}) {
+    this.ensureInitialized();
+    try {
+      return await this.service.findDuplicateGroups(opts);
+    } catch (error) {
+      throw this.wrapError(error, "Failed to find duplicate groups");
+    }
+  }
   async countByType(opts = {}) {
     this.ensureInitialized();
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cognistore/mcp-server",
-  "version": "2.1.2",
+  "version": "2.2.0",
   "private": false,
   "type": "module",
   "description": "MCP server for CogniStore — integrates with Claude Code and GitHub Copilot",