@loreai/core 0.17.1 → 0.19.0

package/src/ltm.ts

@@ -1,7 +1,7 @@
 import { uuidv7 } from "uuidv7";
-import { db, ensureProject } from "./db";
+import { db, ensureProject, getKV, setKV } from "./db";
 import { config } from "./config";
-import { ftsQuery, EMPTY_QUERY, extractTopTerms, runRelaxedSearch } from "./search";
+import { ftsQuery, ftsQueryOr, EMPTY_QUERY, extractTopTerms, filterTerms, runRelaxedSearch } from "./search";
 import * as embedding from "./embedding";
 import * as latReader from "./lat-reader";
 import * as log from "./log";
@@ -50,6 +50,10 @@ export function create(input: {
       ? ensureProject(input.projectPath)
       : null;
 
+  // IF-2: Global entries (pid=null) must be cross-project to avoid a data hole
+  // where forSession() can't find them in either the project or cross-project pool.
+  const crossProject = pid === null ? true : (input.crossProject ?? false);
+
   // Dedup guard: if an entry with the same project_id + title already exists,
   // update its content instead of inserting a duplicate. This prevents the
   // curator from creating multiple entries for the same concept across sessions.
@@ -90,6 +94,16 @@ export function create(input: {
       update(crossExisting.id, { content: input.content });
       return crossExisting.id;
     }
+
+    // Fuzzy dedup: check for title-similar entries via FTS5 + word-overlap.
+    // This catches near-duplicates the curator creates with slightly different
+    // titles for the same concept (e.g. "Upgrade lock bug" vs "Upgrade binary
+    // lock re-entry bug"). Placed after exact checks (cheaper checks first).
+    const fuzzyMatch = findFuzzyDuplicate({ title: input.title, projectId: pid });
+    if (fuzzyMatch) {
+      update(fuzzyMatch.id, { content: input.content });
+      return fuzzyMatch.id;
+    }
   }
 
   const id = input.id ?? uuidv7();
@@ -106,7 +120,7 @@ export function create(input: {
       input.title,
       input.content,
       input.session ?? null,
-      (input.crossProject ?? false) ? 1 : 0,
+      crossProject ? 1 : 0,
       now,
       now,
     );
@@ -130,8 +144,10 @@ export function update(
     params.push(input.content);
   }
   if (input.confidence !== undefined) {
+    // Clamp to [0.0, 1.0] — an LLM-provided value outside this range would
+    // give disproportionate scoring weight (>1) or silently soft-delete (<0.2).
     sets.push("confidence = ?");
-    params.push(input.confidence);
+    params.push(Math.max(0, Math.min(1, input.confidence)));
   }
   sets.push("updated_at = ?");
   params.push(Date.now());
@@ -153,6 +169,100 @@ export function remove(id: string) {
   db().query("DELETE FROM knowledge WHERE id = ?").run(id);
 }
 
+// ---------------------------------------------------------------------------
+// Fuzzy title dedup — word-overlap similarity
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute title word-overlap between two titles.
+ * Returns { coefficient, intersectionSize } where:
+ * - coefficient = |A ∩ B| / min(|A|, |B|) (0–1)
+ * - intersectionSize = number of shared meaningful words
+ * Filters stopwords and single-char tokens for meaningful comparison.
+ */
+function titleOverlap(a: string, b: string): { coefficient: number; intersectionSize: number } {
+  const wordsA = new Set(filterTerms(a).map((w) => w.toLowerCase()));
+  const wordsB = new Set(filterTerms(b).map((w) => w.toLowerCase()));
+  if (wordsA.size === 0 || wordsB.size === 0) return { coefficient: 0, intersectionSize: 0 };
+  const intersection = [...wordsA].filter((w) => wordsB.has(w));
+  return {
+    coefficient: intersection.length / Math.min(wordsA.size, wordsB.size),
+    intersectionSize: intersection.length,
+  };
+}
+
+/** Minimum word-overlap coefficient to consider two titles as duplicates. */
+const FUZZY_DEDUP_THRESHOLD = 0.7;
+/** Minimum number of overlapping meaningful words required for a fuzzy match.
+ *  Prevents false positives on short titles where 2-3 common words produce
+ *  a high overlap coefficient despite being genuinely different entries. */
+const FUZZY_DEDUP_MIN_OVERLAP = 4;
+/** Minimum cosine similarity for embedding-based dedup. Empirically tuned
+ *  against 312 Nomic v1.5 entries:
+ *  - 0.935+: all genuine duplicates (same topic, different wording)
+ *  - 0.92–0.935: contains false positives from same-subsystem entries
+ *    (e.g. "BGE Small unusable" ↔ "Nomic OOM" scored 0.9326 — related
+ *    but distinct bugs). Star clustering amplifies this by bridging.
+ *  - <0.92: mixed or unrelated entries */
+const EMBEDDING_DEDUP_THRESHOLD = 0.935;
+
+/**
+ * Find an existing knowledge entry whose title is fuzzy-similar to the given title.
+ *
+ * Uses FTS5 to find up to 5 candidates, then applies word-overlap filtering.
+ * This is the same algorithm used by `check()` but returns a single match
+ * for use in the `create()` dedup guard.
+ *
+ * @returns The first matching entry (id + title), or null if no fuzzy match.
+ */
+export function findFuzzyDuplicate(input: {
+  title: string;
+  projectId: string | null;
+  excludeId?: string;
+}): { id: string; title: string } | null {
+  const q = ftsQueryOr(input.title);
+  if (q === EMPTY_QUERY) return null;
+
+  const { title: tw, content: cw, category: catw } = config().search.ftsWeights;
+
+  try {
+    // Build query scoped to the same project + cross-project entries
+    const excludeClause = input.excludeId ? "AND k.id != ?" : "";
+    const sql = input.projectId !== null
+      ? `SELECT k.id, k.title FROM knowledge_fts f
+         CROSS JOIN knowledge k ON k.rowid = f.rowid
+         WHERE knowledge_fts MATCH ?
+         AND (k.project_id = ? OR k.cross_project = 1)
+         AND k.confidence > 0.2
+         ${excludeClause}
+         ORDER BY bm25(knowledge_fts, ?, ?, ?) LIMIT 5`
+      : `SELECT k.id, k.title FROM knowledge_fts f
+         CROSS JOIN knowledge k ON k.rowid = f.rowid
+         WHERE knowledge_fts MATCH ?
+         AND (k.project_id IS NULL OR k.cross_project = 1)
+         AND k.confidence > 0.2
+         ${excludeClause}
+         ORDER BY bm25(knowledge_fts, ?, ?, ?) LIMIT 5`;
+
+    const params: (string | number)[] = input.projectId !== null
+      ? [q, input.projectId, ...(input.excludeId ? [input.excludeId] : []), tw, cw, catw]
+      : [q, ...(input.excludeId ? [input.excludeId] : []), tw, cw, catw];
+
+    const candidates = db().query(sql).all(...params) as Array<{ id: string; title: string }>;
+
+    for (const candidate of candidates) {
+      const { coefficient, intersectionSize } = titleOverlap(input.title, candidate.title);
+      if (coefficient >= FUZZY_DEDUP_THRESHOLD && intersectionSize >= FUZZY_DEDUP_MIN_OVERLAP) {
+        return candidate;
+      }
+    }
+  } catch {
+    // FTS5 error — fall through to no match
+  }
+
+  return null;
+}
+
 export function forProject(
   projectPath: string,
   includeCross = true,
@@ -245,6 +355,26 @@ function scoreEntriesFTS(sessionContext: string): Map<string, number> {
   }
 }
 
+/**
+ * Well-known knowledge entry categories managed by the curator.
+ * The DB column is a free-form string, but these are the standard values.
+ */
+export type KnowledgeCategory = "decision" | "pattern" | "preference" | "architecture" | "gotcha";
+
+/** Options for `forSession()` to control entry selection. */
+export type ForSessionOptions = {
+  /** Caller-provided context (e.g., user's current message) for relevance
+   *  scoring when no session context exists in the DB yet. */
+  contextHint?: string;
+  /** Restrict to these categories (e.g., `['preference']` for turn 1). */
+  categories?: (KnowledgeCategory | (string & {}))[];
+  /** Exclude these categories (e.g., `['preference']` for context-bound
+   *  entries when preferences are already injected in a separate block).
+   *  Mutually exclusive with `categories` — if both are provided,
+   *  `categories` (include) wins. */
+  excludeCategories?: (KnowledgeCategory | (string & {}))[];
+};
+
 /**
  * Build a relevance-ranked, budget-capped list of knowledge entries for injection
  * into the system prompt of a live session.
@@ -252,43 +382,61 @@ function scoreEntriesFTS(sessionContext: string): Map<string, number> {
  * Strategy:
  * 1. Both project-specific and cross-project entries are scored for relevance
  *    against recent session context (last distillation + recent raw messages).
- * 2. Project entries get a safety net: the top PROJECT_SAFETY_NET entries by
+ * 2. When embeddings are available, vector cosine similarity is used for scoring
+ *    (captures semantic matches that keyword overlap misses). Falls back to
+ *    FTS5 BM25 when embeddings are unavailable.
+ * 3. Project entries get a safety net: the top PROJECT_SAFETY_NET entries by
  *    confidence are always included even if they have zero relevance score.
  *    This ensures the most important project knowledge is never lost to
- *    coarse term-overlap scoring.
- * 3. All scored entries are merged into a single pool and greedily packed
+ *    coarse scoring.
+ * 4. All scored entries are merged into a single pool and greedily packed
  *    into the token budget by score descending.
- * 4. If there's no session context yet (first turn), fall back to top entries
+ * 5. If there's no session context yet (first turn), fall back to top entries
  *    by confidence only (capped at NO_CONTEXT_FALLBACK_CAP per pool).
  *
  * @param projectPath   Current project path
  * @param sessionID     Current session ID (for context extraction)
  * @param maxTokens     Hard token budget for the entire formatted block
+ * @param options       Optional category filter and context hint
  */
-export function forSession(
+export async function forSession(
   projectPath: string,
   sessionID: string | undefined,
   maxTokens: number,
-): KnowledgeEntry[] {
+  options?: ForSessionOptions,
+): Promise<KnowledgeEntry[]> {
   const pid = ensureProject(projectPath);
+  const categoryFilter = options?.categories;
+  const excludeFilter = options?.excludeCategories;
+
+  // Build optional SQL category clauses (include / exclude are mutually exclusive)
+  let categoryClause = "";
+  let categoryParams: string[] = [];
+  if (categoryFilter?.length) {
+    categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
+    categoryParams = categoryFilter;
+  } else if (excludeFilter?.length) {
+    categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
+    categoryParams = excludeFilter;
+  }
 
   // --- 1. Load project-specific entries ---
   const projectEntries = db()
     .query(
       `SELECT ${KNOWLEDGE_COLS} FROM knowledge
-       WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
+       WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
        ORDER BY confidence DESC, updated_at DESC`,
     )
-    .all(pid) as KnowledgeEntry[];
+    .all(pid, ...categoryParams) as KnowledgeEntry[];
 
   // --- 2. Load cross-project candidates ---
   const crossEntries = db()
     .query(
       `SELECT ${KNOWLEDGE_COLS} FROM knowledge
-       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
+       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
        ORDER BY confidence DESC, updated_at DESC`,
     )
-    .all() as KnowledgeEntry[];
+    .all(...categoryParams) as KnowledgeEntry[];
 
   if (!crossEntries.length && !projectEntries.length) return [];
 
@@ -317,38 +465,82 @@ export function forSession(
     }
   }
 
+  // Fall back to caller-provided context hint (e.g., user's first message)
+  if (!sessionContext.trim() && options?.contextHint) {
+    sessionContext = options.contextHint;
+  }
+
   // --- 4. Score both pools by relevance ---
   let scoredProject: Scored[];
   let scoredCross: Scored[];
 
-  if (sessionContext.trim().length > 20) {
-    // Use FTS5 BM25 to score all knowledge entries against session context
-    const ftsScores = scoreEntriesFTS(sessionContext);
-
-    // Score project entries: FTS relevance × confidence, with safety net
-    const rawScored: Scored[] = projectEntries.map((entry) => ({
-      entry,
-      score: (ftsScores.get(entry.id) ?? 0) * entry.confidence,
-    }));
-    const matched = rawScored.filter((s) => s.score > 0);
-    const matchedIds = new Set(matched.map((s) => s.entry.id));
-
-    // Safety net: top PROJECT_SAFETY_NET entries by confidence that weren't already matched.
-    // Given a tiny score (0.001 * confidence) so they sort below genuinely matched entries.
-    const safetyNet = projectEntries
-      .filter((e) => !matchedIds.has(e.id))
-      .slice(0, PROJECT_SAFETY_NET)
-      .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
-
-    scoredProject = [...matched, ...safetyNet];
+  if (sessionContext.trim().length > 20 && embedding.isAvailable()) {
+    // Vector scoring: embed session context, score entries by cosine similarity.
+    // Captures semantic matches (e.g., "OpenAI Batch API" ↔ "batch queue worker")
+    // that keyword-based FTS5 misses.
+    let vectorScores: Map<string, number>;
+    try {
+      const [contextVec] = await embedding.embed([sessionContext], "query");
+      const hits = embedding.vectorSearch(contextVec, 50, excludeFilter);
+      vectorScores = new Map(hits.map((h) => [h.id, h.similarity]));
+    } catch (err) {
+      log.warn("Vector scoring failed, falling back to FTS5:", err);
+      vectorScores = new Map();
+    }
 
-    // Score cross-project entries — only include entries with FTS match
-    scoredCross = crossEntries
-      .filter((e) => ftsScores.has(e.id))
-      .map((e) => ({
-        entry: e,
-        score: (ftsScores.get(e.id) ?? 0) * e.confidence,
-      }));
+    if (vectorScores.size > 0) {
+      // Hybrid scoring: vector search only covers entries with stored embeddings.
+      // Entries without embeddings (e.g. newly created, async embed not yet done)
+      // fall back to FTS5 so they aren't invisible to scoring.
+      const ftsScores = scoreEntriesFTS(sessionContext);
+
+      // Score project entries: prefer vector similarity, fall back to FTS5
+      const rawScored: Scored[] = projectEntries.map((entry) => {
+        const vecScore = vectorScores.get(entry.id);
+        const score = vecScore != null
+          ? vecScore * entry.confidence
+          : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
+        return { entry, score };
+      });
+      const matched = rawScored.filter((s) => s.score > 0);
+      const matchedIds = new Set(matched.map((s) => s.entry.id));
+
+      // Safety net: top PROJECT_SAFETY_NET entries by confidence that weren't already matched.
+      // Given a tiny score (0.001 * confidence) so they sort below genuinely matched entries.
+      const safetyNet = projectEntries
+        .filter((e) => !matchedIds.has(e.id))
+        .slice(0, PROJECT_SAFETY_NET)
+        .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
+
+      scoredProject = [...matched, ...safetyNet];
+
+      // Cross-project: include entries matched by vector OR FTS5
+      scoredCross = crossEntries
+        .filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id))
+        .map((e) => {
+          const vecScore = vectorScores.get(e.id);
+          const score = vecScore != null
+            ? vecScore * e.confidence
+            : (ftsScores.get(e.id) ?? 0) * e.confidence;
+          return { entry: e, score };
+        });
+    } else {
+      // Vector failed — fall through to FTS5
+      const ftsScores = scoreEntriesFTS(sessionContext);
+      ({ scoredProject, scoredCross } = scoreFTS(
+        projectEntries,
+        crossEntries,
+        ftsScores,
+      ));
+    }
+  } else if (sessionContext.trim().length > 20) {
+    // Embeddings unavailable — use FTS5 BM25 as fallback
+    const ftsScores = scoreEntriesFTS(sessionContext);
+    ({ scoredProject, scoredCross } = scoreFTS(
+      projectEntries,
+      crossEntries,
+      ftsScores,
+    ));
   } else {
     // No session context — fall back to top entries by confidence, capped
     scoredProject = projectEntries
@@ -410,6 +602,36 @@ export function forSession(
   return result;
 }
 
+/** Score entries using FTS5 BM25 — extracted for reuse in the vector-fallback path. */
+function scoreFTS(
+  projectEntries: KnowledgeEntry[],
+  crossEntries: KnowledgeEntry[],
+  ftsScores: Map<string, number>,
+): { scoredProject: Scored[]; scoredCross: Scored[] } {
+  const rawScored: Scored[] = projectEntries.map((entry) => ({
+    entry,
+    score: (ftsScores.get(entry.id) ?? 0) * entry.confidence,
+  }));
+  const matched = rawScored.filter((s) => s.score > 0);
+  const matchedIds = new Set(matched.map((s) => s.entry.id));
+
+  const safetyNet = projectEntries
+    .filter((e) => !matchedIds.has(e.id))
+    .slice(0, PROJECT_SAFETY_NET)
+    .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
+
+  const scoredProject = [...matched, ...safetyNet];
+
+  const scoredCross = crossEntries
+    .filter((e) => ftsScores.has(e.id))
+    .map((e) => ({
+      entry: e,
+      score: (ftsScores.get(e.id) ?? 0) * e.confidence,
+    }));
+
+  return { scoredProject, scoredCross };
+}
+
 export function all(): KnowledgeEntry[] {
   return db()
     .query(
@@ -418,6 +640,17 @@ export function all(): KnowledgeEntry[] {
     .all() as KnowledgeEntry[];
 }
 
+/** Return all cross-project and global (user-level) knowledge entries. */
+export function crossProject(): KnowledgeEntry[] {
+  return db()
+    .query(
+      `SELECT ${KNOWLEDGE_COLS} FROM knowledge
+       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
+       ORDER BY confidence DESC, updated_at DESC`,
+    )
+    .all() as KnowledgeEntry[];
+}
+
 // LIKE-based fallback for when FTS5 fails unexpectedly.
 function searchLike(input: {
   query: string;
@@ -832,3 +1065,518 @@ export function check(projectPath: string): IntegrityIssue[] {
 
   return issues;
 }
+
+// ---------------------------------------------------------------------------
+// Deduplication — embedding-based semantic clustering with word-overlap fallback
+// ---------------------------------------------------------------------------
+
+export type DedupCluster = {
+  surviving: { id: string; title: string };
+  merged: Array<{ id: string; title: string }>;
+};
+
+/** Stable pair key for two entry IDs — sorted to ensure order-independence. */
+export function dedupPairKey(idA: string, idB: string): string {
+  return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
+}
+
+export type DedupResult = {
+  clusters: DedupCluster[];
+  totalRemoved: number;
+  /** Pairwise embedding cosine similarities. Key: dedupPairKey(idA, idB). */
+  pairSimilarities: Map<string, number>;
+  /** All entry titles by ID — for feedback recording after entries are deleted. */
+  entryTitles: Map<string, string>;
+};
+
+/**
+ * Deduplicate knowledge entries for a project.
+ *
+ * Uses two complementary signals with "star" clustering (no transitive
+ * chains) to prevent snowball merging:
+ *
+ * 1. **Title word-overlap** (Jaccard on meaningful words) — catches entries
+ *    with similar titles regardless of content wording.
+ * 2. **Embedding cosine similarity** (when embeddings are available) — catches
+ *    entries with different titles but semantically identical content. Nomic
+ *    v1.5 produces a same-domain spread of 0.46–0.70 for distinct entries,
+ *    making threshold-based dedup viable at 0.935+ (lower thresholds catch
+ *    related-but-distinct entries as false positives, especially via star
+ *    clustering where a hub entry bridges two distinct topics).
+ *
+ * Pairs matching either signal are clustered together. For each cluster,
+ * picks a survivor (highest confidence, then most recently updated, then
+ * shortest title) and removes the rest.
+ *
+ * @param projectPath   Project root path
+ * @param opts.dryRun   If true (default), report clusters without deleting
+ * @returns             Cluster report and count of removed entries
+ */
+/** Core dedup logic — operates on an arbitrary list of entries. */
+function _dedup(
+  entries: KnowledgeEntry[],
+  dryRun: boolean,
+  embeddingThreshold: number = EMBEDDING_DEDUP_THRESHOLD,
+): DedupResult {
+  if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: new Map(), entryTitles: new Map() };
+
+  // --- Build neighbor map using title overlap + embedding similarity ---
+  // Two entries are considered neighbors (potential duplicates) if EITHER:
+  //   (a) title word-overlap ≥ 0.7 with ≥ 4 shared words, OR
+  //   (b) embedding cosine similarity ≥ embeddingThreshold (default 0.935)
+  // Star clustering (no transitivity) prevents snowball merging.
+  // O(n²) pairwise comparison — acceptable for n ≤ 25 (maxEntries cap).
+
+  // Load embeddings for the given entries (if available).
+  // We query directly rather than using vectorSearch() because we need
+  // pairwise comparison among entries, not a query-vs-all search.
+  const embeddingMap = new Map<string, Float32Array>();
+  {
+    const entryIds = entries.map((e) => e.id);
+    // Build parameterized IN clause for the entry IDs
+    const placeholders = entryIds.map(() => "?").join(",");
+    const rows = db()
+      .query(`SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND id IN (${placeholders})`)
+      .all(...entryIds) as Array<{ id: string; embedding: Buffer }>;
+    for (const row of rows) {
+      try {
+        embeddingMap.set(row.id, embedding.fromBlob(row.embedding));
+      } catch {
+        // Skip corrupted embeddings — entry falls back to title-overlap only.
+        log.info(`skipping corrupted embedding for entry ${row.id}`);
+      }
+    }
+  }
+
+  // Pre-compute neighbors for all pairs
+  type DedupHit = { id: string; score: number };
+  const neighborMap = new Map<string, DedupHit[]>();
+  // Collect all pairwise embedding similarities (for feedback/calibration).
+  const pairSimilarities = new Map<string, number>();
+
+  for (const entry of entries) {
+    const neighbors: DedupHit[] = [];
+    const entryVec = embeddingMap.get(entry.id);
+
+    for (const other of entries) {
+      if (other.id === entry.id) continue;
+
+      // Signal 1: title word-overlap
+      const { coefficient, intersectionSize } = titleOverlap(entry.title, other.title);
+      const titleMatch = coefficient >= FUZZY_DEDUP_THRESHOLD && intersectionSize >= FUZZY_DEDUP_MIN_OVERLAP;
+
+      // Signal 2: embedding cosine similarity
+      let embeddingMatch = false;
+      let similarity = 0;
+      if (entryVec) {
+        const otherVec = embeddingMap.get(other.id);
+        if (otherVec && entryVec.length === otherVec.length) {
+          similarity = embedding.cosineSimilarity(entryVec, otherVec);
+          embeddingMatch = similarity >= embeddingThreshold;
+        }
+      }
+
+      // Track all pairwise embedding similarities for calibration signals
+      if (similarity > 0) {
+        const pk = dedupPairKey(entry.id, other.id);
+        if (!pairSimilarities.has(pk)) {
+          pairSimilarities.set(pk, similarity);
+        }
+      }
+
+      if (titleMatch || embeddingMatch) {
+        // Use the stronger signal as the match score for cluster priority
+        neighbors.push({ id: other.id, score: Math.max(coefficient, similarity) });
+      }
+    }
+    neighbors.sort((a, b) => b.score - a.score);
+    neighborMap.set(entry.id, neighbors);
+  }
+
+  // Greedy star clustering — process entries with most neighbors first
+  const claimed = new Set<string>();
+  const rawClusters = new Map<string, string[]>();
+
+  const sortedIds = [...neighborMap.keys()].sort(
+    (a, b) => neighborMap.get(b)!.length - neighborMap.get(a)!.length,
+  );
+
+  for (const centerId of sortedIds) {
+    if (claimed.has(centerId)) continue;
+    claimed.add(centerId);
+    const members = [centerId];
+
+    for (const { id: neighborId } of neighborMap.get(centerId)!) {
+      if (claimed.has(neighborId)) continue;
+      claimed.add(neighborId);
+      members.push(neighborId);
+    }
+
+    if (members.length > 1) {
+      rawClusters.set(centerId, members);
+    }
+  }
+
+  // Build clusters and pick survivors
+  const entryById = new Map(entries.map((e) => [e.id, e]));
+  const result: DedupCluster[] = [];
+  let totalRemoved = 0;
+
+  for (const members of rawClusters.values()) {
+    if (members.length < 2) continue;
+
+    // Pick survivor: highest confidence → most recent → shortest title
+    const sorted = members
+      .map((id) => entryById.get(id)!)
+      .filter(Boolean)
+      .sort((a, b) => {
+        if (b.confidence !== a.confidence) return b.confidence - a.confidence;
+        if (b.updated_at !== a.updated_at) return b.updated_at - a.updated_at;
+        return a.title.length - b.title.length;
+      });
+
+    const survivor = sorted[0];
+    const merged = sorted.slice(1);
+
+    result.push({
+      surviving: { id: survivor.id, title: survivor.title },
+      merged: merged.map((e) => ({ id: e.id, title: e.title })),
+    });
+
+    if (!dryRun) {
+      for (const entry of merged) {
+        remove(entry.id);
+      }
+    }
+
+    totalRemoved += merged.length;
+  }
+
+  // Sort clusters by size descending for readability
+  result.sort((a, b) => b.merged.length - a.merged.length);
+
+  // Build title map from all input entries — survives entry deletion.
+  const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
+
+  return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
+}
+
+export async function deduplicate(
+  projectPath: string,
+  opts?: { dryRun?: boolean },
+): Promise<DedupResult> {
+  const pid = ensureProject(projectPath);
+  const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
+  const entries = forProject(projectPath, false);
+  return _dedup(entries, opts?.dryRun ?? true, threshold);
+}
+
+/** Deduplicate global (cross-project) entries that have no project_id. */
+export async function deduplicateGlobal(
+  opts?: { dryRun?: boolean },
+): Promise<DedupResult> {
+  const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
+  const entries = db()
+    .query(
+      `SELECT ${KNOWLEDGE_COLS} FROM knowledge
+       WHERE project_id IS NULL
+       AND confidence > 0.2
+       ORDER BY confidence DESC, updated_at DESC`,
+    )
+    .all() as KnowledgeEntry[];
+  return _dedup(entries, opts?.dryRun ?? true, threshold);
+}
+
+// ---------------------------------------------------------------------------
+// Dedup feedback & adaptive threshold calibration
+// ---------------------------------------------------------------------------
+
+export type DedupFeedbackSource = "auto_dedup" | "cli_yes" | "cli_interactive";
+
+const MIN_CALIBRATION_SAMPLES = 20;
+const DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
+/** Only record auto-signals for pairs with similarity >= this floor. */
+const AUTO_SIGNAL_MIN_SIMILARITY = 0.80;
+/** Max auto-signal pairs to record per dedup run (closest to threshold). */
+const AUTO_SIGNAL_MAX_PAIRS = 50;
+
+/** Record a single dedup feedback row. */
+export function recordDedupFeedback(input: {
+  projectId: string | null;
+  entryATitle: string;
+  entryBTitle: string;
+  similarity: number;
+  accepted: boolean;
+  source: DedupFeedbackSource;
+}): void {
+  db()
+    .query(
+      `INSERT INTO dedup_feedback
+         (project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      input.projectId,
+      input.entryATitle,
+      input.entryBTitle,
+      input.similarity,
+      input.accepted ? 1 : 0,
+      input.source,
+      Date.now(),
+    );
+}
+
+/**
+ * Bulk-record feedback for all merged pairs in a DedupResult.
+ * Only records pairs with embedding similarity > 0 (title-overlap-only
+ * matches are excluded from calibration).
+ */
+export function recordDedupResultFeedback(
+  projectId: string | null,
+  result: DedupResult,
+  accepted: boolean,
+  source: DedupFeedbackSource,
+): void {
+  for (const cluster of result.clusters) {
+    for (const merged of cluster.merged) {
+      const pk = dedupPairKey(cluster.surviving.id, merged.id);
+      const similarity = result.pairSimilarities.get(pk);
+      if (similarity != null && similarity > 0) {
+        recordDedupFeedback({
+          projectId,
+          entryATitle: cluster.surviving.title,
+          entryBTitle: merged.title,
+          similarity,
+          accepted,
+          source,
+        });
+      }
+    }
+  }
+}
+
+/**
+ * Record automatic calibration signals from a post-curation dedup sweep.
+ *
+ * Only records **reject** signals — non-merged pairs with similarity in
+ * [0.80, threshold). Accept signals from auto-dedup are tautological (the
+ * pair was merged *because* its similarity exceeded the threshold), so they
+ * provide no new information and would create a self-reinforcing feedback
+ * loop. Manual signals (cli_yes, cli_interactive) provide the accept side.
+ *
+ * Caps at AUTO_SIGNAL_MAX_PAIRS most interesting pairs per run (closest
+ * to the threshold boundary) to avoid table bloat.
+ */
+export function recordAutoSignals(
+  projectId: string | null,
+  result: DedupResult,
+): void {
+  // Collect merged pair IDs for quick lookup (to exclude from reject signals)
+  const mergedPairs = new Set<string>();
+  for (const cluster of result.clusters) {
+    for (const merged of cluster.merged) {
+      mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
+    }
+  }
+
+  // Build a title map — we need titles for reject signals (non-merged pairs).
+  // Use entryTitles from result first, then fall back to cluster data.
+  const titleMap = new Map<string, string>(result.entryTitles);
+  for (const cluster of result.clusters) {
+    if (!titleMap.has(cluster.surviving.id)) {
+      titleMap.set(cluster.surviving.id, cluster.surviving.title);
+    }
+    for (const m of cluster.merged) {
+      if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
+    }
+  }
+
+  // Collect reject signals: non-merged pairs with high similarity
+  type Signal = { entryATitle: string; entryBTitle: string; similarity: number };
+  const signals: Signal[] = [];
+
+  for (const [pk, sim] of result.pairSimilarities) {
+    if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
+    if (mergedPairs.has(pk)) continue; // merged pair — skip (tautological accept)
+
+    const [idA, idB] = pk.split(":");
+    const titleA = titleMap.get(idA);
+    const titleB = titleMap.get(idB);
+    if (!titleA || !titleB) continue;
+
+    signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
+  }
+
+  // Sort by distance to threshold boundary (most informative first), cap
+  const currentThreshold = loadCalibratedThreshold(projectId) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
+  signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
+  const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
+
+  // Prune old feedback to prevent unbounded table growth
+  pruneDedupFeedback(projectId);
+
+  for (const s of capped) {
+    recordDedupFeedback({
+      projectId,
+      entryATitle: s.entryATitle,
+      entryBTitle: s.entryBTitle,
+      similarity: s.similarity,
+      accepted: false,
+      source: "auto_dedup",
+    });
+  }
+}
+
+/** Get all feedback for a project (for calibration). */
+export function getDedupFeedback(
+  projectId: string | null,
+): Array<{ similarity: number; accepted: boolean; source: string }> {
+  const rows = (
+    projectId !== null
+      ? db()
+          .query(
+            "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity",
+          )
+          .all(projectId)
+      : db()
+          .query(
+            "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity",
+          )
+          .all()
+  ) as Array<{ similarity: number; accepted: number; source: string }>;
+  return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
+}
+
+/** Quick count of feedback rows for a project. */
+export function getDedupFeedbackCount(projectId: string | null): number {
+  const row = (
+    projectId !== null
+      ? db()
+          .query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?")
+          .get(projectId)
+      : db()
+          .query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL")
+          .get()
+  ) as { cnt: number } | null;
+  return row?.cnt ?? 0;
+}
+
+/** Max feedback rows to keep per project (prevents unbounded growth). */
+const MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
+
+/**
+ * Prune old feedback rows for a project, keeping the most recent
+ * MAX_FEEDBACK_ROWS_PER_PROJECT rows. Called from recordAutoSignals
+ * to prevent unbounded table growth.
+ */
+export function pruneDedupFeedback(projectId: string | null): void {
+  const count = getDedupFeedbackCount(projectId);
+  if (count <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
+
+  const excess = count - MAX_FEEDBACK_ROWS_PER_PROJECT;
+  if (projectId !== null) {
+    db()
+      .query(
+        `DELETE FROM dedup_feedback WHERE id IN (
+           SELECT id FROM dedup_feedback WHERE project_id = ?
+           ORDER BY created_at ASC LIMIT ?
+         )`,
+      )
+      .run(projectId, excess);
+  } else {
+    db()
+      .query(
+        `DELETE FROM dedup_feedback WHERE id IN (
+           SELECT id FROM dedup_feedback WHERE project_id IS NULL
+           ORDER BY created_at ASC LIMIT ?
+         )`,
+      )
+      .run(excess);
+  }
+}
+
+/**
+ * Compute an optimal embedding dedup threshold from user feedback.
+ *
+ * Algorithm:
+ * 1. Load all (similarity, accepted) pairs for the project.
+ * 2. If fewer than MIN_CALIBRATION_SAMPLES, return null (use default).
+ * 3. If all feedback is "accept" (no rejects), return the minimum
+ *    accepted similarity minus a small margin (0.005).
+ * 4. If all feedback is "reject" (no accepts), return null.
+ * 5. Otherwise, find the threshold that maximizes separation:
+ *    - For each candidate threshold (midpoint between consecutive
+ *      distinct similarity values), compute accuracy:
+ *        correct = accepted_pairs_above + rejected_pairs_below
+ *        accuracy = correct / total
+ *    - Pick the threshold with highest accuracy.
+ *    - Tie-break: prefer higher threshold (conservative).
+ *    - Clamp to [0.85, 0.98].
+ */
+export function calibrateDedupThreshold(projectId: string | null): number | null {
+  const feedback = getDedupFeedback(projectId);
+  if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
+
+  const accepted = feedback.filter((f) => f.accepted);
+  const rejected = feedback.filter((f) => !f.accepted);
+
+  // Edge case: all accept, no rejects
+  if (rejected.length === 0) {
+    const minAccepted = Math.min(...accepted.map((f) => f.similarity));
+    return Math.max(0.85, minAccepted - 0.005);
+  }
+
+  // Edge case: all reject, no accepts
+  if (accepted.length === 0) {
+    log.warn("dedup calibration: all feedback is reject — keeping default threshold");
+    return null;
+  }
+
+  // Find optimal threshold via accuracy maximization
+  const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
+
+  let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
+  let bestAccuracy = -1;
+
+  for (let i = 0; i < allSims.length - 1; i++) {
+    const candidate = (allSims[i] + allSims[i + 1]) / 2;
+
+    // Pairs above threshold are predicted "merge" — should be accepted
+    // Pairs below threshold are predicted "keep separate" — should be rejected
+    const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
+    const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
+    const accuracy = (correctAccepted + correctRejected) / feedback.length;
+
+    // Tie-break: prefer higher threshold (conservative — fewer false merges)
+    if (accuracy > bestAccuracy || (accuracy === bestAccuracy && candidate > bestThreshold)) {
+      bestAccuracy = accuracy;
+      bestThreshold = candidate;
+    }
+  }
+
+  // Clamp to sane range
+  return Math.max(0.85, Math.min(0.98, bestThreshold));
+}
+
+/** Persist the calibrated threshold for a project. */
+export function saveCalibratedThreshold(
+  projectId: string | null,
+  threshold: number,
+  sampleSize: number,
+): void {
+  const key = `dedup_threshold:${projectId ?? "global"}`;
+  setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
+}
+
+/** Load the calibrated threshold for a project, or null if not calibrated. */
+export function loadCalibratedThreshold(projectId: string | null): number | null {
+  const key = `dedup_threshold:${projectId ?? "global"}`;
+  const raw = getKV(key);
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw);
+    return typeof parsed.threshold === "number" ? parsed.threshold : null;
+  } catch {
+    return null;
+  }
+}