@loreai/core 0.18.0 → 0.19.0

package/src/ltm.ts

@@ -1,5 +1,5 @@
 import { uuidv7 } from "uuidv7";
-import { db, ensureProject } from "./db";
+import { db, ensureProject, getKV, setKV } from "./db";
 import { config } from "./config";
 import { ftsQuery, ftsQueryOr, EMPTY_QUERY, extractTopTerms, filterTerms, runRelaxedSearch } from "./search";
 import * as embedding from "./embedding";
@@ -355,6 +355,26 @@ function scoreEntriesFTS(sessionContext: string): Map<string, number> {
   }
 }
 
+/**
+ * Well-known knowledge entry categories managed by the curator.
+ * The DB column is a free-form string, but these are the standard values.
+ */
+export type KnowledgeCategory = "decision" | "pattern" | "preference" | "architecture" | "gotcha";
+
+/** Options for `forSession()` to control entry selection. */
+export type ForSessionOptions = {
+  /** Caller-provided context (e.g., user's current message) for relevance
+   *  scoring when no session context exists in the DB yet. */
+  contextHint?: string;
+  /** Restrict to these categories (e.g., `['preference']` for turn 1). */
+  categories?: (KnowledgeCategory | (string & {}))[];
+  /** Exclude these categories (e.g., `['preference']` for context-bound
+   *  entries when preferences are already injected in a separate block).
+   *  Mutually exclusive with `categories` — if both are provided,
+   *  `categories` (include) wins. */
+  excludeCategories?: (KnowledgeCategory | (string & {}))[];
+};
+
 /**
  * Build a relevance-ranked, budget-capped list of knowledge entries for injection
  * into the system prompt of a live session.
@@ -362,43 +382,61 @@ function scoreEntriesFTS(sessionContext: string): Map<string, number> {
  * Strategy:
  * 1. Both project-specific and cross-project entries are scored for relevance
  *    against recent session context (last distillation + recent raw messages).
- * 2. Project entries get a safety net: the top PROJECT_SAFETY_NET entries by
+ * 2. When embeddings are available, vector cosine similarity is used for scoring
+ *    (captures semantic matches that keyword overlap misses). Falls back to
+ *    FTS5 BM25 when embeddings are unavailable.
+ * 3. Project entries get a safety net: the top PROJECT_SAFETY_NET entries by
  *    confidence are always included even if they have zero relevance score.
  *    This ensures the most important project knowledge is never lost to
- *    coarse term-overlap scoring.
- * 3. All scored entries are merged into a single pool and greedily packed
+ *    coarse scoring.
+ * 4. All scored entries are merged into a single pool and greedily packed
  *    into the token budget by score descending.
- * 4. If there's no session context yet (first turn), fall back to top entries
+ * 5. If there's no session context yet (first turn), fall back to top entries
  *    by confidence only (capped at NO_CONTEXT_FALLBACK_CAP per pool).
  *
  * @param projectPath   Current project path
  * @param sessionID     Current session ID (for context extraction)
  * @param maxTokens     Hard token budget for the entire formatted block
+ * @param options       Optional category filter and context hint
  */
-export function forSession(
+export async function forSession(
   projectPath: string,
   sessionID: string | undefined,
   maxTokens: number,
-): KnowledgeEntry[] {
+  options?: ForSessionOptions,
+): Promise<KnowledgeEntry[]> {
   const pid = ensureProject(projectPath);
+  const categoryFilter = options?.categories;
+  const excludeFilter = options?.excludeCategories;
+
+  // Build optional SQL category clauses (include / exclude are mutually exclusive)
+  let categoryClause = "";
+  let categoryParams: string[] = [];
+  if (categoryFilter?.length) {
+    categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
+    categoryParams = categoryFilter;
+  } else if (excludeFilter?.length) {
+    categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
+    categoryParams = excludeFilter;
+  }
 
   // --- 1. Load project-specific entries ---
   const projectEntries = db()
     .query(
       `SELECT ${KNOWLEDGE_COLS} FROM knowledge
-       WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
+       WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
        ORDER BY confidence DESC, updated_at DESC`,
     )
-    .all(pid) as KnowledgeEntry[];
+    .all(pid, ...categoryParams) as KnowledgeEntry[];
 
   // --- 2. Load cross-project candidates ---
   const crossEntries = db()
     .query(
       `SELECT ${KNOWLEDGE_COLS} FROM knowledge
-       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
+       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
        ORDER BY confidence DESC, updated_at DESC`,
     )
-    .all() as KnowledgeEntry[];
+    .all(...categoryParams) as KnowledgeEntry[];
 
   if (!crossEntries.length && !projectEntries.length) return [];
 
@@ -427,38 +465,82 @@ export function forSession(
     }
   }
 
+  // Fall back to caller-provided context hint (e.g., user's first message)
+  if (!sessionContext.trim() && options?.contextHint) {
+    sessionContext = options.contextHint;
+  }
+
   // --- 4. Score both pools by relevance ---
   let scoredProject: Scored[];
   let scoredCross: Scored[];
 
-  if (sessionContext.trim().length > 20) {
-    // Use FTS5 BM25 to score all knowledge entries against session context
-    const ftsScores = scoreEntriesFTS(sessionContext);
-
-    // Score project entries: FTS relevance × confidence, with safety net
-    const rawScored: Scored[] = projectEntries.map((entry) => ({
-      entry,
-      score: (ftsScores.get(entry.id) ?? 0) * entry.confidence,
-    }));
-    const matched = rawScored.filter((s) => s.score > 0);
-    const matchedIds = new Set(matched.map((s) => s.entry.id));
-
-    // Safety net: top PROJECT_SAFETY_NET entries by confidence that weren't already matched.
-    // Given a tiny score (0.001 * confidence) so they sort below genuinely matched entries.
-    const safetyNet = projectEntries
-      .filter((e) => !matchedIds.has(e.id))
-      .slice(0, PROJECT_SAFETY_NET)
-      .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
-
-    scoredProject = [...matched, ...safetyNet];
+  if (sessionContext.trim().length > 20 && embedding.isAvailable()) {
+    // Vector scoring: embed session context, score entries by cosine similarity.
+    // Captures semantic matches (e.g., "OpenAI Batch API" ↔ "batch queue worker")
+    // that keyword-based FTS5 misses.
+    let vectorScores: Map<string, number>;
+    try {
+      const [contextVec] = await embedding.embed([sessionContext], "query");
+      const hits = embedding.vectorSearch(contextVec, 50, excludeFilter);
+      vectorScores = new Map(hits.map((h) => [h.id, h.similarity]));
+    } catch (err) {
+      log.warn("Vector scoring failed, falling back to FTS5:", err);
+      vectorScores = new Map();
+    }
 
-    // Score cross-project entries — only include entries with FTS match
-    scoredCross = crossEntries
-      .filter((e) => ftsScores.has(e.id))
-      .map((e) => ({
-        entry: e,
-        score: (ftsScores.get(e.id) ?? 0) * e.confidence,
-      }));
+    if (vectorScores.size > 0) {
+      // Hybrid scoring: vector search only covers entries with stored embeddings.
+      // Entries without embeddings (e.g. newly created, async embed not yet done)
+      // fall back to FTS5 so they aren't invisible to scoring.
+      const ftsScores = scoreEntriesFTS(sessionContext);
+
+      // Score project entries: prefer vector similarity, fall back to FTS5
+      const rawScored: Scored[] = projectEntries.map((entry) => {
+        const vecScore = vectorScores.get(entry.id);
+        const score = vecScore != null
+          ? vecScore * entry.confidence
+          : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
+        return { entry, score };
+      });
+      const matched = rawScored.filter((s) => s.score > 0);
+      const matchedIds = new Set(matched.map((s) => s.entry.id));
+
+      // Safety net: top PROJECT_SAFETY_NET entries by confidence that weren't already matched.
+      // Given a tiny score (0.001 * confidence) so they sort below genuinely matched entries.
+      const safetyNet = projectEntries
+        .filter((e) => !matchedIds.has(e.id))
+        .slice(0, PROJECT_SAFETY_NET)
+        .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
+
+      scoredProject = [...matched, ...safetyNet];
+
+      // Cross-project: include entries matched by vector OR FTS5
+      scoredCross = crossEntries
+        .filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id))
+        .map((e) => {
+          const vecScore = vectorScores.get(e.id);
+          const score = vecScore != null
+            ? vecScore * e.confidence
+            : (ftsScores.get(e.id) ?? 0) * e.confidence;
+          return { entry: e, score };
+        });
+    } else {
+      // Vector failed — fall through to FTS5
+      const ftsScores = scoreEntriesFTS(sessionContext);
+      ({ scoredProject, scoredCross } = scoreFTS(
+        projectEntries,
+        crossEntries,
+        ftsScores,
+      ));
+    }
+  } else if (sessionContext.trim().length > 20) {
+    // Embeddings unavailable — use FTS5 BM25 as fallback
+    const ftsScores = scoreEntriesFTS(sessionContext);
+    ({ scoredProject, scoredCross } = scoreFTS(
+      projectEntries,
+      crossEntries,
+      ftsScores,
+    ));
   } else {
     // No session context — fall back to top entries by confidence, capped
     scoredProject = projectEntries
@@ -520,6 +602,36 @@ export function forSession(
   return result;
 }
 
+/** Score entries using FTS5 BM25 — extracted for reuse in the vector-fallback path. */
+function scoreFTS(
+  projectEntries: KnowledgeEntry[],
+  crossEntries: KnowledgeEntry[],
+  ftsScores: Map<string, number>,
+): { scoredProject: Scored[]; scoredCross: Scored[] } {
+  const rawScored: Scored[] = projectEntries.map((entry) => ({
+    entry,
+    score: (ftsScores.get(entry.id) ?? 0) * entry.confidence,
+  }));
+  const matched = rawScored.filter((s) => s.score > 0);
+  const matchedIds = new Set(matched.map((s) => s.entry.id));
+
+  const safetyNet = projectEntries
+    .filter((e) => !matchedIds.has(e.id))
+    .slice(0, PROJECT_SAFETY_NET)
+    .map((e) => ({ entry: e, score: 0.001 * e.confidence }));
+
+  const scoredProject = [...matched, ...safetyNet];
+
+  const scoredCross = crossEntries
+    .filter((e) => ftsScores.has(e.id))
+    .map((e) => ({
+      entry: e,
+      score: (ftsScores.get(e.id) ?? 0) * e.confidence,
+    }));
+
+  return { scoredProject, scoredCross };
+}
+
 export function all(): KnowledgeEntry[] {
   return db()
     .query(
@@ -963,9 +1075,18 @@ export type DedupCluster = {
   merged: Array<{ id: string; title: string }>;
 };
 
+/** Stable pair key for two entry IDs — sorted to ensure order-independence. */
+export function dedupPairKey(idA: string, idB: string): string {
+  return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
+}
+
 export type DedupResult = {
   clusters: DedupCluster[];
   totalRemoved: number;
+  /** Pairwise embedding cosine similarities. Key: dedupPairKey(idA, idB). */
+  pairSimilarities: Map<string, number>;
+  /** All entry titles by ID — for feedback recording after entries are deleted. */
+  entryTitles: Map<string, string>;
 };
 
 /**
@@ -992,13 +1113,17 @@ export type DedupResult = {
  * @returns             Cluster report and count of removed entries
  */
 /** Core dedup logic — operates on an arbitrary list of entries. */
-function _dedup(entries: KnowledgeEntry[], dryRun: boolean): DedupResult {
-  if (entries.length < 2) return { clusters: [], totalRemoved: 0 };
+function _dedup(
+  entries: KnowledgeEntry[],
+  dryRun: boolean,
+  embeddingThreshold: number = EMBEDDING_DEDUP_THRESHOLD,
+): DedupResult {
+  if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: new Map(), entryTitles: new Map() };
 
   // --- Build neighbor map using title overlap + embedding similarity ---
   // Two entries are considered neighbors (potential duplicates) if EITHER:
   //   (a) title word-overlap ≥ 0.7 with ≥ 4 shared words, OR
-  //   (b) embedding cosine similarity ≥ 0.935
+  //   (b) embedding cosine similarity ≥ embeddingThreshold (default 0.935)
   // Star clustering (no transitivity) prevents snowball merging.
   // O(n²) pairwise comparison — acceptable for n ≤ 25 (maxEntries cap).
 
@@ -1026,6 +1151,8 @@ function _dedup(entries: KnowledgeEntry[], dryRun: boolean): DedupResult {
   // Pre-compute neighbors for all pairs
   type DedupHit = { id: string; score: number };
   const neighborMap = new Map<string, DedupHit[]>();
+  // Collect all pairwise embedding similarities (for feedback/calibration).
+  const pairSimilarities = new Map<string, number>();
 
   for (const entry of entries) {
     const neighbors: DedupHit[] = [];
@@ -1045,7 +1172,15 @@ function _dedup(entries: KnowledgeEntry[], dryRun: boolean): DedupResult {
         const otherVec = embeddingMap.get(other.id);
         if (otherVec && entryVec.length === otherVec.length) {
           similarity = embedding.cosineSimilarity(entryVec, otherVec);
-          embeddingMatch = similarity >= EMBEDDING_DEDUP_THRESHOLD;
+          embeddingMatch = similarity >= embeddingThreshold;
+        }
+      }
+
+      // Track all pairwise embedding similarities for calibration signals
+      if (similarity > 0) {
+        const pk = dedupPairKey(entry.id, other.id);
+        if (!pairSimilarities.has(pk)) {
+          pairSimilarities.set(pk, similarity);
         }
       }
 
@@ -1120,21 +1255,27 @@ function _dedup(entries: KnowledgeEntry[], dryRun: boolean): DedupResult {
   // Sort clusters by size descending for readability
   result.sort((a, b) => b.merged.length - a.merged.length);
 
-  return { clusters: result, totalRemoved };
+  // Build title map from all input entries — survives entry deletion.
+  const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
+
+  return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
 }
 
 export async function deduplicate(
   projectPath: string,
   opts?: { dryRun?: boolean },
 ): Promise<DedupResult> {
+  const pid = ensureProject(projectPath);
+  const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
   const entries = forProject(projectPath, false);
-  return _dedup(entries, opts?.dryRun ?? true);
+  return _dedup(entries, opts?.dryRun ?? true, threshold);
 }
 
 /** Deduplicate global (cross-project) entries that have no project_id. */
 export async function deduplicateGlobal(
   opts?: { dryRun?: boolean },
 ): Promise<DedupResult> {
+  const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
   const entries = db()
     .query(
       `SELECT ${KNOWLEDGE_COLS} FROM knowledge
@@ -1143,5 +1284,299 @@ export async function deduplicateGlobal(
        ORDER BY confidence DESC, updated_at DESC`,
     )
     .all() as KnowledgeEntry[];
-  return _dedup(entries, opts?.dryRun ?? true);
+  return _dedup(entries, opts?.dryRun ?? true, threshold);
+}
+
+// ---------------------------------------------------------------------------
+// Dedup feedback & adaptive threshold calibration
+// ---------------------------------------------------------------------------
+
+export type DedupFeedbackSource = "auto_dedup" | "cli_yes" | "cli_interactive";
+
+const MIN_CALIBRATION_SAMPLES = 20;
+const DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
+/** Only record auto-signals for pairs with similarity >= this floor. */
+const AUTO_SIGNAL_MIN_SIMILARITY = 0.80;
+/** Max auto-signal pairs to record per dedup run (closest to threshold). */
+const AUTO_SIGNAL_MAX_PAIRS = 50;
+
+/** Record a single dedup feedback row. */
+export function recordDedupFeedback(input: {
+  projectId: string | null;
+  entryATitle: string;
+  entryBTitle: string;
+  similarity: number;
+  accepted: boolean;
+  source: DedupFeedbackSource;
+}): void {
+  db()
+    .query(
+      `INSERT INTO dedup_feedback
+         (project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      input.projectId,
+      input.entryATitle,
+      input.entryBTitle,
+      input.similarity,
+      input.accepted ? 1 : 0,
+      input.source,
+      Date.now(),
+    );
+}
+
+/**
+ * Bulk-record feedback for all merged pairs in a DedupResult.
+ * Only records pairs with embedding similarity > 0 (title-overlap-only
+ * matches are excluded from calibration).
+ */
+export function recordDedupResultFeedback(
+  projectId: string | null,
+  result: DedupResult,
+  accepted: boolean,
+  source: DedupFeedbackSource,
+): void {
+  for (const cluster of result.clusters) {
+    for (const merged of cluster.merged) {
+      const pk = dedupPairKey(cluster.surviving.id, merged.id);
+      const similarity = result.pairSimilarities.get(pk);
+      if (similarity != null && similarity > 0) {
+        recordDedupFeedback({
+          projectId,
+          entryATitle: cluster.surviving.title,
+          entryBTitle: merged.title,
+          similarity,
+          accepted,
+          source,
+        });
+      }
+    }
+  }
+}
+
+/**
+ * Record automatic calibration signals from a post-curation dedup sweep.
+ *
+ * Only records **reject** signals — non-merged pairs with similarity in
+ * [0.80, threshold). Accept signals from auto-dedup are tautological (the
+ * pair was merged *because* its similarity exceeded the threshold), so they
+ * provide no new information and would create a self-reinforcing feedback
+ * loop. Manual signals (cli_yes, cli_interactive) provide the accept side.
+ *
+ * Caps at AUTO_SIGNAL_MAX_PAIRS most interesting pairs per run (closest
+ * to the threshold boundary) to avoid table bloat.
+ */
+export function recordAutoSignals(
+  projectId: string | null,
+  result: DedupResult,
+): void {
+  // Collect merged pair IDs for quick lookup (to exclude from reject signals)
+  const mergedPairs = new Set<string>();
+  for (const cluster of result.clusters) {
+    for (const merged of cluster.merged) {
+      mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
+    }
+  }
+
+  // Build a title map — we need titles for reject signals (non-merged pairs).
+  // Use entryTitles from result first, then fall back to cluster data.
+  const titleMap = new Map<string, string>(result.entryTitles);
+  for (const cluster of result.clusters) {
+    if (!titleMap.has(cluster.surviving.id)) {
+      titleMap.set(cluster.surviving.id, cluster.surviving.title);
+    }
+    for (const m of cluster.merged) {
+      if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
+    }
+  }
+
+  // Collect reject signals: non-merged pairs with high similarity
+  type Signal = { entryATitle: string; entryBTitle: string; similarity: number };
+  const signals: Signal[] = [];
+
+  for (const [pk, sim] of result.pairSimilarities) {
+    if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
+    if (mergedPairs.has(pk)) continue; // merged pair — skip (tautological accept)
+
+    const [idA, idB] = pk.split(":");
+    const titleA = titleMap.get(idA);
+    const titleB = titleMap.get(idB);
+    if (!titleA || !titleB) continue;
+
+    signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
+  }
+
+  // Sort by distance to threshold boundary (most informative first), cap
+  const currentThreshold = loadCalibratedThreshold(projectId) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
+  signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
+  const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
+
+  // Prune old feedback to prevent unbounded table growth
+  pruneDedupFeedback(projectId);
+
+  for (const s of capped) {
+    recordDedupFeedback({
+      projectId,
+      entryATitle: s.entryATitle,
+      entryBTitle: s.entryBTitle,
+      similarity: s.similarity,
+      accepted: false,
+      source: "auto_dedup",
+    });
+  }
+}
+
+/** Get all feedback for a project (for calibration). */
+export function getDedupFeedback(
+  projectId: string | null,
+): Array<{ similarity: number; accepted: boolean; source: string }> {
+  const rows = (
+    projectId !== null
+      ? db()
+          .query(
+            "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity",
+          )
+          .all(projectId)
+      : db()
+          .query(
+            "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity",
+          )
+          .all()
+  ) as Array<{ similarity: number; accepted: number; source: string }>;
+  return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
+}
+
+/** Quick count of feedback rows for a project. */
+export function getDedupFeedbackCount(projectId: string | null): number {
+  const row = (
+    projectId !== null
+      ? db()
+          .query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?")
+          .get(projectId)
+      : db()
+          .query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL")
+          .get()
+  ) as { cnt: number } | null;
+  return row?.cnt ?? 0;
+}
+
+/** Max feedback rows to keep per project (prevents unbounded growth). */
+const MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
+
+/**
+ * Prune old feedback rows for a project, keeping the most recent
+ * MAX_FEEDBACK_ROWS_PER_PROJECT rows. Called from recordAutoSignals
+ * to prevent unbounded table growth.
+ */
+export function pruneDedupFeedback(projectId: string | null): void {
+  const count = getDedupFeedbackCount(projectId);
+  if (count <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
+
+  const excess = count - MAX_FEEDBACK_ROWS_PER_PROJECT;
+  if (projectId !== null) {
+    db()
+      .query(
+        `DELETE FROM dedup_feedback WHERE id IN (
+           SELECT id FROM dedup_feedback WHERE project_id = ?
+           ORDER BY created_at ASC LIMIT ?
+         )`,
+      )
+      .run(projectId, excess);
+  } else {
+    db()
+      .query(
+        `DELETE FROM dedup_feedback WHERE id IN (
+           SELECT id FROM dedup_feedback WHERE project_id IS NULL
+           ORDER BY created_at ASC LIMIT ?
+         )`,
+      )
+      .run(excess);
+  }
+}
+
+/**
+ * Compute an optimal embedding dedup threshold from user feedback.
+ *
+ * Algorithm:
+ * 1. Load all (similarity, accepted) pairs for the project.
+ * 2. If fewer than MIN_CALIBRATION_SAMPLES, return null (use default).
+ * 3. If all feedback is "accept" (no rejects), return the minimum
+ *    accepted similarity minus a small margin (0.005).
+ * 4. If all feedback is "reject" (no accepts), return null.
+ * 5. Otherwise, find the threshold that maximizes separation:
+ *    - For each candidate threshold (midpoint between consecutive
+ *      distinct similarity values), compute accuracy:
+ *        correct = accepted_pairs_above + rejected_pairs_below
+ *        accuracy = correct / total
+ *    - Pick the threshold with highest accuracy.
+ *    - Tie-break: prefer higher threshold (conservative).
+ *    - Clamp to [0.85, 0.98].
+ */
+export function calibrateDedupThreshold(projectId: string | null): number | null {
+  const feedback = getDedupFeedback(projectId);
+  if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
+
+  const accepted = feedback.filter((f) => f.accepted);
+  const rejected = feedback.filter((f) => !f.accepted);
+
+  // Edge case: all accept, no rejects
+  if (rejected.length === 0) {
+    const minAccepted = Math.min(...accepted.map((f) => f.similarity));
+    return Math.max(0.85, minAccepted - 0.005);
+  }
+
+  // Edge case: all reject, no accepts
+  if (accepted.length === 0) {
+    log.warn("dedup calibration: all feedback is reject — keeping default threshold");
+    return null;
+  }
+
+  // Find optimal threshold via accuracy maximization
+  const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
+
+  let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
+  let bestAccuracy = -1;
+
+  for (let i = 0; i < allSims.length - 1; i++) {
+    const candidate = (allSims[i] + allSims[i + 1]) / 2;
+
+    // Pairs above threshold are predicted "merge" — should be accepted
+    // Pairs below threshold are predicted "keep separate" — should be rejected
+    const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
+    const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
+    const accuracy = (correctAccepted + correctRejected) / feedback.length;
+
+    // Tie-break: prefer higher threshold (conservative — fewer false merges)
+    if (accuracy > bestAccuracy || (accuracy === bestAccuracy && candidate > bestThreshold)) {
+      bestAccuracy = accuracy;
+      bestThreshold = candidate;
+    }
+  }
+
+  // Clamp to sane range
+  return Math.max(0.85, Math.min(0.98, bestThreshold));
+}
+
+/** Persist the calibrated threshold for a project. */
+export function saveCalibratedThreshold(
+  projectId: string | null,
+  threshold: number,
+  sampleSize: number,
+): void {
+  const key = `dedup_threshold:${projectId ?? "global"}`;
+  setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
+}
+
+/** Load the calibrated threshold for a project, or null if not calibrated. */
+export function loadCalibratedThreshold(projectId: string | null): number | null {
+  const key = `dedup_threshold:${projectId ?? "global"}`;
+  const raw = getKV(key);
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw);
+    return typeof parsed.threshold === "number" ? parsed.threshold : null;
+  } catch {
+    return null;
+  }
 }

package/src/session-limiter.ts

@@ -0,0 +1,47 @@
+/**
+ * Per-key concurrency limiter using p-limit.
+ *
+ * Each key (typically a session ID) gets its own p-limit(1) instance,
+ * serializing async operations on the same key while allowing different
+ * keys to run fully in parallel.
+ *
+ * Two independent limiter pools are provided — one for distillation and
+ * one for curation — so they don't block each other.
+ */
+
+import pLimit from "p-limit";
+
+type LimitFunction = ReturnType<typeof pLimit>;
+
+function createLimiterPool() {
+  const limiters = new Map<string, LimitFunction>();
+
+  /** Get or create a p-limit(1) limiter for the given key. */
+  function get(key: string): LimitFunction {
+    let limiter = limiters.get(key);
+    if (!limiter) {
+      limiter = pLimit(1);
+      limiters.set(key, limiter);
+    }
+    return limiter;
+  }
+
+  /** Check if a limiter for `key` is currently busy (active or pending work). */
+  function isBusy(key: string): boolean {
+    const limiter = limiters.get(key);
+    return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
+  }
+
+  /** Clear all limiters (for test cleanup). */
+  function clear(): void {
+    limiters.clear();
+  }
+
+  return { get, isBusy, clear };
+}
+
+/** Serializes distillation.run() and metaDistill() per session. */
+export const distillLimiter = createLimiterPool();
+
+/** Serializes curator.run() per session with skip-if-busy semantics. */
+export const curatorLimiter = createLimiterPool();