@rohirik/openltm-core 2.12.3 → 2.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rohirik/openltm-core",
-  "version": "2.12.3",
+  "version": "2.13.0",
   "description": "Shared LTM storage engine — path-agnostic SQLite core used by Claude Code, OpenCode, and Pi adapters",
   "type": "module",
   "main": "./src/index.ts",

package/src/__tests__/quality.test.ts

@@ -0,0 +1,251 @@
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { Database } from "bun:sqlite";
+import { readFileSync, unlinkSync } from "fs";
+import { join } from "path";
+
+const dbPath = `/tmp/test-openltm-quality-${process.pid}-${Date.now()}.db`;
+const SCHEMA_PATH = join(import.meta.dir, "..", "schema.sql");
+
+type Core = typeof import("../index.js");
+let core: Core;
+
+beforeAll(async () => {
+  const mod = await import("../index.js");
+  core = mod;
+  const db = new Database(dbPath, { create: true });
+  db.exec("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;");
+  db.exec(readFileSync(SCHEMA_PATH, "utf-8"));
+  await mod.runPendingMigrations(db);
+  mod._setDbForTesting(db);
+}, 30_000);
+
+afterAll(() => {
+  try { unlinkSync(dbPath); } catch {}
+  try { unlinkSync(`${dbPath}-shm`); } catch {}
+  try { unlinkSync(`${dbPath}-wal`); } catch {}
+});
+
+describe("similarity — isNearDuplicate", () => {
+  it("treats an elaboration as a duplicate", () => {
+    expect(
+      core.isNearDuplicate(
+        "docker hub rate limits unauthenticated pulls",
+        "docker hub rate limits unauthenticated pulls at 100 per 6 hours",
+      ),
+    ).toBe(true);
+  });
+
+  it("does not merge deliberate siblings that differ by a word", () => {
+    expect(
+      core.isNearDuplicate("Phase4 high importance sort check", "Phase4 low importance sort check"),
+    ).toBe(false);
+  });
+
+  it("does not merge memories differing only by a short marker", () => {
+    expect(core.isNearDuplicate("concurrent audit 123 -A", "concurrent audit 123 -B")).toBe(false);
+  });
+
+  it("does not merge unrelated content", () => {
+    expect(core.isNearDuplicate("docker hub rate limits", "use bun instead of npm")).toBe(false);
+  });
+
+  it("is a no-op on very short strings", () => {
+    expect(core.isNearDuplicate("ok", "okay")).toBe(false);
+  });
+});
+
+describe("hygiene — isOperationalNoise", () => {
+  it("flags runtime chatter", () => {
+    expect(core.isOperationalNoise("ok")).toBe(true);
+    expect(core.isOperationalNoise("done")).toBe(true);
+    expect(core.isOperationalNoise("process took 320ms")).toBe(true);
+    expect(core.isOperationalNoise("exit status 0")).toBe(true);
+  });
+
+  it("keeps anything that states a durable rule", () => {
+    expect(core.isOperationalNoise("Always use bun, never npm, for scripts")).toBe(false);
+    expect(core.isOperationalNoise("Avoid running migrations on the live host")).toBe(false);
+  });
+
+  it("keeps normal knowledge that merely looks operational", () => {
+    expect(
+      core.isOperationalNoise("The graph server batches debounced WAL writes every 3 seconds"),
+    ).toBe(false);
+  });
+});
+
+describe("hygiene — learn() noise downgrade", () => {
+  it("clamps importance for operational noise but still stores it", () => {
+    const result = core.learn({ content: "background process completed", category: "pattern", importance: 5, skipExport: true });
+    expect(result.action).toBe("created");
+    const row = core.getDb()
+      .query<{ importance: number }, [number]>("SELECT importance FROM memories WHERE id=?")
+      .get(result.id);
+    expect(row?.importance).toBeLessThanOrEqual(2);
+  });
+});
+
+describe("hygiene — near-duplicate reinforcement", () => {
+  it("reinforces an elaboration instead of creating a second row", () => {
+    const first = core.learn({
+      content: "Infisical holds the homelab database credentials for every service",
+      category: "architecture",
+      importance: 4,
+      project_scope: "hygiene-project",
+      skipExport: true,
+    });
+    const second = core.learn({
+      content: "Infisical holds the homelab database credentials for every service including Traefik",
+      category: "architecture",
+      importance: 4,
+      project_scope: "hygiene-project",
+      skipExport: true,
+    });
+    expect(second.action).toBe("reinforced");
+    expect(second.id).toBe(first.id);
+  });
+
+  it("keeps parallel siblings separate", () => {
+    const a = core.learn({ content: "rank probe alpha variant", category: "pattern", importance: 3, skipExport: true });
+    const b = core.learn({ content: "rank probe beta variant", category: "pattern", importance: 3, skipExport: true });
+    expect(b.action).toBe("created");
+    expect(b.id).not.toBe(a.id);
+  });
+});
+
+describe("recall v2 — ranking", () => {
+  it("ranks a project-scoped memory above a stronger global", () => {
+    const global = core.learn({
+      content: "ranking scoped preference — always verify the compose file before deploy",
+      category: "preference",
+      importance: 5,
+      skipExport: true,
+    });
+    const scoped = core.learn({
+      content: "ranking scoped preference for this project only",
+      category: "preference",
+      importance: 3,
+      project_scope: "ranking-project",
+      skipExport: true,
+    });
+
+    const rows = [global.id, scoped.id].map((id) =>
+      core.getDb()
+        .query<never, [number]>("SELECT * FROM memories WHERE id=?")
+        .get(id),
+    );
+    const ranked = core.rankRecallResults(rows as never[], { limit: 2, project: "ranking-project" });
+    expect(ranked[0]!.id).toBe(scoped.id);
+  });
+
+  it("demotes a stale memory without dropping it", () => {
+    const fresh = { id: 1, importance: 3, recall_count: 0, decay_score: 1, stale_flagged_at: null } as never;
+    const stale = { id: 2, importance: 3, recall_count: 0, decay_score: 1, stale_flagged_at: "2026-01-01" } as never;
+    const ranked = core.rankRecallResults([stale, fresh], { limit: 2 });
+    expect(ranked.map((m) => m.id)).toEqual([1, 2]);
+  });
+
+  it("is deterministic for identical input", () => {
+    const rows = [
+      { id: 3, importance: 2, recall_count: 1, decay_score: 0.5, stale_flagged_at: null },
+      { id: 1, importance: 2, recall_count: 1, decay_score: 0.5, stale_flagged_at: null },
+      { id: 2, importance: 2, recall_count: 1, decay_score: 0.5, stale_flagged_at: null },
+    ] as never[];
+    const first = core.rankRecallResults(rows, { limit: 3 }).map((m) => m.id);
+    const second = core.rankRecallResults([...rows].reverse(), { limit: 3 }).map((m) => m.id);
+    expect(first).toEqual(second);
+    expect(first).toEqual([1, 2, 3]);
+  });
+
+  it("progressively demotes near-duplicate clusters without dropping them", () => {
+    const base = { importance: 3, recall_count: 0, decay_score: 1, stale_flagged_at: null, project_scope: null };
+    const rows = [
+      { ...base, id: 1, content: "alpha duplicate probe entry" },
+      { ...base, id: 2, content: "alpha duplicate probe entry for the deploy path" },
+      { ...base, id: 3, content: "completely separate unrelated statement about typography" },
+    ] as never[];
+    const ranked = core.rankRecallResults(rows, { limit: 3 }).map((m) => m.id);
+    // The near-duplicate (2) is demoted below the unrelated memory (3) but is
+    // still returned, and the cluster's best member (1) stays on top.
+    expect(ranked).toEqual([1, 3, 2]);
+  });
+
+  it("honours explicit sort_by requests", () => {
+    const rows = [
+      { id: 1, created_at: "2026-01-01", importance: 3, recall_count: 0, decay_score: 1, stale_flagged_at: null },
+      { id: 2, created_at: "2026-06-01", importance: 3, recall_count: 0, decay_score: 1, stale_flagged_at: null },
+    ] as never[];
+    expect(core.rankRecallResults(rows, { limit: 2, sortBy: "created" }).map((m) => m.id)).toEqual([2, 1]);
+  });
+});
+
+describe("prefill v2 — quotas and dedupe", () => {
+  it("respects per-category quotas and fills leftover slots", () => {
+    for (let i = 0; i < 6; i++) {
+      core.learn({
+        content: `quota gotcha number ${i} about the staging deploy pipeline failing intermittently`,
+        category: "gotcha",
+        importance: 3,
+        project_scope: "quota-project",
+        skipExport: true,
+      });
+    }
+    core.learn({
+      content: "quota decision: staging deploys are frozen on fridays",
+      category: "architecture",
+      importance: 3,
+      project_scope: "quota-project",
+      skipExport: true,
+    });
+
+    const { scoped, globals, report } = core.selectPrefillMemories("quota-project", {
+      maxMemories: 4,
+      quotas: { gotcha: 1, architecture: 1 },
+    });
+
+    const categories = scoped.map((m) => m.category);
+    expect(categories.filter((c) => c === "gotcha").length).toBeLessThanOrEqual(2); // 1 quota + 1 fill
+    expect(categories).toContain("architecture");
+    expect(scoped.length).toBeLessThanOrEqual(4);
+    expect(report.selected).toBe(scoped.length + globals.length);
+  });
+
+  it("suppresses near-duplicate entries from the block", () => {
+    core.learn({
+      content: "dedupe probe: the wiki docs are generated and must never be hand edited",
+      category: "constraint",
+      importance: 4,
+      project_scope: "dedupe-project",
+      skipExport: true,
+    });
+    core.learn({
+      content: "dedupe probe: the wiki docs are generated and must never be hand edited at all",
+      category: "constraint",
+      importance: 4,
+      project_scope: "dedupe-project",
+      skipExport: true,
+    });
+
+    const { scoped, report } = core.selectPrefillMemories("dedupe-project", { maxMemories: 6 });
+    const texts = scoped.map((m) => m.content);
+    expect(new Set(texts).size).toBe(texts.length);
+    expect(report.suppressedDuplicates).toBeGreaterThanOrEqual(0);
+  });
+
+  it("puts the project section before the global section", () => {
+    const block = core.buildPrefillContext({
+      project: "order-project",
+      maxMemories: 6,
+      maxLines: 24,
+    });
+    const projectIdx = block.indexOf("Project (order-project):");
+    const globalIdx = block.indexOf("Global:");
+    if (projectIdx !== -1 && globalIdx !== -1) {
+      expect(projectIdx).toBeLessThan(globalIdx);
+    }
+  });
+
+  it("respects the shared host budget", () => {
+    expect(core.PREFILL_DEFAULTS).toEqual({ maxMemories: 10, maxLines: 18 });
+  });
+});

package/src/cli/hook.ts

@@ -5,7 +5,7 @@
  * openltm-core so users still get an "already pre-filled" experience without
  * the full Claude plugin checkout. Other hook events are safe no-ops.
  */
-import { buildPrefillContext, deriveProjectFromCwd } from "../prefill.js";
+import { buildPrefillContext, deriveProjectFromCwd, PREFILL_DEFAULTS } from "../prefill.js";
 
 function parseHookCwd(raw: string): string {
   if (!raw.trim()) return "";
@@ -27,7 +27,7 @@ export async function buildHookOutput(name: string, rawInput: string): Promise<s
       if (!cwd) return "";
       const project = deriveProjectFromCwd(cwd);
       if (!project) return "";
-      return buildPrefillContext({ project, maxMemories: 10, maxLines: 18 });
+      return buildPrefillContext({ project, ...PREFILL_DEFAULTS });
     }
     case "PreCompact":
     case "PostEditCheck":

package/src/db.ts

@@ -6,6 +6,7 @@ import type { Database } from "bun:sqlite";
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { join } from "path";
 import { normalizeKey } from "./dedup.js";
+import { isNearDuplicate } from "./similarity.js";
 import { normalizeAnchorPaths } from "./anchors.js";
 import { getDb, DB_PATH, configure as configureDb } from "./shared-db.js";
 import { enqueueEmbedding } from "./queue/index.js";
@@ -140,6 +141,32 @@ function tryAudit(fn: () => void): void {
   try { fn(); } catch (e) { process.stderr.write(`[audit] write failed: ${e}\n`); }
 }
 
+function oneLineForLog(text: string): string {
+  const line = text.replace(/\s+/g, " ").trim();
+  return line.length > 80 ? `${line.slice(0, 77)}…` : line;
+}
+
+/**
+ * findNearDuplicate — locate an existing memory that says the same thing.
+ * Scoped to the same project scope (a global memory never absorbs a scoped one)
+ * and only considers recently active rows to keep the probe cheap.
+ */
+function findNearDuplicate(
+  db: Database,
+  content: string,
+  projectScope: string | null,
+  dedupKey: string,
+): Memory | null {
+  const rows = db.query<Memory, [string, string | null, string | null]>(
+    `SELECT * FROM memories
+      WHERE status='active' AND dedup_key<>?
+        AND (project_scope IS ? OR project_scope = ?)
+      ORDER BY decay_score DESC, id DESC
+      LIMIT 100`
+  ).all(dedupKey, projectScope, projectScope);
+  return rows.find((row) => isNearDuplicate(row.content, content)) ?? null;
+}
+
 function upsertTag(db: Database, name: string): number {
   db.run(`INSERT OR IGNORE INTO tags (name) VALUES (?)`, [name]);
   return db.query<{ id: number }, [string]>(`SELECT id FROM tags WHERE name=?`).get(name)!.id;
@@ -410,6 +437,123 @@ async function autoDetectRelations(
   }
 }
 
+// ── Recall ranking (Recall v2) ────────────────────────────────────────────────
+
+/** Weights for the composite relevance score. Kept in one place so the
+ *  explainer and the sort can never drift apart. */
+export const RANK_WEIGHTS = {
+  decay: 1.0,
+  importance: 0.6,
+  projectScope: 0.9,
+  recallFrequency: 0.25,
+  stalePenalty: 0.8,
+  duplicatePenalty: 0.5,
+} as const;
+
+/** Penalty applied to the k-th occurrence of a near-duplicate cluster. */
+const DUPLICATE_DECAY_FACTOR = 0.4;
+
+/**
+ * rankRecallResults — order recall candidates for relevance.
+ *
+ * Behaviour:
+ *   - explicit `sort_by` requests are honoured verbatim
+ *   - project-scoped memories outrank globals of otherwise equal strength
+ *   - stale (code-invalidated) memories are demoted but still returned
+ *   - near-duplicate clusters are demoted progressively, never dropped
+ *   - ties break on ascending id so identical inputs give identical output
+ */
+export function rankRecallResults(
+  candidates: Memory[],
+  opts: {
+    limit: number;
+    project?: string;
+    defaultSort?: boolean;
+    sortBy?: "relevance" | "created" | "last_recalled" | "recall_count";
+  },
+): Memory[] {
+  const rows = [...candidates];
+
+  // Explicit user-requested sorts stay literal.
+  if (opts.sortBy === "created") {
+    return rows
+      .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() || a.id - b.id)
+      .slice(0, opts.limit);
+  }
+  if (opts.sortBy === "last_recalled") {
+    return rows
+      .sort((a, b) => new Date(b.last_recalled_at ?? "1970").getTime() - new Date(a.last_recalled_at ?? "1970").getTime() || a.id - b.id)
+      .slice(0, opts.limit);
+  }
+  if (opts.sortBy === "recall_count") {
+    return rows
+      .sort((a, b) => (b.recall_count ?? 0) - (a.recall_count ?? 0) || a.id - b.id)
+      .slice(0, opts.limit);
+  }
+
+  const scored = rows.map((memory) => {
+    const decay = memory.decay_score ?? computeDecayScore(memory);
+    const importance = memory.importance / 5;
+    const projectScope = opts.project && memory.project_scope === opts.project ? 1 : 0;
+    const recallFrequency = Math.min(1, Math.log2((memory.recall_count ?? 0) + 1) / 4);
+    const stale = memory.stale_flagged_at ? RANK_WEIGHTS.stalePenalty : 0;
+    const score =
+      decay * RANK_WEIGHTS.decay +
+      importance * RANK_WEIGHTS.importance +
+      projectScope * RANK_WEIGHTS.projectScope +
+      recallFrequency * RANK_WEIGHTS.recallFrequency -
+      stale;
+    return { memory, score, decay, stale };
+  });
+
+  scored.sort((a, b) => b.score - a.score || b.decay - a.decay || a.memory.id - b.memory.id);
+
+  // Progressive near-duplicate demotion: first occurrence keeps its rank,
+  // later ones decay by a fixed factor each. Nothing is removed.
+  const seen: Memory[] = [];
+  const adjusted = scored.map((entry) => {
+    const clusterIndex = seen.findIndex((m) => isNearDuplicate(m.content, entry.memory.content));
+    if (clusterIndex === -1) {
+      seen.push(entry.memory);
+      return { ...entry, finalScore: entry.score };
+    }
+    const penalty = Math.min(1, entry.score * RANK_WEIGHTS.duplicatePenalty * Math.pow(DUPLICATE_DECAY_FACTOR, clusterIndex));
+    return { ...entry, finalScore: entry.score - penalty };
+  });
+
+  adjusted.sort((a, b) => b.finalScore - a.finalScore || b.decay - a.decay || a.memory.id - b.memory.id);
+  return adjusted.slice(0, opts.limit).map((entry) => entry.memory);
+}
+
+// ── Memory hygiene (hygiene v2) ───────────────────────────────────────────────
+
+/** Operational noise: runtime chatter, not durable knowledge. */
+const NOISE_PATTERNS: RegExp[] = [
+  /^\s*(?:ok|okay|done|thanks|thank you|sure|got it|understood|acknowledged)\b[.!]?\s*$/i,
+  /\b(?:running|running\.\.\.|in progress|queued|processing)\b\s*$/i,
+  /\btook \d+(?:\.\d+)?\s*m?s\b/i,
+  /\bexit(?:ed)?\s+(?:code|status)\s+\d+\b/i,
+  /\b(?:compacted|compaction)\b.{0,40}\b(?:reference only|summary only)\b/i,
+  /\b(?:background|async|delegation)\b.{0,30}\b(?:process|task|batch|job)?\b.*\b(?:complete|completed|done|finished)\b/i,
+  /\bnotification\b.{0,30}\b(?:background|completed)\b/i,
+  /^\s*[\[\(<{].{0,20}[\]\)>}].{0,40}$/,
+];
+
+/** Noise that is only noise when it carries no durable signal. */
+const DURABLE_SIGNAL_RE = /\b(must|never|always|avoid|prefer|require|constraint|gotcha|decision|instead of|do not|don't)\b/i;
+
+/**
+ * isOperationalNoise — true when text looks like runtime chatter.
+ * A message that still states a rule ("always use bun, not npm") is kept even
+ * if it matches, because the durable signal outweighs the shape.
+ */
+export function isOperationalNoise(content: string): boolean {
+  const text = content.trim();
+  if (text.length < 12) return true;
+  if (DURABLE_SIGNAL_RE.test(text)) return false;
+  return NOISE_PATTERNS.some((pattern) => pattern.test(text));
+}
+
 export function learn(input: LearnInput): LearnResult {
   const db = getDb();
 
@@ -420,10 +564,24 @@ export function learn(input: LearnInput): LearnResult {
   }
   const content = scrubbed;
 
+  // Hygiene: operational noise is downgraded rather than stored verbatim.
+  if (isOperationalNoise(content)) {
+    input = {
+      ...input,
+      importance: Math.min(input.importance ?? 3, 2),
+      category: (input.category ?? "pattern") as MemoryCategory,
+    };
+    if ((input.importance ?? 3) >= 4) {
+      process.stderr.write(`[learn] Downgraded operational-noise memory: "${oneLineForLog(content)}"\n`);
+    }
+  }
+
   const dedupKey = normalizeKey(content);
   const skipExport = input.skipExport ?? false;
 
-  const existing = db.query<Memory, [string]>(`SELECT * FROM memories WHERE dedup_key=?`).get(dedupKey);
+  const existing = db.query<Memory, [string]>(`SELECT * FROM memories WHERE dedup_key=?`).get(dedupKey)
+    // Near-duplicate reinforcement: same knowledge, different wording.
+    ?? findNearDuplicate(db, content, input.project_scope ?? null, dedupKey);
 
   const actor = input.actor ?? "mcp:ltm_learn";
 
@@ -636,37 +794,26 @@ export async function recall(input: RecallInput = {}): Promise<MemoryWithRelatio
   // default sort (no query): ORDER BY decay_score DESC pushed to SQL → O(log N)
   const defaultSqlSort = (!input.sort_by || input.sort_by === "relevance") && ids === null;
   const orderBy = defaultSqlSort ? "ORDER BY decay_score DESC" : "";
-  const rows = db.query<Memory, typeof params>(
-    `SELECT id, content, category, importance, confidence, source, project_scope, dedup_key,
-            created_at, last_confirmed_at, last_used_at, confirm_count, status,
-            first_recalled_at, last_recalled_at, recall_count, superseded_by, superseded_at,
-            workspace_id, agent_id, decay_score, stale_flagged_at, stale_reason
-     FROM memories ${where} ${orderBy} LIMIT ${limit}`
-  ).all(...params);
-
-  let sorted: typeof rows;
-  if (defaultSqlSort) {
-    sorted = rows; // already ordered by decay_score DESC in SQL
-  } else if (input.sort_by === "created") {
-    sorted = rows.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
-  } else if (input.sort_by === "last_recalled") {
-    sorted = rows.sort((a, b) => new Date(b.last_recalled_at ?? "1970").getTime() - new Date(a.last_recalled_at ?? "1970").getTime());
-  } else if (input.sort_by === "recall_count") {
-    sorted = rows.sort((a, b) => (b.recall_count ?? 0) - (a.recall_count ?? 0));
-  } else {
-    // FTS/semantic path: small result set (≤50), JS sort is O(k log k) — negligible.
-    // Use materialised decay_score if available, fall back to computing it.
-    sorted = rows
-      .map(m => ({ m, score: m.decay_score ?? computeDecayScore(m) }))
-      .sort((a, b) => b.score - a.score)
-      .map(({ m }) => m);
-  }
-  // Downrank stale (code-invalidated) memories: stable partition pushes them
-  // after fresh ones at equal relevance — still returned, just demoted.
-  sorted = [
-    ...sorted.filter(m => !m.stale_flagged_at),
-    ...sorted.filter(m => m.stale_flagged_at),
-  ];
+  // Recall v2 ranking inputs need project scope + staleness, so both are
+  // selected up front even on the default sort path.
+  const rankSql = defaultSqlSort
+    ? `SELECT id, content, category, importance, confidence, source, project_scope, dedup_key,
+              created_at, last_confirmed_at, last_used_at, confirm_count, status,
+              first_recalled_at, last_recalled_at, recall_count, superseded_by, superseded_at,
+              workspace_id, agent_id, decay_score, stale_flagged_at, stale_reason
+         FROM memories ${where} ORDER BY decay_score DESC LIMIT ${limit * 3}`
+    : `SELECT id, content, category, importance, confidence, source, project_scope, dedup_key,
+              created_at, last_confirmed_at, last_used_at, confirm_count, status,
+              first_recalled_at, last_recalled_at, recall_count, superseded_by, superseded_at,
+              workspace_id, agent_id, decay_score, stale_flagged_at, stale_reason
+         FROM memories ${where} ${orderBy} LIMIT ${limit * 3}`;
+  const candidateRows = db.query<Memory, typeof params>(rankSql).all(...params);
+  const sorted = rankRecallResults(candidateRows, {
+    limit,
+    project: input.project,
+    defaultSort: defaultSqlSort,
+    sortBy: input.sort_by,
+  });
   if (sorted.length > 0) {
     const placeholders = sorted.map(() => "?").join(",");
     db.run(

package/src/index.ts

@@ -45,7 +45,15 @@ export type { MemoryTemperature, RecallExplainer, ExplainerInput } from "./recal
 
 // Session prefill helpers
 export { buildPrefillContext, deriveProjectFromCwd, selectPrefillMemories } from "./prefill.js";
-export type { PrefillOptions, PrefillSelection } from "./prefill.js";
+export type { PrefillOptions, PrefillSelection, PrefillCategory, PrefillQuotaReport } from "./prefill.js";
+export { PREFILL_DEFAULTS } from "./prefill.js";
+
+// Recall ranking
+export { rankRecallResults, isOperationalNoise } from "./db.js";
+export { RANK_WEIGHTS } from "./db.js";
+
+// Text similarity (prefill dedupe + learn hygiene)
+export { isNearDuplicate, jaccardSimilarity, tokenize, tokenizeAll } from "./similarity.js";
 
 // Embedding providers
 export * from "./providers/index.js";

package/src/prefill.ts

@@ -3,9 +3,16 @@
  *
  * Keeps the "already pre-filled" experience consistent across Claude Code,
  * OpenCode, Pi, and any future host that can call into openltm-core.
+ *
+ * Prefill v2 behaviour:
+ *   - project-scoped memories are prioritised over globals
+ *   - category quotas stop one noisy category from filling the block
+ *   - near-duplicate entries are suppressed
+ *   - the block is hard-capped by line budget for every host
  */
-import { getContextMerge, type Memory } from "./db.js";
+import { getContextMerge, computeDecayScore, type Memory } from "./db.js";
 import { readConfigSync } from "./config.js";
+import { isNearDuplicate } from "./similarity.js";
 
 export interface PrefillOptions {
   project: string;
@@ -13,6 +20,8 @@ export interface PrefillOptions {
   maxGlobalMemories?: number;
   maxLines?: number;
   header?: string;
+  /** Per-category caps applied before the global budget. */
+  quotas?: Partial<Record<PrefillCategory, number>>;
 }
 
 export interface PrefillSelection {
@@ -20,8 +29,53 @@ export interface PrefillSelection {
   scoped: Memory[];
 }
 
+export type PrefillCategory =
+  | "preference"
+  | "architecture"
+  | "gotcha"
+  | "pattern"
+  | "workflow"
+  | "constraint";
+
+export interface PrefillQuotaReport {
+  selected: number;
+  suppressedDuplicates: number;
+  quotaLimited: number;
+}
+
 const DEFAULT_HEADER = "## Prior Knowledge (LTM)";
 
+/**
+ * Shared prefill budget for every host adapter.
+ *
+ * Adapters pass these instead of literal numbers so Claude, OpenCode, and Pi
+ * cannot drift apart: the block a user sees is the same shape everywhere.
+ */
+export const PREFILL_DEFAULTS = { maxMemories: 10, maxLines: 18 } as const;
+
+/**
+ * Category quotas. The block stays useful when a project has 40 gotchas and no
+ * decisions: gotcha is capped and the freed slots go to the other categories.
+ */
+const DEFAULT_QUOTAS: Record<PrefillCategory, number> = {
+  preference: 2,
+  architecture: 2,
+  gotcha: 3,
+  pattern: 3,
+  workflow: 2,
+  constraint: 2,
+};
+
+/** Order used when a category is over its quota and slots are contested. */
+const CATEGORY_PRIORITY: PrefillCategory[] = [
+  "constraint",
+  "preference",
+  "gotcha",
+  "architecture",
+  "workflow",
+  "pattern",
+];
+
 function oneLine(text: string): string {
   return text.replace(/\s+/g, " ").trim();
 }
@@ -31,8 +85,9 @@ function clampPositiveInt(value: number | undefined, fallback: number): number {
   return Math.floor(value);
 }
 
-function formatMemory(memory: Pick<Memory, "id" | "content" | "category" | "importance">): string {
-  return `- [${memory.id}] (${memory.category}/${memory.importance}) ${oneLine(memory.content)}`;
+function clampNonNegativeInt(value: number | undefined, fallback: number): number {
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return fallback;
+  return Math.floor(value);
 }
 
 function trimLines(lines: string[], maxLines: number): string[] {
@@ -46,24 +101,104 @@ export function deriveProjectFromCwd(cwd: string): string {
   return cwd.replace(/\/$/, "").split("/").pop() ?? "";
 }
 
-export function selectPrefillMemories(project: string, opts: Omit<PrefillOptions, "project"> = {}): PrefillSelection {
+/** Rank key: higher is better. Project scope wins ties against globals. */
+function rank(memory: Memory, scopeRank: number): number {
+  return memory.importance * 1000 + computeDecayScore(memory) * 10 + scopeRank;
+}
+
+function sortForSelection(memories: Memory[], scopeRank: number): Memory[] {
+  return [...memories].sort((a, b) => {
+    const diff = rank(b, scopeRank) - rank(a, scopeRank);
+    if (diff !== 0) return diff;
+    return a.id - b.id; // deterministic tie-break
+  });
+}
+
+/**
+ * Apply per-category quotas, then fill any leftover slots with the highest
+ * ranked remaining memories so the block is never artificially small.
+ */
+function applyQuotas(
+  ordered: Memory[],
+  quotas: Partial<Record<PrefillCategory, number>>,
+  budget: number,
+): { picked: Memory[]; quotaLimited: number } {
+  const counts = new Map<PrefillCategory, number>();
+  const picked: Memory[] = [];
+  const deferred: Memory[] = [];
+  let quotaLimited = 0;
+
+  for (const memory of ordered) {
+    if (picked.length >= budget) break;
+    const category = memory.category as PrefillCategory;
+    const cap = quotas[category];
+    const used = counts.get(category) ?? 0;
+    if (cap !== undefined && used >= cap) {
+      quotaLimited++;
+      deferred.push(memory);
+      continue;
+    }
+    counts.set(category, used + 1);
+    picked.push(memory);
+  }
+
+  for (const memory of deferred) {
+    if (picked.length >= budget) break;
+    picked.push(memory);
+  }
+
+  return { picked, quotaLimited };
+}
+
+/** Suppress near-duplicate memories, keeping the highest ranked occurrence. */
+function dedupe(memories: Memory[]): { kept: Memory[]; suppressed: number } {
+  const kept: Memory[] = [];
+  let suppressed = 0;
+  for (const memory of memories) {
+    const duplicate = kept.some((existing) => isNearDuplicate(existing.content, memory.content));
+    if (duplicate) {
+      suppressed++;
+      continue;
+    }
+    kept.push(memory);
+  }
+  return { kept, suppressed };
+}
+
+export function selectPrefillMemories(
+  project: string,
+  opts: Omit<PrefillOptions, "project"> = {},
+): PrefillSelection & { report: PrefillQuotaReport } {
   const cfg = readConfigSync();
   const configuredTopN = cfg.ltm?.injectTopN;
   const maxMemories = clampPositiveInt(opts.maxMemories, configuredTopN ?? 8);
-  const desiredGlobals = clampPositiveInt(opts.maxGlobalMemories, Math.min(3, Math.max(1, Math.ceil(maxMemories / 3))));
 
-  const merged = getContextMerge(project);
-
-  const globals = merged.globals.slice(0, Math.min(desiredGlobals, maxMemories));
-  const remaining = Math.max(0, maxMemories - globals.length);
-  const scoped = merged.scoped.slice(0, remaining);
+  const quotas: Partial<Record<PrefillCategory, number>> = { ...DEFAULT_QUOTAS, ...(opts.quotas ?? {}) };
 
-  if (scoped.length < remaining && globals.length < maxMemories) {
-    const refillGlobals = merged.globals.slice(globals.length, Math.min(merged.globals.length, maxMemories - scoped.length));
-    globals.push(...refillGlobals);
-  }
+  const merged = getContextMerge(project);
 
-  return { globals, scoped };
+  // Project-scoped candidates come first and outrank globals of equal strength.
+  const scopedOrdered = sortForSelection(merged.scoped, 2);
+  const globalCeiling = clampNonNegativeInt(opts.maxGlobalMemories, Math.max(1, Math.ceil(maxMemories / 3)));
+  const globalsOrdered = sortForSelection(merged.globals, 0).slice(0, Math.min(merged.globals.length, globalCeiling + maxMemories));
+
+  const scopedBudget = Math.max(0, maxMemories - Math.min(globalCeiling, merged.globals.length));
+  const scopedResult = applyQuotas(scopedOrdered, quotas, scopedBudget);
+  const globalsResult = applyQuotas(globalsOrdered, quotas, maxMemories - scopedResult.picked.length);
+
+  const { kept: scoped, suppressed: scopedDupes } = dedupe(scopedResult.picked);
+  const { kept: globals, suppressed: globalDupes } = dedupe(globalsResult.picked);
+  const crossDupes = globals.filter((g) => scoped.some((s) => isNearDuplicate(s.content, g.content))).length;
+
+  return {
+    scoped,
+    globals: globals.filter((g) => !scoped.some((s) => isNearDuplicate(s.content, g.content))),
+    report: {
+      selected: scoped.length + globals.length,
+      suppressedDuplicates: scopedDupes + globalDupes + crossDupes,
+      quotaLimited: scopedResult.quotaLimited + globalsResult.quotaLimited,
+    },
+  };
 }
 
 export function buildPrefillContext(opts: PrefillOptions): string {
@@ -73,17 +208,21 @@ export function buildPrefillContext(opts: PrefillOptions): string {
 
   const lines: string[] = [opts.header ?? DEFAULT_HEADER, ""];
 
-  if (globals.length > 0) {
-    lines.push("Global:");
-    for (const memory of globals) lines.push(formatMemory(memory));
+  if (scoped.length > 0) {
+    lines.push(`Project (${opts.project}):`);
+    for (const memory of scoped) lines.push(renderLine(memory));
     lines.push("");
   }
 
-  if (scoped.length > 0) {
-    lines.push(`Project (${opts.project}):`);
-    for (const memory of scoped) lines.push(formatMemory(memory));
+  if (globals.length > 0) {
+    lines.push("Global:");
+    for (const memory of globals) lines.push(renderLine(memory));
     lines.push("");
   }
 
   return trimLines(lines, maxLines).join("\n").trimEnd() + "\n";
 }
+
+function renderLine(memory: Pick<Memory, "id" | "content" | "category" | "importance">): string {
+  return `- [${memory.id}] (${memory.category}/${memory.importance}) ${oneLine(memory.content)}`;
+}

package/src/similarity.ts

@@ -0,0 +1,82 @@
+/**
+ * similarity.ts — Dependency-free text similarity helpers.
+ *
+ * Used by prefill selection (near-duplicate suppression) and learn hygiene
+ * (near-duplicate reinforcement) so both paths agree on "same memory".
+ */
+import { normalizeKey } from "./dedup.js";
+
+const STOPWORDS = new Set([
+  "a", "an", "the", "is", "are", "was", "were", "be", "been", "to", "of", "in", "on",
+  "for", "with", "and", "or", "but", "not", "no", "it", "its", "this", "that", "these",
+  "those", "as", "at", "by", "from", "if", "then", "so", "than", "we", "you", "our",
+]);
+
+/** Tokenize into comparable lowercase content tokens (stopwords removed). */
+export function tokenize(text: string): string[] {
+  if (typeof text !== "string") return [];
+  return normalizeKey(text)
+    .split(" ")
+    .filter((t) => t.length > 1 && !STOPWORDS.has(t));
+}
+
+/** Tokenize without dropping anything — used where a difference matters. */
+export function tokenizeAll(text: string): string[] {
+  if (typeof text !== "string") return [];
+  return normalizeKey(text).split(" ").filter(Boolean);
+}
+
+/** Jaccard token similarity in [0,1]. Empty-vs-empty is 0, not 1. */
+export function jaccardSimilarity(a: string, b: string): number {
+  const ta = new Set(tokenize(a));
+  const tb = new Set(tokenize(b));
+  if (ta.size === 0 || tb.size === 0) return 0;
+  let intersection = 0;
+  for (const token of ta) {
+    if (tb.has(token)) intersection++;
+  }
+  const union = ta.size + tb.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/** Tokens that look like a counter, suffix, or marker rather than content. */
+function isShortMarker(token: string): boolean {
+  return token.length <= 2;
+}
+
+/**
+ * isNearDuplicate — true when two contents describe the same knowledge.
+ *
+ * Deliberately conservative, because this feeds the learn() path where a false
+ * positive silently merges two memories the caller meant to keep apart. Only
+ * one rule qualifies: full token containment, i.e. one memory is an elaboration
+ * of the other. Looser similarity thresholds merged deliberate siblings
+ * ("X high importance" vs "X low importance") and are deliberately absent —
+ * reworded duplicates are left to the embedding-based `autoRelate` path.
+ */
+export function isNearDuplicate(a: string, b: string): boolean {
+  if (typeof a !== "string" || typeof b !== "string") return false;
+  const ta = new Set(tokenize(a));
+  const tb = new Set(tokenize(b));
+  if (ta.size < 3 || tb.size < 3) return false;
+
+  // Symmetric difference is computed on the *unfiltered* token list, because
+  // stopword/short-token filtering would hide exactly the "…-A" vs "…-B"
+  // difference this guard exists to protect.
+  const allA = new Set(tokenizeAll(a));
+  const allB = new Set(tokenizeAll(b));
+  const differing = [
+    ...[...allA].filter((t) => !allB.has(t)),
+    ...[...allB].filter((t) => !allA.has(t)),
+  ];
+
+  // Differing only by short markers (counter/suffix) → intentionally distinct.
+  if (differing.length > 0 && differing.every(isShortMarker)) return false;
+
+  const [smaller, larger] = ta.size <= tb.size ? [ta, tb] : [tb, ta];
+  let contained = 0;
+  for (const token of smaller) {
+    if (larger.has(token)) contained++;
+  }
+  return contained === smaller.size;
+}