opencode-lore 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-lore",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "license": "MIT",
   "description": "Three-tier memory architecture for OpenCode — distillation, not summarization",
@@ -17,6 +17,7 @@
   },
   "dependencies": {
     "remark": "^15.0.1",
+    "uuidv7": "^1.1.0",
     "zod": "^3.25.0"
   },
   "devDependencies": {

package/src/agents-file.ts

@@ -0,0 +1,318 @@
+/**
+ * agents-file.ts — AGENTS.md export/import/sync for lore.
+ *
+ * Lore owns a clearly delimited section inside the file, bounded by HTML
+ * comment markers. Everything outside those markers is preserved verbatim.
+ * Each knowledge entry is preceded by a hidden <!-- lore:UUID --> comment so
+ * the same entry can be tracked across machines and merge conflicts resolved
+ * without duplication.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { dirname } from "path";
+import * as ltm from "./ltm";
+import { formatKnowledge } from "./prompt";
+import { unescapeMarkdown } from "./markdown";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+export const LORE_SECTION_START =
+  "<!-- This section is auto-maintained by lore (https://github.com/BYK/opencode-lore) -->";
+export const LORE_SECTION_END = "<!-- End lore-managed section -->";
+
+/** Regex matching a valid UUID (v4 or v7) — 8-4-4-4-12 hex groups. */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/** Matches `<!-- lore:UUID -->` tracking markers. */
+const MARKER_RE = /^<!--\s*lore:([0-9a-f-]+)\s*-->$/;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type ParsedFileEntry = {
+  /** UUID from `<!-- lore:UUID -->` marker, or null for hand-written entries. */
+  id: string | null;
+  category: string;
+  title: string;
+  content: string;
+};
+
+// ---------------------------------------------------------------------------
+// Section extraction helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Split file content into three parts: before, lore section body, after.
+ * Returns null for section body when markers are absent.
+ */
+function splitFile(fileContent: string): {
+  before: string;
+  section: string | null;
+  after: string;
+} {
+  const startIdx = fileContent.indexOf(LORE_SECTION_START);
+  const endIdx = fileContent.indexOf(LORE_SECTION_END);
+
+  if (startIdx === -1 || endIdx === -1 || endIdx < startIdx) {
+    return { before: fileContent, section: null, after: "" };
+  }
+
+  const before = fileContent.slice(0, startIdx);
+  const section = fileContent.slice(
+    startIdx + LORE_SECTION_START.length,
+    endIdx,
+  );
+  const after = fileContent.slice(endIdx + LORE_SECTION_END.length);
+  return { before, section, after };
+}
+
+// ---------------------------------------------------------------------------
+// Parse entries from a lore section body (or any markdown block)
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract ParsedFileEntry objects from a markdown section body.
+ * Handles:
+ * - `<!-- lore:UUID -->` markers before bullet points  → id set
+ * - Bare bullet points without markers                → id null
+ * - Category derived from the nearest preceding `### Heading`
+ * - Malformed or non-UUID markers                    → id null (hand-written)
+ * - Duplicate UUIDs                                  → both returned; caller deduplicates
+ */
+export function parseEntriesFromSection(section: string): ParsedFileEntry[] {
+  const lines = section.split("\n");
+  const entries: ParsedFileEntry[] = [];
+  let currentCategory = "pattern";
+  let pendingId: string | null = null;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+
+    // Category heading: ### Decision / ### Gotcha / etc.
+    const headingMatch = line.match(/^###\s+(.+)$/);
+    if (headingMatch) {
+      currentCategory = headingMatch[1].toLowerCase();
+      pendingId = null;
+      continue;
+    }
+
+    // Marker line: <!-- lore:UUID -->
+    const markerMatch = line.match(MARKER_RE);
+    if (markerMatch) {
+      const candidate = markerMatch[1];
+      pendingId = UUID_RE.test(candidate) ? candidate : null;
+      continue;
+    }
+
+    // Bullet entry: * **Title**: Content
+    const bulletMatch = line.match(/^\*\s+\*\*(.+?)\*\*:\s*(.+)$/);
+    if (bulletMatch) {
+      // Unescape remark's markdown escapes (e.g. \< → <, \\ → \).
+      // Without this, each export/import cycle doubles the backslash-escapes,
+      // exponentially inflating stored content.
+      entries.push({
+        id: pendingId,
+        category: currentCategory,
+        title: unescapeMarkdown(bulletMatch[1].trim()),
+        content: unescapeMarkdown(bulletMatch[2].trim()),
+      });
+      pendingId = null; // consume the pending marker
+      continue;
+    }
+
+    // Any non-matching non-empty line resets the pending marker
+    if (line !== "" && !line.startsWith("##") && !line.startsWith("<!--")) {
+      pendingId = null;
+    }
+  }
+
+  return entries;
+}
+
+// ---------------------------------------------------------------------------
+// Content hash (for change detection)
+// ---------------------------------------------------------------------------
+
+function hashSection(section: string): string {
+  let h = 0;
+  for (let i = 0; i < section.length; i++) {
+    h = (Math.imul(31, h) + section.charCodeAt(i)) | 0;
+  }
+  // Convert to unsigned hex string
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+
+// ---------------------------------------------------------------------------
+// Build the lore section body from DB entries
+// ---------------------------------------------------------------------------
+
+function buildSection(projectPath: string): string {
+  // Export only project-specific entries (cross_project=0, project_id = this project).
+  // Cross-project entries live in the shared DB on each machine and don't belong
+  // in a per-project AGENTS.md — including them would inflate the file with
+  // unrelated knowledge from every other project the user has worked on.
+  const entries = ltm.forProject(projectPath, false);
+  if (!entries.length) {
+    return "\n";
+  }
+  const formatted = formatKnowledge(
+    entries.map((e) => ({ category: e.category, title: e.title, content: e.content })),
+  );
+  if (!formatted) return "\n";
+
+  // Inject <!-- lore:UUID --> above each bullet line
+  const idByTitle = new Map(entries.map((e) => [e.title, e.id]));
+  const lines = formatted.split("\n");
+  const out: string[] = [""];
+  for (const line of lines) {
+    const bulletMatch = line.match(/^\*\s+\*\*(.+?)\*\*/);
+    if (bulletMatch) {
+      const id = idByTitle.get(bulletMatch[1]);
+      if (id) out.push(`<!-- lore:${id} -->`);
+    }
+    out.push(line);
+  }
+  out.push("");
+  return out.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Export
+// ---------------------------------------------------------------------------
+
+/**
+ * Write current knowledge entries into the AGENTS.md file, preserving all
+ * non-lore content. Creates the file if it doesn't exist.
+ */
+export function exportToFile(input: {
+  projectPath: string;
+  filePath: string;
+}): void {
+  const sectionBody = buildSection(input.projectPath);
+  const newSection =
+    LORE_SECTION_START + sectionBody + LORE_SECTION_END + "\n";
+
+  let fileContent = "";
+  if (existsSync(input.filePath)) {
+    fileContent = readFileSync(input.filePath, "utf8");
+  }
+
+  const { before, after } = splitFile(fileContent);
+
+  // Ensure there's a blank line separator before the section when appending
+  const prefix = before.trimEnd();
+  const prefixWithSep = prefix.length > 0 ? prefix + "\n\n" : "";
+  const suffix = after.trimStart();
+  const suffixWithSep = suffix.length > 0 ? "\n" + suffix : "";
+
+  const result = prefixWithSep + newSection + suffixWithSep;
+
+  mkdirSync(dirname(input.filePath), { recursive: true });
+  writeFileSync(input.filePath, result, "utf8");
+}
+
+// ---------------------------------------------------------------------------
+// shouldImport
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns true if the file needs to be imported:
+ * - File exists and has never been processed (no lore markers)
+ * - File exists and its lore section differs from what lore would currently produce
+ */
+export function shouldImport(input: {
+  projectPath: string;
+  filePath: string;
+}): boolean {
+  if (!existsSync(input.filePath)) return false;
+
+  const fileContent = readFileSync(input.filePath, "utf8");
+  const { section } = splitFile(fileContent);
+
+  if (section === null) {
+    // No lore markers — this is a hand-written file that hasn't been imported
+    return fileContent.trim().length > 0;
+  }
+
+  // Compare the file's lore section body against what we'd produce now
+  const expected = buildSection(input.projectPath);
+  return hashSection(section) !== hashSection(expected);
+}
+
+// ---------------------------------------------------------------------------
+// Import
+// ---------------------------------------------------------------------------
+
+/**
+ * Import knowledge entries from the agents file into the local DB.
+ *
+ * Behaviour per entry:
+ * - Known UUID (already in DB)  → update content if it changed (manual edit)
+ * - Unknown UUID (other machine)→ create with that exact ID
+ * - No UUID (hand-written)      → create with a new UUIDv7
+ * - Duplicate UUID in same file → first occurrence wins, rest ignored
+ */
+export function importFromFile(input: {
+  projectPath: string;
+  filePath: string;
+}): void {
+  if (!existsSync(input.filePath)) return;
+
+  const fileContent = readFileSync(input.filePath, "utf8");
+  const { section, before } = splitFile(fileContent);
+
+  // Determine what to parse:
+  // - If lore markers exist: parse ONLY the lore section body (avoid re-importing our own output)
+  // - If no markers: parse the full file (first-time hand-written AGENTS.md import)
+  const textToParse = section ?? fileContent;
+
+  const fileEntries = parseEntriesFromSection(textToParse);
+  if (!fileEntries.length) return;
+
+  const seenIds = new Set<string>();
+
+  for (const entry of fileEntries) {
+    if (entry.id !== null) {
+      // Deduplicate: if same UUID appears twice in file, first wins
+      if (seenIds.has(entry.id)) continue;
+      seenIds.add(entry.id);
+
+      const existing = ltm.get(entry.id);
+      if (existing) {
+        // Known entry — update only if content changed (manual edit in file)
+        if (existing.content !== entry.content) {
+          ltm.update(entry.id, { content: entry.content });
+        }
+      } else {
+        // Unknown UUID — entry came from another machine, preserve its ID
+        ltm.create({
+          projectPath: input.projectPath,
+          category: entry.category,
+          title: entry.title,
+          content: entry.content,
+          scope: "project",
+          id: entry.id,
+        });
+      }
+    } else {
+      // Hand-written entry — create with a new UUIDv7
+      // Check for a near-duplicate by title to avoid double-import on re-runs
+      const existing = ltm.forProject(input.projectPath, true);
+      const titleMatch = existing.find(
+        (e) => e.title.toLowerCase() === entry.title.toLowerCase(),
+      );
+      if (!titleMatch) {
+        ltm.create({
+          projectPath: input.projectPath,
+          category: entry.category,
+          title: entry.title,
+          content: entry.content,
+          scope: "project",
+        });
+      }
+    }
+  }
+}

package/src/config.ts

@@ -12,6 +12,8 @@ export const LoreConfig = z.object({
       distilled: z.number().min(0.05).max(0.5).default(0.25),
       raw: z.number().min(0.1).max(0.7).default(0.4),
       output: z.number().min(0.1).max(0.5).default(0.25),
+      /** Max fraction of usable context reserved for LTM system-prompt injection. Default: 0.10 (10%). */
+      ltm: z.number().min(0.02).max(0.3).default(0.10),
     })
     .default({}),
   distillation: z
@@ -29,6 +31,14 @@ export const LoreConfig = z.object({
     })
     .default({}),
   crossProject: z.boolean().default(true),
+  agentsFile: z
+    .object({
+      /** Set to false to disable all AGENTS.md export/import behaviour. */
+      enabled: z.boolean().default(true),
+      /** Path to the agents file, relative to the project root. */
+      path: z.string().default("AGENTS.md"),
+    })
+    .default({}),
 });
 
 export type LoreConfig = z.infer<typeof LoreConfig>;
@@ -40,14 +50,11 @@ export function config(): LoreConfig {
 }
 
 export async function load(directory: string): Promise<LoreConfig> {
-  const paths = [`${directory}/.opencode/lore.json`, `${directory}/lore.json`];
-  for (const path of paths) {
-    const file = Bun.file(path);
-    if (await file.exists()) {
-      const raw = await file.json();
-      current = LoreConfig.parse(raw);
-      return current;
-    }
+  const file = Bun.file(`${directory}/.lore.json`);
+  if (await file.exists()) {
+    const raw = await file.json();
+    current = LoreConfig.parse(raw);
+    return current;
   }
   current = LoreConfig.parse({});
   return current;

package/src/curator.ts

@@ -5,6 +5,14 @@ import * as ltm from "./ltm";
 import { CURATOR_SYSTEM, curatorUser } from "./prompt";
 import { workerSessionIDs } from "./distillation";
 
+/**
+ * Maximum length (chars) for a single knowledge entry's content.
+ * ~500 tokens. Entries exceeding this are truncated with a notice.
+ * The curator prompt also instructs the model to stay within this limit,
+ * so truncation is a last-resort safety net.
+ */
+const MAX_ENTRY_CONTENT_LENGTH = 2000;
+
 type Client = ReturnType<typeof createOpencodeClient>;
 
 const workerSessions = new Map<string, string>();
@@ -120,11 +128,18 @@ export async function run(input: {
 
   for (const op of ops) {
     if (op.op === "create") {
+      // Truncate oversized content — the model should stay within the prompt's
+      // 500-word limit, but enforce it here as a hard safety net.
+      const content =
+        op.content.length > MAX_ENTRY_CONTENT_LENGTH
+          ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+            " [truncated — entry too long]"
+          : op.content;
       ltm.create({
         projectPath: op.scope === "project" ? input.projectPath : undefined,
         category: op.category,
         title: op.title,
-        content: op.content,
+        content,
         session: input.sessionID,
         scope: op.scope,
         crossProject: op.crossProject ?? true,
@@ -133,7 +148,12 @@ export async function run(input: {
     } else if (op.op === "update") {
       const entry = ltm.get(op.id);
       if (entry) {
-        ltm.update(op.id, { content: op.content, confidence: op.confidence });
+        const content =
+          op.content !== undefined && op.content.length > MAX_ENTRY_CONTENT_LENGTH
+            ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+              " [truncated — entry too long]"
+            : op.content;
+        ltm.update(op.id, { content, confidence: op.confidence });
         updated++;
       }
     } else if (op.op === "delete") {

package/src/gradient.ts

@@ -40,11 +40,37 @@ const FIRST_TURN_OVERHEAD = 15_000;
 // Null = not yet calibrated (first turn). Updated after every assistant response.
 let calibratedOverhead: number | null = null;
 
+// LTM tokens injected via system transform hook this turn.
+// Set by setLtmTokens() after the system hook runs; consumed by transform().
+let ltmTokens = 0;
+
 export function setModelLimits(limits: { context: number; output: number }) {
   contextLimit = limits.context || 200_000;
   outputReserved = Math.min(limits.output || 32_000, 32_000);
 }
 
+/** Called by the system transform hook after formatting LTM knowledge. */
+export function setLtmTokens(tokens: number) {
+  ltmTokens = tokens;
+}
+
+/** Returns the current LTM token count (for tests and diagnostics). */
+export function getLtmTokens(): number {
+  return ltmTokens;
+}
+
+/**
+ * Returns the token budget available for LTM system-prompt injection.
+ * This is the usable context (after output + overhead) multiplied by
+ * the configured ltm budget fraction. Call this from the system transform
+ * hook to cap how many tokens formatKnowledge may use.
+ */
+export function getLtmBudget(ltmFraction: number): number {
+  const overhead = calibratedOverhead ?? FIRST_TURN_OVERHEAD;
+  const usable = Math.max(0, contextLimit - outputReserved - overhead);
+  return Math.floor(usable * ltmFraction);
+}
+
 // Called after each assistant message completes with real token usage data.
 // actualInput = tokens.input + tokens.cache.read (all tokens that went into the model)
 // messageEstimate = our chars/4 estimate of the messages we sent
@@ -385,7 +411,11 @@ export function transform(input: {
   const cfg = config();
   const overhead = getOverhead();
   // Usable = full context minus output reservation minus fixed overhead (system + tools)
-  const usable = contextLimit - outputReserved - overhead;
+  // minus LTM tokens already injected into the system prompt this turn.
+  const usable = Math.max(
+    0,
+    contextLimit - outputReserved - overhead - ltmTokens,
+  );
   const distilledBudget = Math.floor(usable * cfg.budget.distilled);
   const rawBudget = Math.floor(usable * cfg.budget.raw);
 

package/src/index.ts

@@ -11,6 +11,8 @@ import {
   needsUrgentDistillation,
   calibrate,
   estimateMessages,
+  setLtmTokens,
+  getLtmBudget,
 } from "./gradient";
 import { formatKnowledge } from "./prompt";
 import { createRecallTool } from "./reflect";
@@ -20,6 +22,14 @@ export const LorePlugin: Plugin = async (ctx) => {
   await load(ctx.directory);
   ensureProject(projectPath);
 
+  // Prune any corrupted/oversized knowledge entries left by the AGENTS.md
+  // backslash-escaping bug or curator hallucinations. Sets confidence → 0
+  // (below the 0.2 query threshold) so they stop polluting the context.
+  const pruned = ltm.pruneOversized(2000);
+  if (pruned > 0) {
+    console.error(`[lore] pruned ${pruned} oversized knowledge entries (confidence set to 0)`);
+  }
+
   // Track user turns for periodic curation
   let turnsSinceCuration = 0;
 
@@ -197,14 +207,19 @@ export const LorePlugin: Plugin = async (ctx) => {
       }
     },
 
-    // Inject LTM knowledge into system prompt
+    // Inject LTM knowledge into system prompt — relevance-ranked and budget-capped.
     "experimental.chat.system.transform": async (input, output) => {
       if (input.model?.limit) {
         setModelLimits(input.model.limit);
       }
 
-      const entries = ltm.forProject(projectPath, config().crossProject);
-      if (!entries.length) return;
+      const cfg = config();
+      const ltmBudget = getLtmBudget(cfg.budget.ltm);
+      const entries = ltm.forSession(projectPath, input.sessionID, ltmBudget);
+      if (!entries.length) {
+        setLtmTokens(0);
+        return;
+      }
 
       const formatted = formatKnowledge(
         entries.map((e) => ({
@@ -212,9 +227,17 @@ export const LorePlugin: Plugin = async (ctx) => {
           title: e.title,
           content: e.content,
         })),
+        ltmBudget,
       );
+
       if (formatted) {
+        // Track how many tokens we actually consumed so the gradient manager
+        // can deduct them from the usable budget for message injection.
+        const ltmTokenCount = Math.ceil(formatted.length / 4);
+        setLtmTokens(ltmTokenCount);
         output.system.push(formatted);
+      } else {
+        setLtmTokens(0);
       }
     },
 
@@ -224,9 +247,6 @@ export const LorePlugin: Plugin = async (ctx) => {
 
       const sessionID = output.messages[0]?.info.sessionID;
 
-      const lastUserMsg = [...output.messages].reverse().find((m) => m.info.role === "user");
-      const statsPart = lastUserMsg?.parts.find((p) => p.type === "text");
-
       const result = transform({
         messages: output.messages,
         projectPath,
@@ -252,6 +272,12 @@ export const LorePlugin: Plugin = async (ctx) => {
         backgroundDistill(sessionID);
       }
 
+      // Look up statsPart AFTER the transform so the PATCHed text is clean
+      // (system-reminder wrappers stripped). Looking up before would persist
+      // ephemeral system-reminder content, making it visible in the UI.
+      const lastUserMsg = [...output.messages].reverse().find((m) => m.info.role === "user");
+      const statsPart = lastUserMsg?.parts.find((p) => p.type === "text");
+
       if (sessionID && statsPart && lastUserMsg) {
         const loreMeta = {
           layer: result.layer,

package/src/ltm.ts

@@ -1,6 +1,12 @@
+import { uuidv7 } from "uuidv7";
 import { db, ensureProject } from "./db";
 import { ftsQuery } from "./temporal";
 
+// Rough token estimate: ~4 chars per token
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
 export type KnowledgeEntry = {
   id: string;
   project_id: string | null;
@@ -23,12 +29,14 @@ export function create(input: {
   session?: string;
   scope: "project" | "global";
   crossProject?: boolean;
+  /** Explicit ID to use — for cross-machine import via agents-file. Defaults to a new UUIDv7. */
+  id?: string;
 }): string {
   const pid =
     input.scope === "project" && input.projectPath
       ? ensureProject(input.projectPath)
       : null;
-  const id = crypto.randomUUID();
+  const id = input.id ?? uuidv7();
   const now = Date.now();
   db()
     .query(
@@ -100,6 +108,155 @@ export function forProject(
     .all(pid) as KnowledgeEntry[];
 }
 
+/**
+ * Build a relevance-ranked, budget-capped list of knowledge entries for injection
+ * into the system prompt of a live session.
+ *
+ * Strategy:
+ * 1. Project-specific entries (project_id = current project, cross_project = 0)
+ *    always get priority — they were curated specifically for this codebase.
+ * 2. Cross-project entries are scored for relevance against recent session context
+ *    (last distillation + recent raw messages). Only entries that match are included.
+ * 3. All candidates are ranked by score * confidence, then greedily packed into
+ *    the token budget (smallest-first within same score band to maximize count).
+ * 4. If there's no session context yet (first turn), fall back to top entries by
+ *    confidence only.
+ *
+ * @param projectPath   Current project path
+ * @param sessionID     Current session ID (for context extraction)
+ * @param maxTokens     Hard token budget for the entire formatted block
+ */
+export function forSession(
+  projectPath: string,
+  sessionID: string | undefined,
+  maxTokens: number,
+): KnowledgeEntry[] {
+  const pid = ensureProject(projectPath);
+
+  // --- 1. Load project-specific entries (always relevant) ---
+  const projectEntries = db()
+    .query(
+      `SELECT * FROM knowledge
+       WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
+       ORDER BY confidence DESC, updated_at DESC`,
+    )
+    .all(pid) as KnowledgeEntry[];
+
+  // --- 2. Load cross-project candidates ---
+  const crossEntries = db()
+    .query(
+      `SELECT * FROM knowledge
+       WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
+       ORDER BY confidence DESC, updated_at DESC`,
+    )
+    .all() as KnowledgeEntry[];
+
+  if (!crossEntries.length && !projectEntries.length) return [];
+
+  // --- 3. Build session context for relevance scoring ---
+  // Combine the most recent distillation text + last ~10 raw messages for this session
+  let sessionContext = "";
+  if (sessionID) {
+    const distRow = db()
+      .query(
+        `SELECT observations FROM distillations
+         WHERE project_id = ? AND session_id = ?
+         ORDER BY created_at DESC LIMIT 1`,
+      )
+      .get(pid, sessionID) as { observations: string } | null;
+    if (distRow?.observations) {
+      sessionContext += distRow.observations + "\n";
+    }
+    const recentMsgs = db()
+      .query(
+        `SELECT content FROM temporal_messages
+         WHERE project_id = ? AND session_id = ?
+         ORDER BY created_at DESC LIMIT 10`,
+      )
+      .all(pid, sessionID) as Array<{ content: string }>;
+    if (recentMsgs.length) {
+      sessionContext += recentMsgs.map((m) => m.content).join("\n");
+    }
+  }
+
+  // --- 4. Score cross-project entries by relevance ---
+  // Use FTS5 matching: extract terms from session context and score each entry
+  type Scored = { entry: KnowledgeEntry; score: number };
+  let scoredCross: Scored[];
+
+  if (sessionContext.trim().length > 20) {
+    // Build a term set from session context (top 30 meaningful words)
+    const contextTerms = sessionContext
+      .replace(/[^\w\s]/g, " ")
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((w) => w.length > 3)
+      .reduce<Map<string, number>>((acc, w) => {
+        acc.set(w, (acc.get(w) ?? 0) + 1);
+        return acc;
+      }, new Map());
+
+    // Sort by frequency, take top 30 terms
+    const topTerms = [...contextTerms.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 30)
+      .map(([w]) => w);
+
+    scoredCross = crossEntries.map((entry) => {
+      const haystack =
+        (entry.title + " " + entry.content).replace(/[^\w\s]/g, " ").toLowerCase();
+      let hits = 0;
+      for (const term of topTerms) {
+        // Count how many context terms appear in this entry (simple overlap)
+        if (haystack.includes(term)) hits++;
+      }
+      // Score = fraction of top terms matched, weighted by confidence
+      const relevance = topTerms.length > 0 ? hits / topTerms.length : 0;
+      return { entry, score: relevance * entry.confidence };
+    });
+
+    // Only keep entries with at least one term match
+    scoredCross = scoredCross.filter((s) => s.score > 0);
+  } else {
+    // No session context yet — take top cross-project entries by confidence
+    scoredCross = crossEntries.slice(0, 10).map((entry) => ({
+      entry,
+      score: entry.confidence,
+    }));
+  }
+
+  // Sort cross-project by score desc
+  scoredCross.sort((a, b) => b.score - a.score);
+
+  // --- 5. Pack into token budget ---
+  // Project entries get first pick (fully relevant); cross entries fill remaining budget.
+  // Use a greedy fit: iterate candidates and include if they fit.
+  const HEADER_OVERHEAD_TOKENS = 15; // "## Long-term Knowledge\n"
+  let used = HEADER_OVERHEAD_TOKENS;
+  const result: KnowledgeEntry[] = [];
+
+  function tryAdd(entry: KnowledgeEntry): boolean {
+    const cost = estimateTokens(entry.title + entry.content) + 10;
+    if (used + cost > maxTokens) return false;
+    result.push(entry);
+    used += cost;
+    return true;
+  }
+
+  // Project-specific first
+  for (const entry of projectEntries) {
+    tryAdd(entry);
+  }
+
+  // Then cross-project by relevance score
+  for (const { entry } of scoredCross) {
+    if (used >= maxTokens) break;
+    tryAdd(entry);
+  }
+
+  return result;
+}
+
 export function all(): KnowledgeEntry[] {
   return db()
     .query(
@@ -184,3 +341,22 @@ export function get(id: string): KnowledgeEntry | null {
     .query("SELECT * FROM knowledge WHERE id = ?")
     .get(id) as KnowledgeEntry | null;
 }
+
+/**
+ * Prune knowledge entries whose content exceeds maxLength characters.
+ * These are typically corrupted entries from AGENTS.md roundtrip escaping bugs
+ * or curator hallucinations with full code dumps.
+ *
+ * Rather than hard-deleting, sets confidence to 0 so they're excluded from
+ * queries (confidence > 0.2) but can be inspected for debugging.
+ *
+ * @returns Number of entries pruned
+ */
+export function pruneOversized(maxLength: number): number {
+  const result = db()
+    .query(
+      "UPDATE knowledge SET confidence = 0, updated_at = ? WHERE LENGTH(content) > ? AND confidence > 0",
+    )
+    .run(Date.now(), maxLength);
+  return result.changes;
+}

package/src/markdown.ts

@@ -40,6 +40,35 @@ export function normalize(md: string): string {
   return processor.stringify(processor.parse(once));
 }
 
+/**
+ * Unescape a markdown-serialized inline string back to plain text.
+ *
+ * remark's serializer escapes special characters with backslashes
+ * (e.g. `<` → `\<`, `*` → `\*`, `\` → `\\`). When we read content
+ * back from an AGENTS.md file we must unescape it so it round-trips
+ * cleanly — otherwise each export/import cycle doubles the escapes.
+ *
+ * Uses remark's own parser to extract the text value, which handles
+ * all escape sequences correctly.
+ */
+export function unescapeMarkdown(md: string): string {
+  const tree = processor.parse(md);
+  // Collect all text node values from the first paragraph
+  const texts: string[] = [];
+  const para = tree.children[0];
+  if (para && para.type === "paragraph") {
+    for (const child of para.children) {
+      if (child.type === "text") texts.push(child.value);
+      else if (child.type === "strong" || child.type === "emphasis") {
+        for (const gc of child.children) {
+          if (gc.type === "text") texts.push(gc.value);
+        }
+      }
+    }
+  }
+  return texts.join("") || md;
+}
+
 // --- Node builders ---
 
 export function h(depth: 1 | 2 | 3 | 4 | 5 | 6, value: string): Heading {

package/src/prompt.ts

@@ -189,20 +189,31 @@ Do NOT extract:
 - Temporary state (current branch, in-progress work)
 - Information that will change frequently
 
+BREVITY IS CRITICAL — each entry must be concise:
+- content MUST be under 500 words (roughly 2000 characters)
+- Focus on the actionable insight, not the full story behind it
+- If a pattern requires more detail, split into multiple focused entries
+- Omit code examples unless a single short snippet is essential
+- Never include full file contents, large diffs, or complete command outputs
+
+crossProject flag:
+- Default is true — most useful knowledge is worth sharing across projects
+- Set crossProject to false for things that are meaningless outside this specific repo (e.g. a config path, a project-local naming convention that conflicts with your usual style)
+
 Produce a JSON array of operations:
 [
   {
     "op": "create",
     "category": "decision" | "pattern" | "preference" | "architecture" | "gotcha",
     "title": "Short descriptive title",
-    "content": "Detailed knowledge entry",
+    "content": "Concise knowledge entry — under 500 words",
     "scope": "project" | "global",
     "crossProject": true
   },
   {
     "op": "update",
     "id": "existing-entry-id",
-    "content": "Updated content",
+    "content": "Updated content — under 500 words",
     "confidence": 0.0-1.0
   },
   {
@@ -267,13 +278,38 @@ export function formatDistillations(
   return sections.join("\n\n");
 }
 
+// Rough token estimate used for budget-gating knowledge entries.
+// Consistent with gradient.ts: ~4 chars per token.
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
 export function formatKnowledge(
   entries: Array<{ category: string; title: string; content: string }>,
+  maxTokens?: number,
 ): string {
   if (!entries.length) return "";
 
+  // Apply token budget: greedily include entries (already sorted by confidence
+  // DESC from the DB query) until the budget is exhausted. Overhead accounts for
+  // the section heading and per-entry markdown scaffolding (~50 chars each).
+  let included = entries;
+  if (maxTokens !== undefined) {
+    const HEADER_OVERHEAD = 50; // "## Long-term Knowledge\n### Category\n"
+    let used = HEADER_OVERHEAD;
+    const fitting: typeof entries = [];
+    for (const e of entries) {
+      const cost = estimateTokens(e.title + e.content) + 10; // per-entry bullet overhead
+      if (used + cost > maxTokens) continue; // skip; keep trying smaller entries
+      fitting.push(e);
+      used += cost;
+    }
+    included = fitting;
+    if (!included.length) return "";
+  }
+
   const grouped: Record<string, Array<{ title: string; content: string }>> = {};
-  for (const e of entries) {
+  for (const e of included) {
     const group = grouped[e.category] ?? (grouped[e.category] = []);
     group.push(e);
   }