opencode-lore 0.3.9 → 0.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-lore",
-  "version": "0.3.9",
+  "version": "0.4.1",
   "type": "module",
   "license": "MIT",
   "description": "Three-tier memory architecture for OpenCode — distillation, not summarization",

package/src/agents-file.ts

@@ -18,7 +18,7 @@ import { serialize, inline, h, ul, liph, strong, t, root, unescapeMarkdown } fro
 // ---------------------------------------------------------------------------
 
 export const LORE_SECTION_START =
-  "<!-- This section is auto-maintained by lore (https://github.com/BYK/opencode-lore) -->";
+  "<!-- This section is maintained by the coding agent via lore (https://github.com/BYK/opencode-lore) -->";
 export const LORE_SECTION_END = "<!-- End lore-managed section -->";
 
 /** Regex matching a valid UUID (v4 or v7) — 8-4-4-4-12 hex groups. */

package/src/config.ts

@@ -28,6 +28,8 @@ export const LoreConfig = z.object({
       enabled: z.boolean().default(true),
       onIdle: z.boolean().default(true),
       afterTurns: z.number().min(1).default(10),
+      /** Max knowledge entries per project before consolidation triggers. Default: 25. */
+      maxEntries: z.number().min(10).default(25),
     })
     .default({}),
   pruning: z

package/src/curator.ts

@@ -2,16 +2,16 @@ import type { createOpencodeClient } from "@opencode-ai/sdk";
 import { config } from "./config";
 import * as temporal from "./temporal";
 import * as ltm from "./ltm";
-import { CURATOR_SYSTEM, curatorUser } from "./prompt";
+import { CURATOR_SYSTEM, curatorUser, CONSOLIDATION_SYSTEM, consolidationUser } from "./prompt";
 import { workerSessionIDs } from "./distillation";
 
 /**
  * Maximum length (chars) for a single knowledge entry's content.
- * ~500 tokens. Entries exceeding this are truncated with a notice.
+ * ~400 tokens at chars/3. Entries exceeding this are truncated with a notice.
  * The curator prompt also instructs the model to stay within this limit,
  * so truncation is a last-resort safety net.
  */
-const MAX_ENTRY_CONTENT_LENGTH = 2000;
+const MAX_ENTRY_CONTENT_LENGTH = 1200;
 
 type Client = ReturnType<typeof createOpencodeClient>;
 
@@ -172,3 +172,88 @@ export async function run(input: {
 export function resetCurationTracker() {
   lastCuratedAt = 0;
 }
+
+/**
+ * Consolidation pass: reviews ALL project entries and merges/trims/deletes
+ * to reduce entry count to cfg.curator.maxEntries. Only runs when the current
+ * entry count exceeds the target. Uses the same worker session as curation.
+ *
+ * Only "update" and "delete" ops are applied — consolidation never creates entries.
+ */
+export async function consolidate(input: {
+  client: Client;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+}): Promise<{ updated: number; deleted: number }> {
+  const cfg = config();
+  if (!cfg.curator.enabled) return { updated: 0, deleted: 0 };
+
+  const entries = ltm.forProject(input.projectPath, cfg.crossProject);
+  if (entries.length <= cfg.curator.maxEntries) return { updated: 0, deleted: 0 };
+
+  const entriesForPrompt = entries.map((e) => ({
+    id: e.id,
+    category: e.category,
+    title: e.title,
+    content: e.content,
+  }));
+
+  const userContent = consolidationUser({
+    entries: entriesForPrompt,
+    targetMax: cfg.curator.maxEntries,
+  });
+  const workerID = await ensureWorkerSession(input.client, input.sessionID);
+  const model = input.model ?? cfg.model;
+  const parts = [
+    { type: "text" as const, text: `${CONSOLIDATION_SYSTEM}\n\n${userContent}` },
+  ];
+
+  await input.client.session.prompt({
+    path: { id: workerID },
+    body: {
+      parts,
+      agent: "lore-curator",
+      ...(model ? { model } : {}),
+    },
+  });
+
+  const msgs = await input.client.session.messages({
+    path: { id: workerID },
+    query: { limit: 2 },
+  });
+  const last = msgs.data?.at(-1);
+  if (!last || last.info.role !== "assistant") return { updated: 0, deleted: 0 };
+
+  const responsePart = last.parts.find((p) => p.type === "text");
+  if (!responsePart || responsePart.type !== "text") return { updated: 0, deleted: 0 };
+
+  const ops = parseOps(responsePart.text);
+  let updated = 0;
+  let deleted = 0;
+
+  for (const op of ops) {
+    // Consolidation only applies update and delete — never create.
+    if (op.op === "update") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        const content =
+          op.content !== undefined && op.content.length > MAX_ENTRY_CONTENT_LENGTH
+            ? op.content.slice(0, MAX_ENTRY_CONTENT_LENGTH) +
+              " [truncated — entry too long]"
+            : op.content;
+        ltm.update(op.id, { content, confidence: op.confidence });
+        updated++;
+      }
+    } else if (op.op === "delete") {
+      const entry = ltm.get(op.id);
+      if (entry) {
+        ltm.remove(op.id);
+        deleted++;
+      }
+    }
+    // "create" ops are silently ignored — consolidation must not add entries.
+  }
+
+  return { updated, deleted };
+}

package/src/distillation.ts

@@ -153,7 +153,7 @@ function storeDistillation(input: {
   const pid = ensureProject(input.projectPath);
   const id = crypto.randomUUID();
   const sourceJson = JSON.stringify(input.sourceIDs);
-  const tokens = Math.ceil(input.observations.length / 4);
+  const tokens = Math.ceil(input.observations.length / 3);
   db()
     .query(
       `INSERT INTO distillations (id, project_id, session_id, narrative, facts, observations, source_ids, generation, token_count, created_at)

package/src/gradient.ts

@@ -6,9 +6,12 @@ import { normalize } from "./markdown";
 
 type MessageWithParts = { info: Message; parts: Part[] };
 
-// Rough token estimate: ~4 chars per token
+// Token estimate: ~3 chars per token. Validated against real API data across
+// 200+ turn-pairs: chars/3 gives ~1.68x ratio (actual/estimate), best among
+// heuristics tested. The gap is overhead (system prompt, tool definitions,
+// conversation structure) which calibratedOverhead captures via EMA.
 function estimate(text: string): number {
-  return Math.ceil(text.length / 4);
+  return Math.ceil(text.length / 3);
 }
 
 function estimateParts(parts: Part[]): number {
@@ -70,6 +73,8 @@ type SessionState = {
   lastWindowMessageIDs: Set<string>;
   /** One-shot force escalation: skip layers below this on the next transform() */
   forceMinLayer: SafetyLayer;
+  /** Token estimate from the most recent transform() output (compressed window) */
+  lastTransformEstimate: number;
   /** Distilled prefix cache (Approach C) */
   prefixCache: PrefixCache | null;
   /** Raw window pin cache (Approach B) */
@@ -85,6 +90,7 @@ function makeSessionState(): SessionState {
     lastLayer: 0,
     lastWindowMessageIDs: new Set(),
     forceMinLayer: 0,
+    lastTransformEstimate: 0,
     prefixCache: null,
     rawWindowCache: null,
   };
@@ -139,22 +145,36 @@ export function getLtmBudget(ltmFraction: number): number {
 }
 
 // Called after each assistant message completes with real token usage data.
-// actualInput    = tokens.input + tokens.cache.read (all tokens the model saw)
-// messageEstimate = our chars/4 estimate of the messages we sent
+// actualInput    = tokens.input + tokens.cache.read + tokens.cache.write
 // sessionID      = session that produced this response (for exact-tracking validity)
 // messageCount   = number of messages that were sent (for delta estimation)
+//
+// Overhead calibration uses lastTransformEstimate (the token estimate from the
+// compressed window that was actually sent to the model) instead of re-estimating
+// all session messages. On compressed sessions, all-message estimate >> actualInput,
+// which clamped overhead to 0 and broke budget calculations.
 export function calibrate(
   actualInput: number,
-  messageEstimate: number,
   sessionID?: string,
   messageCount?: number,
 ) {
+  // Use the transform's own estimate for the compressed window it produced.
+  // This is the correct baseline: it estimates the same messages the model saw.
+  const messageEstimate = sessionID
+    ? getSessionState(sessionID).lastTransformEstimate
+    : 0;
+
   // Update global overhead calibration (shared across sessions — model-level).
-  const overhead = Math.max(0, actualInput - messageEstimate);
-  calibratedOverhead =
-    calibratedOverhead === null
-      ? overhead
-      : Math.round(calibratedOverhead * 0.7 + overhead * 0.3);
+  // Skip when actualInput > 0 but no transform estimate exists yet (no baseline
+  // to compare against). Allow when both are 0 (test setup to zero overhead) or
+  // when we have a real transform estimate.
+  if (messageEstimate > 0 || actualInput === 0) {
+    const overhead = Math.max(0, actualInput - messageEstimate);
+    calibratedOverhead =
+      calibratedOverhead === null
+        ? overhead
+        : Math.round(calibratedOverhead * 0.7 + overhead * 0.3);
+  }
 
   // Store per-session exact counts for the proactive layer 0 decision.
   if (sessionID !== undefined) {
@@ -800,20 +820,20 @@ function transformInner(input: {
 
   // --- Approach A: Cache-preserving passthrough ---
   // Use exact token count from the previous API response when available.
-  // Only the delta (messages added since last call) uses chars/4 estimation,
-  // making the layer-0 decision 99%+ accurate from the API's own tokenizer.
+   // Only the delta (messages added since last call) uses chars/3 estimation,
+   // making the layer-0 decision highly accurate from the API's own tokenizer.
   // maxInput = absolute ceiling the API enforces: input_tokens + max_tokens <= context
   const maxInput = contextLimit - outputReserved;
 
   // True when we have real API token data from a previous turn in this session.
-  // When false (first turn / session change), chars/4 estimates can undercount by
-  // up to 1.8x — so tryFit output must be validated with a safety multiplier before
-  // being used, to prevent sending an apparently-fitting window that actually overflows.
+  // When false (first turn / session change), chars/3 estimates may still diverge
+  // from the real tokenizer — so tryFit output must be validated with a safety
+  // multiplier before being used.
   const calibrated = sessState.lastKnownInput > 0;
 
   // On uncalibrated turns, apply this multiplier to tryFit's estimated total to
-  // approximate the real token count. 1.5 is conservative but not so aggressive
-  // that it forces layer 4 on modestly-sized sessions.
+  // approximate the real token count. chars/3 undercounts by ~1.68x on real data,
+  // but overhead EMA captures most of the gap. 1.5 provides a safe margin.
   const UNCALIBRATED_SAFETY = 1.5;
 
   // Returns true if the tryFit result is safe to use: either we have calibrated
@@ -830,7 +850,7 @@ function transformInner(input: {
   // Prevents the calibration oscillation: a compressed turn stores
   // lastKnownInput=100K for a 50-message window, but the next turn's
   // input.messages has 300 raw messages. The delta estimation treats the 250
-  // evicted messages as "new" and undercounts them via chars/4, producing an
+  // evicted messages as "new" and undercounts their tokens, producing an
   // expectedInput that fits in layer 0 — but the actual tokens are ~190K.
   // Only applied when calibrated (same session, per-session state) to avoid
   // affecting other sessions including worker sessions.
@@ -851,7 +871,7 @@ function transformInner(input: {
     const ltmDelta = ltmTokens - sessState.lastKnownLtm;
     expectedInput = sessState.lastKnownInput + newMsgTokens + ltmDelta;
   } else {
-    // First turn or session change: fall back to chars/4 + overhead.
+    // First turn or session change: fall back to chars/3 estimate + overhead.
     const messageTokens = input.messages.reduce((s, m) => s + estimateMessage(m), 0);
     expectedInput = messageTokens + overhead + ltmTokens;
   }
@@ -1009,6 +1029,7 @@ export function transform(input: {
   if (sid) {
     const state = getSessionState(sid);
     state.lastTransformedCount = result.messages.length;
+    state.lastTransformEstimate = result.totalTokens;
     state.lastLayer = result.layer;
     state.lastWindowMessageIDs = new Set(result.messages.map((m) => m.info.id));
   }

package/src/index.ts

@@ -10,7 +10,6 @@ import {
   setModelLimits,
   needsUrgentDistillation,
   calibrate,
-  estimateMessages,
   setLtmTokens,
   getLtmBudget,
   setForceMinLayer,
@@ -56,7 +55,7 @@ export const LorePlugin: Plugin = async (ctx) => {
   // Prune any corrupted/oversized knowledge entries left by the AGENTS.md
   // backslash-escaping bug or curator hallucinations. Sets confidence → 0
   // (below the 0.2 query threshold) so they stop polluting the context.
-  const pruned = ltm.pruneOversized(2000);
+  const pruned = ltm.pruneOversized(1200);
   if (pruned > 0) {
     console.error(`[lore] pruned ${pruned} oversized knowledge entries (confidence set to 0)`);
   }
@@ -204,28 +203,15 @@ export const LorePlugin: Plugin = async (ctx) => {
                 backgroundDistill(msg.sessionID);
               }
 
-              // Calibrate overhead estimate using real token counts.
-              // Also store the exact input count + message count for the proactive
-              // layer-0 decision (avoids full chars/4 re-estimation each turn).
-              // actualInput = all tokens the model processed as input, regardless of
-              // whether they were new (input), read from cache (cache.read), or newly
-              // written to cache (cache.write). All three contribute to the context window.
-              const allMsgs = await ctx.client.session.messages({
-                path: { id: msg.sessionID },
-              });
-              if (allMsgs.data) {
-                const withParts = allMsgs.data
-                  .filter((m) => m.info.id !== msg.id)
-                  .map((m) => ({ info: m.info, parts: m.parts }));
-                const msgEstimate = estimateMessages(withParts);
-                const actualInput =
-                  msg.tokens.input + msg.tokens.cache.read + msg.tokens.cache.write;
-                // Use the compressed message count (from the last transform output),
-                // not the total DB count. On layer 0 these are equal. On layers 1-4,
-                // the model only saw the compressed window — calibrate must track that
-                // count so the next turn's delta is computed correctly.
-                calibrate(actualInput, msgEstimate, msg.sessionID, getLastTransformedCount(msg.sessionID) || withParts.length);
-              }
+              // Calibrate overhead using real token counts from the API response.
+              // actualInput = all tokens the model processed (input + cache.read + cache.write).
+              // The message estimate comes from the transform's own output (stored in
+              // session state as lastTransformEstimate), NOT from re-estimating all session
+              // messages. On compressed sessions, all-message estimate >> actualInput, which
+              // previously clamped overhead to 0 and broke budget calculations.
+              const actualInput =
+                msg.tokens.input + msg.tokens.cache.read + msg.tokens.cache.write;
+              calibrate(actualInput, msg.sessionID, getLastTransformedCount(msg.sessionID));
             }
           }
         } catch {
@@ -301,6 +287,29 @@ export const LorePlugin: Plugin = async (ctx) => {
           turnsSinceCuration = 0;
         }
 
+        // Consolidate entries if count exceeds cfg.curator.maxEntries.
+        // Runs after normal curation so newly created entries are counted.
+        // Only triggers when truly over the limit to avoid redundant LLM calls.
+        try {
+          const allEntries = ltm.forProject(projectPath);
+          if (allEntries.length > cfg.curator.maxEntries) {
+            console.error(
+              `[lore] entry count ${allEntries.length} exceeds maxEntries ${cfg.curator.maxEntries} — running consolidation`,
+            );
+            const { updated, deleted } = await curator.consolidate({
+              client: ctx.client,
+              projectPath,
+              sessionID,
+              model: cfg.model,
+            });
+            if (updated > 0 || deleted > 0) {
+              console.error(`[lore] consolidation: ${updated} updated, ${deleted} deleted`);
+            }
+          }
+        } catch (e) {
+          console.error("[lore] consolidation error:", e);
+        }
+
         // Prune temporal messages after distillation and curation have run.
         // Pass 1: TTL — remove distilled messages older than retention period.
         // Pass 2: Size cap — evict oldest distilled messages if over the limit.
@@ -371,12 +380,24 @@ export const LorePlugin: Plugin = async (ctx) => {
       if (formatted) {
         // Track how many tokens we actually consumed so the gradient manager
         // can deduct them from the usable budget for message injection.
-        const ltmTokenCount = Math.ceil(formatted.length / 4);
+        const ltmTokenCount = Math.ceil(formatted.length / 3);
         setLtmTokens(ltmTokenCount);
         output.system.push(formatted);
       } else {
         setLtmTokens(0);
       }
+
+      // Remind the agent to include the agents file in commits.
+      // It is always modified after the lore export runs (post-session) so it
+      // appears as unstaged when the agent goes to commit — the agent must not
+      // skip it just because it looks auto-generated.
+      if (cfg.agentsFile.enabled) {
+        output.system.push(
+          `When making git commits, always check if ${cfg.agentsFile.path} has ` +
+          `unstaged changes and include it in the commit. This file contains ` +
+          `shared project knowledge managed by lore and must be version-controlled.`,
+        );
+      }
     },
 
     // Transform message history: distilled prefix + raw recent.

package/src/ltm.ts

@@ -2,9 +2,9 @@ import { uuidv7 } from "uuidv7";
 import { db, ensureProject } from "./db";
 import { ftsQuery } from "./temporal";
 
-// Rough token estimate: ~4 chars per token
+// ~3 chars per token — validated as best heuristic against real API data.
 function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
+  return Math.ceil(text.length / 3);
 }
 
 export type KnowledgeEntry = {

package/src/prompt.ts

@@ -195,12 +195,24 @@ Do NOT extract:
 - Restatements of what the code obviously does (e.g. "the auth module handles authentication")
 
 BREVITY IS CRITICAL — each entry must be concise:
-- content MUST be under 500 words (roughly 2000 characters)
+- content MUST be under 150 words (~600 characters). Capture ONE specific actionable
+  insight in 2-3 sentences. Prefer terse technical language.
+- Each "gotcha": one specific trap + its fix in 1-2 sentences
+- Each "architecture": one design decision and its key constraint
 - Focus on the actionable insight, not the full story behind it
-- If a pattern requires more detail, split into multiple focused entries
+- If a pattern requires more detail, split into multiple focused entries (each under 150 words)
 - Omit code examples unless a single short snippet is essential
 - Never include full file contents, large diffs, or complete command outputs
 
+PREFER UPDATES OVER CREATES:
+- Before creating a new entry, always check if an existing entry covers the same system
+  or component. Update the existing entry rather than creating a new one.
+- When updating, REPLACE the full content with a concise rewrite — do not append to
+  the existing content or repeat what was already there.
+- If multiple existing entries cover the same system from different angles (e.g. different
+  bugs in the same module), consolidate them: update one with merged insights, delete the
+  rest. Fewer, denser entries are better than many scattered ones.
+
 crossProject flag:
 - Default is true — most useful knowledge is worth sharing across projects
 - Set crossProject to false for things that are meaningless outside this specific repo (e.g. a config path, a project-local naming convention that conflicts with your usual style)
@@ -211,14 +223,14 @@ Produce a JSON array of operations:
     "op": "create",
     "category": "decision" | "pattern" | "preference" | "architecture" | "gotcha",
     "title": "Short descriptive title",
-    "content": "Concise knowledge entry — under 500 words",
+    "content": "Concise knowledge entry — under 150 words",
     "scope": "project" | "global",
     "crossProject": true
   },
   {
     "op": "update",
     "id": "existing-entry-id",
-    "content": "Updated content — under 500 words",
+    "content": "Updated content — under 150 words",
     "confidence": 0.0-1.0
   },
   {
@@ -241,8 +253,9 @@ export function curatorUser(input: {
     content: string;
   }>;
 }): string {
-  const existing = input.existing.length
-    ? `Existing knowledge entries (you may update or delete these):\n${input.existing.map((e) => `- [${e.id}] (${e.category}) ${e.title}: ${e.content}`).join("\n")}`
+  const count = input.existing.length;
+  const existing = count
+    ? `Existing knowledge entries (${count} total — you may update or delete these):\n${input.existing.map((e) => `- [${e.id}] (${e.category}) ${e.title}: ${e.content}`).join("\n")}`
     : "No existing knowledge entries.";
   return `${existing}
 
@@ -252,7 +265,67 @@ Recent conversation to extract knowledge from:
 ${input.messages}
 
 ---
-IMPORTANT: If any new entries you would create are semantically duplicative of existing entries (same concept, different wording), prefer updating the existing entry rather than creating a new one. Only create new entries for genuinely distinct knowledge.`;
+IMPORTANT:
+1. Prefer updating existing entries over creating new ones. If a new insight refines or
+   extends an existing entry on the same topic, update that entry — don't create a new one.
+2. When updating, REPLACE the content with a complete rewrite — never append.
+3. If entries cover the same system from different angles, merge them: update one, delete the rest.
+4. Only create a new entry for genuinely distinct knowledge with no existing home.
+5. Keep all entries under 150 words. If an existing entry is too long, use an update op to trim it.`;
+}
+
+/**
+ * System prompt for the consolidation pass.
+ * Unlike the normal curator (which extracts from conversation), consolidation
+ * reviews the FULL entry corpus and aggressively merges/trims/deletes to reduce
+ * entry count while preserving the most actionable knowledge.
+ */
+export const CONSOLIDATION_SYSTEM = `You are a long-term memory curator performing a consolidation pass. The knowledge base has grown too large and needs to be trimmed.
+
+Your goal: reduce the entry count to the target maximum while preserving the most valuable knowledge.
+
+CONSOLIDATION RULES:
+1. MERGE related entries — if multiple entries describe the same system, module, or concept
+   from different angles (e.g. several bug fixes in the same component), merge them into
+   ONE concise entry. Use an "update" op for the surviving entry and "delete" ops for the rest.
+2. TRIM verbose entries — any entry over 150 words must be trimmed to its essential insight.
+   Use an "update" op with the rewritten content.
+3. DELETE low-value entries:
+   - Stale entries about bugs that have been fixed and no longer need gotcha warnings
+   - Entries whose knowledge is fully subsumed by another entry
+   - Entries about one-off incidents with no recurring applicability
+   - General advice available in any documentation
+4. PRESERVE:
+   - Entries describing non-obvious design decisions specific to this codebase
+   - Entries about recurring traps that a developer would hit again
+   - Entries that capture a hard-won gotcha with a concrete fix
+
+OUTPUT: A JSON array of "update" and "delete" ops only. No "create" ops — you are not
+extracting new knowledge, only consolidating existing knowledge.
+
+- "update": Replace content with a concise rewrite (under 150 words). Use to merge survivors or trim verbose entries.
+- "delete": Remove entries that are merged, stale, or low-value.
+
+Output ONLY valid JSON. No markdown fences, no explanation, no preamble.`;
+
+export function consolidationUser(input: {
+  entries: Array<{
+    id: string;
+    category: string;
+    title: string;
+    content: string;
+  }>;
+  targetMax: number;
+}): string {
+  const count = input.entries.length;
+  const listed = input.entries
+    .map((e) => `- [${e.id}] (${e.category}) ${e.title}: ${e.content}`)
+    .join("\n");
+  return `Current knowledge entries (${count} total, target max: ${input.targetMax}):
+
+${listed}
+
+Produce update/delete ops to reduce entry count to at most ${input.targetMax}. Prioritize merging related entries and trimming verbose ones over outright deletion.`;
 }
 
 // Format distillations for injection into the message context.
@@ -286,10 +359,9 @@ export function formatDistillations(
   return sections.join("\n\n");
 }
 
-// Rough token estimate used for budget-gating knowledge entries.
-// Consistent with gradient.ts: ~4 chars per token.
+// ~3 chars per token — validated as best heuristic against real API data.
 function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
+  return Math.ceil(text.length / 3);
 }
 
 export function formatKnowledge(

package/src/temporal.ts

@@ -1,9 +1,9 @@
 import { db, ensureProject } from "./db";
 import type { Message, Part } from "@opencode-ai/sdk";
 
-// Estimate token count from text length (rough: 1 token ≈ 4 chars)
+// ~3 chars per token — validated as best heuristic against real API data.
 function estimate(text: string): number {
-  return Math.ceil(text.length / 4);
+  return Math.ceil(text.length / 3);
 }
 
 function partsToText(parts: Part[]): string {