opencode-lore 0.1.0

package/src/index.ts

@@ -0,0 +1,324 @@
+import type { Plugin } from "@opencode-ai/plugin";
+import { load, config } from "./config";
+import { ensureProject } from "./db";
+import * as temporal from "./temporal";
+import * as ltm from "./ltm";
+import * as distillation from "./distillation";
+import * as curator from "./curator";
+import {
+  transform,
+  setModelLimits,
+  needsUrgentDistillation,
+  calibrate,
+  estimateMessages,
+} from "./gradient";
+import { formatKnowledge } from "./prompt";
+import { createRecallTool } from "./reflect";
+
+export const LorePlugin: Plugin = async (ctx) => {
+  const projectPath = ctx.worktree || ctx.directory;
+  await load(ctx.directory);
+  ensureProject(projectPath);
+
+  // Track user turns for periodic curation
+  let turnsSinceCuration = 0;
+
+  // Track active sessions for distillation
+  const activeSessions = new Set<string>();
+
+  // Sessions to skip for temporal storage and distillation. Includes worker sessions
+  // (distillation, curator) and child sessions (eval, any other children).
+  // Checked once per session ID and cached to avoid repeated API calls.
+  const skipSessions = new Set<string>();
+
+  async function shouldSkip(sessionID: string): Promise<boolean> {
+    if (distillation.isWorkerSession(sessionID)) return true;
+    if (skipSessions.has(sessionID)) return true;
+    if (activeSessions.has(sessionID)) return false; // already known good
+    // First encounter — check if this is a child session.
+    // session.get() uses exact storage key lookup and only works with full IDs
+    // (e.g. "ses_384e7de8dffeBDc4Z3dK9kfx1k"). Message events deliver short IDs
+    // (e.g. "ses_384e7de8dffe") which cause session.get() to fail with NotFound.
+    // Fall back to the session list to find a session whose full ID starts with
+    // the short ID, then check its parentID.
+    try {
+      const session = await ctx.client.session.get({ path: { id: sessionID } });
+      if (session.data?.parentID) {
+        skipSessions.add(sessionID);
+        return true;
+      }
+    } catch {
+      // session.get failed (likely short ID) — search list for matching full ID
+      try {
+        const list = await ctx.client.session.list();
+        const match = list.data?.find((s) => s.id.startsWith(sessionID));
+        if (match?.parentID) {
+          skipSessions.add(sessionID);
+          return true;
+        }
+      } catch {
+        // If we can't fetch session info, don't skip
+      }
+    }
+    return false;
+  }
+
+  // Background distillation — debounced, non-blocking
+  let distilling = false;
+  async function backgroundDistill(sessionID: string, force?: boolean) {
+    if (distilling) return;
+    distilling = true;
+    try {
+      const cfg = config();
+      const pending = temporal.undistilledCount(projectPath, sessionID);
+      if (
+        force ||
+        pending >= cfg.distillation.minMessages ||
+        needsUrgentDistillation()
+      ) {
+        await distillation.run({
+          client: ctx.client,
+          projectPath,
+          sessionID,
+          model: cfg.model,
+          force,
+        });
+      }
+    } catch (e) {
+      console.error("[lore] distillation error:", e);
+    } finally {
+      distilling = false;
+    }
+  }
+
+  async function backgroundCurate(sessionID: string) {
+    try {
+      const cfg = config();
+      if (!cfg.curator.enabled) return;
+      await curator.run({
+        client: ctx.client,
+        projectPath,
+        sessionID,
+        model: cfg.model,
+      });
+    } catch (e) {
+      console.error("[lore] curator error:", e);
+    }
+  }
+
+  return {
+    // Disable built-in compaction and register hidden worker agents
+    config: async (input) => {
+      const cfg = input as Record<string, unknown>;
+      cfg.compaction = { auto: false, prune: false };
+      cfg.agent = {
+        ...(cfg.agent as Record<string, unknown> | undefined),
+        "lore-distill": {
+          hidden: true,
+          description: "Lore memory distillation worker",
+        },
+        "lore-curator": {
+          hidden: true,
+          description: "Lore knowledge curator worker",
+        },
+      };
+    },
+
+    // Store all messages in temporal DB for full-text search and distillation.
+    // Skips child sessions (eval, worker) to prevent pollution.
+    event: async ({ event }) => {
+      if (event.type === "message.updated") {
+        const msg = event.properties.info;
+        if (await shouldSkip(msg.sessionID)) return;
+        try {
+          const full = await ctx.client.session.message({
+            path: { id: msg.sessionID, messageID: msg.id },
+          });
+          if (full.data) {
+            temporal.store({
+              projectPath,
+              info: full.data.info,
+              parts: full.data.parts,
+            });
+            activeSessions.add(msg.sessionID);
+            if (msg.role === "user") turnsSinceCuration++;
+
+            // Incremental distillation: when undistilled messages accumulate past
+            // maxSegment, distill immediately instead of waiting for session.idle.
+            if (
+              msg.role === "assistant" &&
+              msg.tokens &&
+              (msg.tokens.input > 0 || msg.tokens.cache.read > 0)
+            ) {
+              const pending = temporal.undistilledCount(projectPath, msg.sessionID);
+              if (pending >= config().distillation.maxSegment) {
+                console.error(
+                  `[lore] incremental distillation: ${pending} undistilled messages in ${msg.sessionID.substring(0, 16)}`,
+                );
+                backgroundDistill(msg.sessionID);
+              }
+
+              // Calibrate overhead estimate using real token counts
+              const allMsgs = await ctx.client.session.messages({
+                path: { id: msg.sessionID },
+              });
+              if (allMsgs.data) {
+                const withParts = allMsgs.data
+                  .filter((m) => m.info.id !== msg.id)
+                  .map((m) => ({ info: m.info, parts: m.parts }));
+                const msgEstimate = estimateMessages(withParts);
+                const actualInput = msg.tokens.input + msg.tokens.cache.read;
+                calibrate(actualInput, msgEstimate);
+              }
+            }
+          }
+        } catch {
+          // Message may not be fetchable yet during streaming
+        }
+      }
+
+      if (event.type === "session.idle") {
+        const sessionID = event.properties.sessionID;
+        if (await shouldSkip(sessionID)) return;
+        if (!activeSessions.has(sessionID)) return;
+
+        // Run background distillation for any remaining undistilled messages
+        backgroundDistill(sessionID);
+
+        // Run curator periodically
+        const cfg = config();
+        if (
+          cfg.curator.onIdle ||
+          turnsSinceCuration >= cfg.curator.afterTurns
+        ) {
+          backgroundCurate(sessionID);
+          turnsSinceCuration = 0;
+        }
+      }
+    },
+
+    // Inject LTM knowledge into system prompt
+    "experimental.chat.system.transform": async (input, output) => {
+      if (input.model?.limit) {
+        setModelLimits(input.model.limit);
+      }
+
+      const entries = ltm.forProject(projectPath, config().crossProject);
+      if (!entries.length) return;
+
+      const formatted = formatKnowledge(
+        entries.map((e) => ({
+          category: e.category,
+          title: e.title,
+          content: e.content,
+        })),
+      );
+      if (formatted) {
+        output.system.push(formatted);
+      }
+    },
+
+    // Transform message history: distilled prefix + raw recent
+    "experimental.chat.messages.transform": async (_input, output) => {
+      if (!output.messages.length) return;
+
+      const sessionID = output.messages[0]?.info.sessionID;
+
+      const lastUserMsg = [...output.messages].reverse().find((m) => m.info.role === "user");
+      const statsPart = lastUserMsg?.parts.find((p) => p.type === "text");
+
+      const result = transform({
+        messages: output.messages,
+        projectPath,
+        sessionID,
+      });
+      while (
+        result.messages.length > 0 &&
+        result.messages.at(-1)!.info.role !== "user"
+      ) {
+        const last = result.messages.at(-1)!;
+        if (last.parts.some((p) => p.type === "tool")) break;
+        const dropped = result.messages.pop()!;
+        console.error(
+          "[lore] WARN: dropping trailing",
+          dropped.info.role,
+          "message to prevent prefill error. id:",
+          dropped.info.id,
+        );
+      }
+      output.messages.splice(0, output.messages.length, ...result.messages);
+
+      if (result.layer >= 2 && sessionID) {
+        backgroundDistill(sessionID);
+      }
+
+      if (sessionID && statsPart && lastUserMsg) {
+        const loreMeta = {
+          layer: result.layer,
+          distilledTokens: result.distilledTokens,
+          rawTokens: result.rawTokens,
+          totalTokens: result.totalTokens,
+          usable: result.usable,
+          distilledBudget: result.distilledBudget,
+          rawBudget: result.rawBudget,
+          updatedAt: Date.now(),
+        };
+        const url = new URL(
+          `/session/${sessionID}/message/${lastUserMsg.info.id}/part/${statsPart.id}`,
+          ctx.serverUrl,
+        );
+        const updatedPart = {
+          ...(statsPart as Record<string, unknown>),
+          metadata: {
+            ...((statsPart as { metadata?: Record<string, unknown> }).metadata ?? {}),
+            lore: loreMeta,
+          },
+        };
+        fetch(url, {
+          method: "PATCH",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(updatedPart),
+        }).catch((e: unknown) => {
+          console.error("[lore] failed to write gradient stats to part metadata:", e);
+        });
+      }
+    },
+
+    // Replace compaction prompt with distillation-aware prompt when manual /compact is used
+    "experimental.session.compacting": async (input, output) => {
+      const entries = ltm.forProject(projectPath, config().crossProject);
+      const knowledge = entries.length
+        ? formatKnowledge(
+            entries.map((e) => ({
+              category: e.category,
+              title: e.title,
+              content: e.content,
+            })),
+          )
+        : "";
+
+      output.prompt = `You are creating a distilled memory summary for an AI coding agent. This summary will be the ONLY context available in the next part of the conversation.
+
+Structure your response as follows:
+
+## Session History
+
+For each major topic or task covered in the conversation, write:
+- A 1-3 sentence narrative of what happened (past tense, focus on outcomes)
+- A bullet list of specific, actionable facts (file paths, values, decisions, what failed and why)
+
+PRESERVE: file paths, specific values, decisions with rationale, user preferences, failed approaches with reasons, environment details.
+DROP: debugging back-and-forth, verbose tool output, pleasantries, redundant restatements.
+
+${knowledge ? `\n${knowledge}\n` : ""}
+End with "I'm ready to continue." so the agent knows to pick up where it left off.`;
+    },
+
+    // Register the recall tool
+    tool: {
+      recall: createRecallTool(projectPath),
+    },
+  };
+};
+
+export default LorePlugin;

package/src/ltm.ts

@@ -0,0 +1,186 @@
+import { db, ensureProject } from "./db";
+import { ftsQuery } from "./temporal";
+
+export type KnowledgeEntry = {
+  id: string;
+  project_id: string | null;
+  category: string;
+  title: string;
+  content: string;
+  source_session: string | null;
+  cross_project: number;
+  confidence: number;
+  created_at: number;
+  updated_at: number;
+  metadata: string | null;
+};
+
+export function create(input: {
+  projectPath?: string;
+  category: string;
+  title: string;
+  content: string;
+  session?: string;
+  scope: "project" | "global";
+  crossProject?: boolean;
+}): string {
+  const pid =
+    input.scope === "project" && input.projectPath
+      ? ensureProject(input.projectPath)
+      : null;
+  const id = crypto.randomUUID();
+  const now = Date.now();
+  db()
+    .query(
+      `INSERT INTO knowledge (id, project_id, category, title, content, source_session, cross_project, confidence, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, 1.0, ?, ?)`,
+    )
+    .run(
+      id,
+      pid,
+      input.category,
+      input.title,
+      input.content,
+      input.session ?? null,
+      (input.crossProject ?? true) ? 1 : 0,
+      now,
+      now,
+    );
+  return id;
+}
+
+export function update(
+  id: string,
+  input: { content?: string; confidence?: number },
+) {
+  const sets: string[] = [];
+  const params: unknown[] = [];
+  if (input.content !== undefined) {
+    sets.push("content = ?");
+    params.push(input.content);
+  }
+  if (input.confidence !== undefined) {
+    sets.push("confidence = ?");
+    params.push(input.confidence);
+  }
+  sets.push("updated_at = ?");
+  params.push(Date.now());
+  params.push(id);
+  db()
+    .query(`UPDATE knowledge SET ${sets.join(", ")} WHERE id = ?`)
+    .run(...(params as [string, ...string[]]));
+}
+
+export function remove(id: string) {
+  db().query("DELETE FROM knowledge WHERE id = ?").run(id);
+}
+
+export function forProject(
+  projectPath: string,
+  includeCross = true,
+): KnowledgeEntry[] {
+  const pid = ensureProject(projectPath);
+  if (includeCross) {
+    return db()
+      .query(
+        `SELECT * FROM knowledge
+         WHERE (project_id = ? OR (project_id IS NULL) OR (cross_project = 1))
+         AND confidence > 0.2
+         ORDER BY confidence DESC, updated_at DESC`,
+      )
+      .all(pid) as KnowledgeEntry[];
+  }
+  return db()
+    .query(
+      `SELECT * FROM knowledge
+       WHERE (project_id = ? OR project_id IS NULL)
+       AND confidence > 0.2
+       ORDER BY confidence DESC, updated_at DESC`,
+    )
+    .all(pid) as KnowledgeEntry[];
+}
+
+export function all(): KnowledgeEntry[] {
+  return db()
+    .query(
+      "SELECT * FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC",
+    )
+    .all() as KnowledgeEntry[];
+}
+
+// LIKE-based fallback for when FTS5 fails unexpectedly.
+function searchLike(input: {
+  query: string;
+  projectPath?: string;
+  limit: number;
+}): KnowledgeEntry[] {
+  const terms = input.query
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((t) => t.length > 2);
+  if (!terms.length) return [];
+  const conditions = terms
+    .map(() => "(LOWER(title) LIKE ? OR LOWER(content) LIKE ?)")
+    .join(" AND ");
+  const likeParams = terms.flatMap((t) => [`%${t}%`, `%${t}%`]);
+  if (input.projectPath) {
+    const pid = ensureProject(input.projectPath);
+    return db()
+      .query(
+        `SELECT * FROM knowledge WHERE (project_id = ? OR project_id IS NULL OR cross_project = 1) AND confidence > 0.2 AND ${conditions} ORDER BY updated_at DESC LIMIT ?`,
+      )
+      .all(pid, ...likeParams, input.limit) as KnowledgeEntry[];
+  }
+  return db()
+    .query(
+      `SELECT * FROM knowledge WHERE confidence > 0.2 AND ${conditions} ORDER BY updated_at DESC LIMIT ?`,
+    )
+    .all(...likeParams, input.limit) as KnowledgeEntry[];
+}
+
+export function search(input: {
+  query: string;
+  projectPath?: string;
+  limit?: number;
+}): KnowledgeEntry[] {
+  const limit = input.limit ?? 20;
+  const q = ftsQuery(input.query);
+  if (input.projectPath) {
+    const pid = ensureProject(input.projectPath);
+    try {
+      return db()
+        .query(
+          `SELECT k.* FROM knowledge k
+           WHERE k.rowid IN (SELECT rowid FROM knowledge_fts WHERE knowledge_fts MATCH ?)
+           AND (k.project_id = ? OR k.project_id IS NULL OR k.cross_project = 1)
+           AND k.confidence > 0.2
+           ORDER BY k.updated_at DESC LIMIT ?`,
+        )
+        .all(q, pid, limit) as KnowledgeEntry[];
+    } catch {
+      return searchLike({
+        query: input.query,
+        projectPath: input.projectPath,
+        limit,
+      });
+    }
+  }
+  try {
+    return db()
+      .query(
+        `SELECT k.* FROM knowledge k
+         WHERE k.rowid IN (SELECT rowid FROM knowledge_fts WHERE knowledge_fts MATCH ?)
+         AND k.confidence > 0.2
+         ORDER BY k.updated_at DESC LIMIT ?`,
+      )
+      .all(q, limit) as KnowledgeEntry[];
+  } catch {
+    return searchLike({ query: input.query, limit });
+  }
+}
+
+export function get(id: string): KnowledgeEntry | null {
+  return db()
+    .query("SELECT * FROM knowledge WHERE id = ?")
+    .get(id) as KnowledgeEntry | null;
+}

package/src/markdown.ts

@@ -0,0 +1,81 @@
+import { remark } from "remark";
+import type {
+  Root,
+  Heading,
+  List,
+  ListItem,
+  Paragraph,
+  Text,
+  Strong,
+  BlockContent,
+  PhrasingContent,
+} from "mdast";
+
+// Reuse a single processor — remark freezes on first use anyway
+const processor = remark();
+
+// Serialize an mdast tree to a markdown string.
+// The serializer automatically escapes any characters in text nodes
+// that would be structurally ambiguous (code fences, headings, list
+// markers, thematic breaks, etc.), so callers never need to pre-escape.
+export function serialize(tree: Root): string {
+  return processor.stringify(tree);
+}
+
+// Collapse newlines in LLM-generated text before inserting into a text node.
+// Embedded blank lines (\n\n) cause list items to become "spread" (loose),
+// which then breaks the surrounding markdown structure on re-parse.
+// Newlines within a single fact/narrative are replaced with a space.
+export function inline(value: string): string {
+  return value.replace(/\s*\n\s*/g, " ").trim();
+}
+
+// Normalize arbitrary markdown via parse → stringify roundtrip.
+// Used for content we don't control (e.g. existing text parts in Layer 4
+// after tool parts are stripped out), where we can't build from AST.
+// Two passes are needed: remark's asterisk/underscore escaping can introduce
+// new sequences on the first pass that the second pass then stabilizes.
+export function normalize(md: string): string {
+  const once = processor.stringify(processor.parse(md));
+  return processor.stringify(processor.parse(once));
+}
+
+// --- Node builders ---
+
+export function h(depth: 1 | 2 | 3 | 4 | 5 | 6, value: string): Heading {
+  return { type: "heading", depth, children: [t(value)] };
+}
+
+export function p(value: string): Paragraph {
+  return { type: "paragraph", children: [t(value)] };
+}
+
+export function ul(items: ListItem[]): List {
+  return { type: "list", ordered: false, spread: false, children: items };
+}
+
+export function li(...children: BlockContent[]): ListItem {
+  return { type: "listItem", spread: false, children };
+}
+
+// List item containing a single paragraph (the common case for facts/entries)
+export function lip(value: string): ListItem {
+  return li(p(value));
+}
+
+// List item with inline phrasing content — e.g. **bold**: text
+export function liph(...children: PhrasingContent[]): ListItem {
+  return li({ type: "paragraph", children });
+}
+
+export function t(value: string): Text {
+  return { type: "text", value };
+}
+
+export function strong(value: string): Strong {
+  return { type: "strong", children: [t(value)] };
+}
+
+export function root(...children: Root["children"]): Root {
+  return { type: "root", children };
+}