@loreai/core 0.17.1 → 0.18.0

package/src/import/providers/opencode.ts

@@ -0,0 +1,272 @@
+/**
+ * OpenCode conversation history provider.
+ *
+ * Reads from OpenCode's SQLite database at ~/.local/share/opencode/opencode.db.
+ * The message.data and part.data JSON fields use a format very close to lore's
+ * own LoreMessage/LorePart types (since lore was designed for OpenCode).
+ */
+import { existsSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import { Database } from "#db/driver";
+import type { AgentHistoryProvider, ConversationChunk, DetectedSession } from "../types";
+import { registerProvider } from "./index";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const OPENCODE_DB_PATH = join(
+  process.env.XDG_DATA_HOME || join(homedir(), ".local", "share"),
+  "opencode",
+  "opencode.db",
+);
+const MAX_TOOL_OUTPUT_CHARS = 500;
+const DEFAULT_MAX_TOKENS = 12288;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 3);
+}
+
+function truncate(text: string, max: number): string {
+  if (text.length <= max) return text;
+  return text.slice(0, max) + "...";
+}
+
+/**
+ * Open OpenCode's database read-only.
+ * Returns null if the DB doesn't exist or can't be opened.
+ *
+ * Bun's `Database` uses `{ readonly: true }` while Node.js's `DatabaseSync`
+ * uses `{ readOnly: true }`. We pass both via a cast to cover both runtimes.
+ */
+function openDB(): InstanceType<typeof Database> | null {
+  if (!existsSync(OPENCODE_DB_PATH)) return null;
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return new Database(OPENCODE_DB_PATH, { readonly: true, readOnly: true } as any);
+  } catch {
+    return null;
+  }
+}
+
+/** Check if a table exists in the database. */
+function tableExists(database: InstanceType<typeof Database>, table: string): boolean {
+  const row = database
+    .query("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
+    .get(table) as { name: string } | null;
+  return row != null;
+}
+
+type PartData = {
+  type: string;
+  text?: string;
+  tool?: string;
+  state?: {
+    status: string;
+    output?: string;
+  };
+};
+
+/** Convert part data rows into conversation text. */
+function partsToConversationText(parts: PartData[]): string {
+  const segments: string[] = [];
+  for (const part of parts) {
+    if (part.type === "text" && part.text) {
+      segments.push(part.text);
+    } else if (part.type === "tool" && part.tool && part.state?.status === "completed" && part.state.output) {
+      segments.push(`[tool: ${part.tool}] ${truncate(part.state.output, MAX_TOOL_OUTPUT_CHARS)}`);
+    }
+    // Skip reasoning, step-start, and other non-text parts
+  }
+  return segments.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+const opencodeProvider: AgentHistoryProvider = {
+  name: "opencode",
+  displayName: "OpenCode",
+
+  detect(projectPath: string): DetectedSession[] {
+    const database = openDB();
+    if (!database) return [];
+
+    try {
+      // Check required tables exist
+      if (!tableExists(database, "project") || !tableExists(database, "session") || !tableExists(database, "message")) {
+        return [];
+      }
+
+      // Find the project by worktree path
+      const project = database
+        .query("SELECT id FROM project WHERE worktree = ?")
+        .get(projectPath) as { id: string } | null;
+      if (!project) return [];
+
+      // Get sessions with message counts
+      const sessions = database
+        .query(
+          `SELECT s.id, s.title, s.time_created, s.time_updated,
+                  (SELECT COUNT(*) FROM message m WHERE m.session_id = s.id) as msg_count
+           FROM session s
+           WHERE s.project_id = ? AND s.parent_id IS NULL
+           ORDER BY s.time_updated DESC`,
+        )
+        .all(project.id) as Array<{
+        id: string;
+        title: string;
+        time_created: number;
+        time_updated: number;
+        msg_count: number;
+      }>;
+
+      const results: DetectedSession[] = [];
+      for (const sess of sessions) {
+        // Skip trivially small sessions
+        if (sess.msg_count < 3) continue;
+
+        // Estimate tokens from message count (rough: ~500 tokens/message avg)
+        const estimatedTokens = sess.msg_count * 500;
+        const dateStr = new Date(sess.time_created).toISOString().slice(0, 10);
+        const label = sess.title
+          ? `${dateStr} - ${sess.title} (${sess.msg_count} messages)`
+          : `${dateStr} (${sess.msg_count} messages)`;
+
+        results.push({
+          id: sess.id,
+          label,
+          startedAt: sess.time_created,
+          lastActivityAt: sess.time_updated,
+          estimatedTokens,
+          messageCount: sess.msg_count,
+        });
+      }
+
+      return results;
+    } finally {
+      database.close();
+    }
+  },
+
+  readChunks(
+    _projectPath: string,
+    sessionIds: string[],
+    maxTokens: number = DEFAULT_MAX_TOKENS,
+  ): ConversationChunk[] {
+    const database = openDB();
+    if (!database) return [];
+
+    const chunks: ConversationChunk[] = [];
+
+    try {
+      const hasParts = tableExists(database, "part");
+
+      for (const sessionId of sessionIds) {
+        // Get messages ordered by time
+        const messages = database
+          .query(
+            `SELECT id, data, time_created FROM message
+             WHERE session_id = ?
+             ORDER BY time_created ASC`,
+          )
+          .all(sessionId) as Array<{
+          id: string;
+          data: string;
+          time_created: number;
+        }>;
+
+        if (messages.length === 0) continue;
+
+        const textMessages: { text: string; timestamp: number }[] = [];
+
+        for (const msg of messages) {
+          let msgData: { role?: string };
+          try {
+            msgData = JSON.parse(msg.data);
+          } catch {
+            continue;
+          }
+
+          const role = msgData.role ?? "unknown";
+          let contentText = "";
+
+          if (hasParts) {
+            // Read parts for this message
+            const parts = database
+              .query(
+                `SELECT data FROM part
+                 WHERE message_id = ?
+                 ORDER BY time_created ASC`,
+              )
+              .all(msg.id) as Array<{ data: string }>;
+
+            const parsedParts: PartData[] = [];
+            for (const p of parts) {
+              try {
+                parsedParts.push(JSON.parse(p.data) as PartData);
+              } catch {
+                // Skip malformed parts
+              }
+            }
+            contentText = partsToConversationText(parsedParts);
+          }
+
+          if (!contentText.trim()) continue;
+
+          textMessages.push({
+            text: `[${role}] ${contentText}`,
+            timestamp: msg.time_created,
+          });
+        }
+
+        if (textMessages.length === 0) continue;
+
+        // Build chunks respecting maxTokens boundaries
+        let currentTexts: string[] = [];
+        let currentTokens = 0;
+        let chunkStart = textMessages[0].timestamp;
+        let chunkIndex = 0;
+
+        const flushChunk = () => {
+          if (currentTexts.length === 0) return;
+          chunkIndex++;
+          const text = currentTexts.join("\n\n");
+          chunks.push({
+            label: `OpenCode ${new Date(chunkStart).toISOString().slice(0, 10)} (${chunkIndex})`,
+            text,
+            estimatedTokens: estimateTokens(text),
+            timestamp: chunkStart,
+          });
+          currentTexts = [];
+          currentTokens = 0;
+        };
+
+        for (const msg of textMessages) {
+          const msgTokens = estimateTokens(msg.text);
+          if (currentTokens > 0 && currentTokens + msgTokens > maxTokens) {
+            flushChunk();
+            chunkStart = msg.timestamp;
+          }
+          currentTexts.push(msg.text);
+          currentTokens += msgTokens;
+        }
+
+        flushChunk();
+      }
+    } finally {
+      database.close();
+    }
+
+    return chunks;
+  },
+};
+
+// Auto-register on import
+registerProvider(opencodeProvider);

package/src/import/providers/pi.ts

@@ -0,0 +1,332 @@
+/**
+ * Pi coding agent conversation history provider.
+ *
+ * Reads JSONL session files from ~/.pi/agent/sessions/<encoded-cwd>/<timestamp>_<uuid>.jsonl
+ *
+ * CWD encoding: "--<cwd-with-slashes-replaced-by-dashes>--"
+ * e.g. /home/byk/Code/foo → --home-byk-Code-foo--
+ *
+ * Pi uses a tree-structured session format where each entry has id/parentId.
+ * We reconstruct the linear conversation by following the chain from root to
+ * the latest leaf.
+ */
+import { readdirSync, readFileSync, existsSync, statSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+import type { AgentHistoryProvider, ConversationChunk, DetectedSession } from "../types";
+import { registerProvider } from "./index";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const PI_DIR = join(homedir(), ".pi", "agent", "sessions");
+const MAX_TOOL_OUTPUT_CHARS = 500;
+const DEFAULT_MAX_TOKENS = 12288;
+
+// ---------------------------------------------------------------------------
+// JSONL types (only the fields we read)
+// ---------------------------------------------------------------------------
+
+type PiLine =
+  | {
+      type: "session";
+      id: string;
+      timestamp: string;
+      cwd: string;
+      version?: number;
+    }
+  | {
+      type: "message";
+      id: string;
+      parentId: string;
+      timestamp: string;
+      message: {
+        role: "user" | "assistant";
+        content: string;
+        provider?: string;
+        model?: string;
+      };
+    }
+  | {
+      type: "compaction";
+      id: string;
+      parentId: string;
+      summary: string;
+    }
+  | {
+      type: "custom" | "custom_message";
+      id: string;
+      parentId: string;
+      customType?: string;
+      content?: string;
+    }
+  | {
+      type: string;
+      id?: string;
+      parentId?: string;
+      timestamp?: string;
+    };
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 3);
+}
+
+function truncate(text: string, max: number): string {
+  if (text.length <= max) return text;
+  return text.slice(0, max) + "...";
+}
+
+/**
+ * Encode a CWD path to Pi's directory naming convention.
+ * /home/byk/Code/foo → --home-byk-Code-foo--
+ */
+function encodeCwd(cwd: string): string {
+  // Remove leading slash, replace remaining slashes with dashes
+  const encoded = cwd.replace(/^\//, "").replace(/\//g, "-");
+  return `--${encoded}--`;
+}
+
+/** Parse a JSONL file into typed lines. */
+function parseJSONL(filePath: string): PiLine[] {
+  let raw: string;
+  try {
+    raw = readFileSync(filePath, "utf-8");
+  } catch {
+    return [];
+  }
+
+  const lines: PiLine[] = [];
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      lines.push(JSON.parse(line) as PiLine);
+    } catch {
+      // Skip malformed
+    }
+  }
+  return lines;
+}
+
+/**
+ * Reconstruct the linear conversation from tree-structured entries.
+ *
+ * Pi sessions use a tree structure where each entry has id/parentId.
+ * Multiple branches can exist (e.g. user edits a message, creating a fork).
+ * We follow the longest path from root to the latest leaf to get the
+ * "main" conversation thread.
+ */
+function linearize(lines: PiLine[]): PiLine[] {
+  if (lines.length === 0) return [];
+
+  // Build adjacency map: parentId → children
+  const children = new Map<string, PiLine[]>();
+  const byId = new Map<string, PiLine>();
+  let rootLine: PiLine | null = null;
+
+  for (const line of lines) {
+    if (line.type === "session") {
+      rootLine = line;
+      continue;
+    }
+
+    if (!line.id) continue;
+    byId.set(line.id, line);
+
+    const pid = (line as { parentId?: string }).parentId;
+    if (pid) {
+      const siblings = children.get(pid) ?? [];
+      siblings.push(line);
+      children.set(pid, siblings);
+    }
+  }
+
+  if (!rootLine || !rootLine.id) return lines.filter((l) => l.type === "message");
+
+  // Walk from root, always picking the child with the latest timestamp
+  // (or the last one appended if timestamps are equal)
+  const result: PiLine[] = [];
+  let currentId: string | undefined = rootLine.id;
+
+  while (currentId) {
+    const kids = children.get(currentId);
+    if (!kids || kids.length === 0) break;
+
+    // Pick the last child (append-only means last is most recent on the main branch)
+    const next = kids[kids.length - 1];
+    result.push(next);
+    currentId = next.id;
+  }
+
+  return result;
+}
+
+/** Get session metadata from a JSONL file. */
+function getSessionMeta(
+  filePath: string,
+): {
+  id: string;
+  cwd: string;
+  timestamp: number;
+  messageCount: number;
+  fileSize: number;
+} | null {
+  const lines = parseJSONL(filePath);
+  if (lines.length === 0) return null;
+
+  const header = lines[0];
+  if (header.type !== "session") return null;
+  const session = header as Extract<PiLine, { type: "session" }>;
+
+  const messageCount = lines.filter((l) => l.type === "message").length;
+  let fileSize: number;
+  try {
+    fileSize = statSync(filePath).size;
+  } catch {
+    fileSize = 0;
+  }
+
+  const ts = new Date(session.timestamp).getTime();
+
+  return {
+    id: session.id,
+    cwd: session.cwd,
+    timestamp: Number.isNaN(ts) ? Date.now() : ts,
+    messageCount,
+    fileSize,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+const piProvider: AgentHistoryProvider = {
+  name: "pi",
+  displayName: "Pi",
+
+  detect(projectPath: string): DetectedSession[] {
+    const encoded = encodeCwd(projectPath);
+    const dir = join(PI_DIR, encoded);
+
+    let entries: string[];
+    try {
+      entries = readdirSync(dir);
+    } catch {
+      return []; // Directory doesn't exist
+    }
+
+    const sessions: DetectedSession[] = [];
+    for (const entry of entries) {
+      if (!entry.endsWith(".jsonl")) continue;
+
+      const filePath = join(dir, entry);
+      const meta = getSessionMeta(filePath);
+      if (!meta) continue;
+
+      // Skip trivially small sessions
+      if (meta.messageCount < 3) continue;
+
+      const dateStr = new Date(meta.timestamp).toISOString().slice(0, 10);
+      const estimatedTokens = Math.ceil(meta.fileSize / 5);
+
+      sessions.push({
+        id: filePath,
+        label: `${dateStr} (${meta.messageCount} messages)`,
+        startedAt: meta.timestamp,
+        lastActivityAt: meta.timestamp,
+        estimatedTokens,
+        messageCount: meta.messageCount,
+      });
+    }
+
+    return sessions.sort((a, b) => b.lastActivityAt - a.lastActivityAt);
+  },
+
+  readChunks(
+    _projectPath: string,
+    sessionIds: string[],
+    maxTokens: number = DEFAULT_MAX_TOKENS,
+  ): ConversationChunk[] {
+    const chunks: ConversationChunk[] = [];
+
+    for (const filePath of sessionIds) {
+      const allLines = parseJSONL(filePath);
+      const linearLines = linearize(allLines);
+
+      // Find session timestamp for labeling
+      let sessionTimestamp = Date.now();
+      const header = allLines.find((l) => l.type === "session");
+      if (header?.type === "session") {
+        const session = header as Extract<PiLine, { type: "session" }>;
+        const ts = new Date(session.timestamp).getTime();
+        if (!Number.isNaN(ts)) sessionTimestamp = ts;
+      }
+
+      const messages: { text: string; timestamp: number }[] = [];
+
+      for (const line of linearLines) {
+        if (line.type === "message") {
+          const msg = line as Extract<PiLine, { type: "message" }>;
+          const content = msg.message.content;
+          if (!content) continue;
+
+          const ts = new Date(msg.timestamp).getTime();
+          messages.push({
+            text: `[${msg.message.role}] ${content}`,
+            timestamp: Number.isNaN(ts) ? sessionTimestamp : ts,
+          });
+        } else if (line.type === "compaction") {
+          const comp = line as Extract<PiLine, { type: "compaction" }>;
+          if (comp.summary) {
+            messages.push({
+              text: `[summary] ${truncate(comp.summary, MAX_TOOL_OUTPUT_CHARS * 2)}`,
+              timestamp: sessionTimestamp,
+            });
+          }
+        }
+      }
+
+      if (messages.length === 0) continue;
+
+      // Build chunks respecting maxTokens boundaries
+      let currentTexts: string[] = [];
+      let currentTokens = 0;
+      let chunkIndex = 0;
+
+      const flushChunk = () => {
+        if (currentTexts.length === 0) return;
+        chunkIndex++;
+        const text = currentTexts.join("\n\n");
+        chunks.push({
+          label: `Pi ${new Date(sessionTimestamp).toISOString().slice(0, 10)} (${chunkIndex})`,
+          text,
+          estimatedTokens: estimateTokens(text),
+          timestamp: sessionTimestamp,
+        });
+        currentTexts = [];
+        currentTokens = 0;
+      };
+
+      for (const msg of messages) {
+        const msgTokens = estimateTokens(msg.text);
+        if (currentTokens > 0 && currentTokens + msgTokens > maxTokens) {
+          flushChunk();
+        }
+        currentTexts.push(msg.text);
+        currentTokens += msgTokens;
+      }
+
+      flushChunk();
+    }
+
+    return chunks;
+  },
+};
+
+// Auto-register on import
+registerProvider(piProvider);

package/src/import/types.ts

@@ -0,0 +1,91 @@
+/**
+ * Types for the conversation import system.
+ *
+ * The import system detects and reads conversation history from external
+ * AI coding agents (Claude Code, OpenCode, Aider, etc.) and extracts
+ * long-term knowledge entries via the curator LLM prompt.
+ */
+
+/**
+ * A chunk of conversation text from a foreign agent, ready for
+ * knowledge extraction. Chunks are sized to fit within a single
+ * LLM curator call (~12K tokens).
+ */
+export type ConversationChunk = {
+  /** Human-readable label, e.g. "Claude Code session 2025-05-10 (1 of 3)" */
+  label: string;
+  /** Plain text of the conversation (role-prefixed lines) */
+  text: string;
+  /** Estimated token count (~text.length / 3) */
+  estimatedTokens: number;
+  /** When this chunk's messages were created (epoch ms), for sorting */
+  timestamp: number;
+};
+
+/**
+ * A detected session from a foreign agent.
+ */
+export type DetectedSession = {
+  /** Unique identifier (agent-specific: file path, session UUID, etc.) */
+  id: string;
+  /** Human-readable label */
+  label: string;
+  /** When this session started (epoch ms) */
+  startedAt: number;
+  /** When this session last had activity (epoch ms) */
+  lastActivityAt: number;
+  /** Estimated total tokens across all messages */
+  estimatedTokens: number;
+  /** Number of conversation messages (user + assistant) */
+  messageCount: number;
+};
+
+/**
+ * Summary of what was detected for a project from one agent.
+ */
+export type DetectionResult = {
+  /** Internal agent name (e.g. "claude-code", "opencode", "aider") */
+  agentName: string;
+  /** Human-readable agent name (e.g. "Claude Code") */
+  agentDisplayName: string;
+  /** Sessions found for this project */
+  sessions: DetectedSession[];
+  /** Total estimated tokens across all sessions */
+  totalTokens: number;
+  /** Total messages across all sessions */
+  totalMessages: number;
+};
+
+/**
+ * Adapter interface for reading conversation history from a specific
+ * AI coding agent. Each agent implements this interface.
+ */
+export interface AgentHistoryProvider {
+  /** Internal name (e.g. "claude-code", "opencode", "aider") */
+  readonly name: string;
+  /** Display name (e.g. "Claude Code") */
+  readonly displayName: string;
+
+  /**
+   * Detect whether this agent has conversation history for the given
+   * project path. Should be a fast check — avoid reading full file contents.
+   *
+   * @returns Array of detected sessions, or empty array if none found.
+   */
+  detect(projectPath: string): DetectedSession[];
+
+  /**
+   * Read conversation text from specific sessions, chunked for LLM
+   * consumption. Each chunk is <= maxTokens estimated tokens.
+   *
+   * @param projectPath  The project directory
+   * @param sessionIds   Which sessions to import (from detect() results)
+   * @param maxTokens    Max estimated tokens per chunk (default: 12288)
+   * @returns Conversation chunks ready for knowledge extraction
+   */
+  readChunks(
+    projectPath: string,
+    sessionIds: string[],
+    maxTokens?: number,
+  ): ConversationChunk[];
+}