@loreai/core 0.17.1 → 0.18.0

package/src/import/extract.ts

@@ -0,0 +1,137 @@
+/**
+ * Knowledge extraction from imported conversations.
+ *
+ * Takes conversation chunks and feeds them to the curator LLM to extract
+ * knowledge entries directly, without going through the temporal → distill
+ * pipeline. This is cheaper and faster than full-pipeline import.
+ */
+import * as ltm from "../ltm";
+import { parseOps, applyOps } from "../curator";
+import { CURATOR_SYSTEM, curatorUser } from "../prompt";
+import type { LLMClient } from "../types";
+import type { ConversationChunk } from "./types";
+
+/**
+ * System prompt for import extraction.
+ * Extends the standard curator prompt with guidance for historical conversations.
+ */
+const IMPORT_CURATOR_SYSTEM = `${CURATOR_SYSTEM}
+
+ADDITIONAL CONTEXT: You are extracting knowledge from HISTORICAL conversations with a different AI coding agent. Focus on durable insights that are still relevant:
+- Architecture decisions, design patterns, and project conventions
+- Gotchas, non-obvious bugs, and their fixes
+- Developer preferences and workflow patterns
+- Key technical choices and their rationale
+
+Ignore:
+- References to the other agent's specific capabilities or limitations
+- Task-specific state that is no longer current (e.g. "currently debugging X")
+- Debugging steps for issues that were already resolved
+- Transient conversation artifacts (greetings, acknowledgments, status updates)`;
+
+export type ExtractionProgress = {
+  /** Current chunk being processed (1-based) */
+  current: number;
+  /** Total chunks to process */
+  total: number;
+  /** Knowledge entries created so far */
+  created: number;
+  /** Knowledge entries updated (dedup hit) so far */
+  updated: number;
+};
+
+export type ExtractionResult = {
+  /** Total knowledge entries created */
+  created: number;
+  /** Total entries that hit dedup (updated existing) */
+  updated: number;
+  /** Total entries deleted */
+  deleted: number;
+  /** Chunks processed successfully */
+  chunksProcessed: number;
+  /** Chunks that failed (LLM error) */
+  chunksFailed: number;
+};
+
+/**
+ * Extract knowledge entries from conversation chunks via the curator LLM.
+ *
+ * Processes chunks sequentially (not parallel) to avoid rate limits
+ * and to let later chunks see entries created by earlier chunks
+ * (better dedup via the existing entries list in the prompt).
+ */
+export async function extractKnowledge(input: {
+  llm: LLMClient;
+  projectPath: string;
+  chunks: ConversationChunk[];
+  sessionID?: string;
+  model?: { providerID: string; modelID: string };
+  onProgress?: (progress: ExtractionProgress) => void;
+}): Promise<ExtractionResult> {
+  const result: ExtractionResult = {
+    created: 0,
+    updated: 0,
+    deleted: 0,
+    chunksProcessed: 0,
+    chunksFailed: 0,
+  };
+
+  // Sort chunks chronologically so knowledge builds up naturally
+  const sorted = [...input.chunks].sort((a, b) => a.timestamp - b.timestamp);
+
+  for (let i = 0; i < sorted.length; i++) {
+    const chunk = sorted[i];
+
+    // Get existing entries (refreshed each iteration for dedup)
+    const existing = ltm.forProject(input.projectPath, false);
+    const existingForPrompt = existing.map((e) => ({
+      id: e.id,
+      category: e.category,
+      title: e.title,
+      content: e.content,
+    }));
+
+    const userContent = curatorUser({
+      messages: chunk.text,
+      existing: existingForPrompt,
+    });
+
+    try {
+      const response = await input.llm.prompt(
+        IMPORT_CURATOR_SYSTEM,
+        userContent,
+        {
+          model: input.model,
+          workerID: "lore-import",
+          thinking: false,
+          maxTokens: 4096,
+          sessionID: input.sessionID,
+        },
+      );
+
+      if (response) {
+        const ops = parseOps(response);
+        const applied = applyOps(ops, {
+          projectPath: input.projectPath,
+          sessionID: input.sessionID,
+        });
+        result.created += applied.created;
+        result.updated += applied.updated;
+        result.deleted += applied.deleted;
+      }
+
+      result.chunksProcessed++;
+    } catch {
+      result.chunksFailed++;
+    }
+
+    input.onProgress?.({
+      current: i + 1,
+      total: sorted.length,
+      created: result.created,
+      updated: result.updated,
+    });
+  }
+
+  return result;
+}

package/src/import/history.ts

@@ -0,0 +1,99 @@
+/**
+ * Import history — tracks which external agent sessions have been imported
+ * to prevent re-importing unchanged sources.
+ */
+import { db, ensureProject } from "../db";
+
+export type ImportRecord = {
+  id: string;
+  project_id: string;
+  agent_name: string;
+  source_id: string;
+  source_hash: string;
+  entries_created: number;
+  entries_updated: number;
+  imported_at: number;
+};
+
+/**
+ * Check if a specific source has already been imported with the same hash.
+ *
+ * @returns The existing record if found with the same hash, or null if
+ *          the source hasn't been imported or the hash has changed.
+ */
+export function isImported(
+  projectPath: string,
+  agentName: string,
+  sourceId: string,
+  sourceHash: string,
+): ImportRecord | null {
+  const projectId = ensureProject(projectPath);
+  const row = db()
+    .query(
+      `SELECT * FROM import_history
+       WHERE project_id = ? AND agent_name = ? AND source_id = ?`,
+    )
+    .get(projectId, agentName, sourceId) as ImportRecord | null;
+
+  if (!row) return null;
+  // Hash changed — source has new content since last import
+  if (row.source_hash !== sourceHash) return null;
+  return row;
+}
+
+/**
+ * Record a successful import of a source.
+ * Uses INSERT OR REPLACE to handle re-imports of changed sources.
+ */
+export function recordImport(
+  projectPath: string,
+  agentName: string,
+  sourceId: string,
+  sourceHash: string,
+  stats: { created: number; updated: number },
+): void {
+  const projectId = ensureProject(projectPath);
+  db()
+    .query(
+      `INSERT OR REPLACE INTO import_history
+       (id, project_id, agent_name, source_id, source_hash, entries_created, entries_updated, imported_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      crypto.randomUUID(),
+      projectId,
+      agentName,
+      sourceId,
+      sourceHash,
+      stats.created,
+      stats.updated,
+      Date.now(),
+    );
+}
+
+/**
+ * Get all import records for a project.
+ * Excludes legacy "__declined__" sentinel rows from pre-v22 databases.
+ */
+export function listImports(projectPath: string): ImportRecord[] {
+  const projectId = ensureProject(projectPath);
+  return db()
+    .query(
+      `SELECT * FROM import_history
+       WHERE project_id = ? AND source_id != '__declined__'
+       ORDER BY imported_at DESC`,
+    )
+    .all(projectId) as ImportRecord[];
+}
+
+/**
+ * Compute a simple hash string for idempotency checks.
+ * Uses a fast non-cryptographic approach: file size + message count + last timestamp.
+ */
+export function computeHash(parts: {
+  size?: number;
+  messageCount?: number;
+  lastTimestamp?: number;
+}): string {
+  return `${parts.size ?? 0}:${parts.messageCount ?? 0}:${parts.lastTimestamp ?? 0}`;
+}

package/src/import/index.ts

@@ -0,0 +1,45 @@
+/**
+ * Conversation import system — detects and imports knowledge from
+ * external AI coding agent conversation histories.
+ */
+
+// Types
+export type {
+  ConversationChunk,
+  DetectedSession,
+  DetectionResult,
+  AgentHistoryProvider,
+} from "./types";
+
+// Detection
+export { detectAll } from "./detect";
+
+// Provider registry
+export {
+  registerProvider,
+  getProviders,
+  getProvider,
+  clearProviders,
+} from "./providers";
+
+// Extraction (lazy — avoid pulling in LLM/curator deps for detection-only use)
+export { extractKnowledge, type ExtractionProgress, type ExtractionResult } from "./extract";
+
+// Idempotency
+export {
+  isImported,
+  recordImport,
+  computeHash,
+  listImports,
+  type ImportRecord,
+} from "./history";
+
+// Register built-in providers on first import.
+// Each provider module calls registerProvider() at load time.
+import "./providers/claude-code";
+import "./providers/codex";
+import "./providers/opencode";
+import "./providers/cline";
+import "./providers/continue";
+import "./providers/pi";
+import "./providers/aider";

package/src/import/providers/aider.ts

@@ -0,0 +1,207 @@
+/**
+ * Aider conversation history provider.
+ *
+ * Reads from Aider's per-project chat history file:
+ *   <project-dir>/.aider.chat.history.md
+ *
+ * Format: Markdown with role headers like "#### user" / "#### assistant"
+ * separated by horizontal rules (---).
+ */
+import { existsSync, readFileSync, statSync } from "fs";
+import { join } from "path";
+import type { AgentHistoryProvider, ConversationChunk, DetectedSession } from "../types";
+import { registerProvider } from "./index";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const HISTORY_FILE = ".aider.chat.history.md";
+const DEFAULT_MAX_TOKENS = 12288;
+
+// Aider uses "#### role" headers and "---" separators
+const ROLE_HEADER_RE = /^####\s+(user|assistant|system)\s*$/i;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 3);
+}
+
+type ParsedMessage = {
+  role: string;
+  text: string;
+};
+
+/**
+ * Parse Aider's markdown chat history into messages.
+ *
+ * Format:
+ * ```
+ * #### user
+ * message text here
+ *
+ * #### assistant
+ * response text here
+ * ```
+ *
+ * Messages are separated by `---` or by new `#### role` headers.
+ */
+function parseAiderHistory(content: string): ParsedMessage[] {
+  const lines = content.split("\n");
+  const messages: ParsedMessage[] = [];
+  let currentRole: string | null = null;
+  let currentLines: string[] = [];
+
+  const flush = () => {
+    if (currentRole && currentLines.length > 0) {
+      const text = currentLines.join("\n").trim();
+      if (text) {
+        messages.push({ role: currentRole, text });
+      }
+    }
+    currentLines = [];
+  };
+
+  for (const line of lines) {
+    // Check for role header
+    const match = ROLE_HEADER_RE.exec(line);
+    if (match) {
+      flush();
+      currentRole = match[1].toLowerCase();
+      continue;
+    }
+
+    // Check for separator — starts a new conversation turn
+    if (line.trim() === "---") {
+      flush();
+      currentRole = null;
+      continue;
+    }
+
+    // Accumulate content if we're in a message
+    if (currentRole) {
+      currentLines.push(line);
+    }
+  }
+
+  // Flush final message
+  flush();
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+const aiderProvider: AgentHistoryProvider = {
+  name: "aider",
+  displayName: "Aider",
+
+  detect(projectPath: string): DetectedSession[] {
+    const filePath = join(projectPath, HISTORY_FILE);
+    if (!existsSync(filePath)) return [];
+
+    let stat;
+    try {
+      stat = statSync(filePath);
+    } catch {
+      return [];
+    }
+
+    if (!stat.isFile() || stat.size === 0) return [];
+
+    // Quick scan to count messages without full parsing
+    let content: string;
+    try {
+      content = readFileSync(filePath, "utf-8");
+    } catch {
+      return [];
+    }
+
+    const messages = parseAiderHistory(content);
+    if (messages.length < 3) return [];
+
+    const estimatedTokens = estimateTokens(content);
+
+    return [
+      {
+        id: filePath,
+        label: `Chat history (${messages.length} messages, ${Math.round(stat.size / 1024)}KB)`,
+        startedAt: stat.birthtimeMs || stat.ctimeMs,
+        lastActivityAt: stat.mtimeMs,
+        estimatedTokens,
+        messageCount: messages.length,
+      },
+    ];
+  },
+
+  readChunks(
+    projectPath: string,
+    sessionIds: string[],
+    maxTokens: number = DEFAULT_MAX_TOKENS,
+  ): ConversationChunk[] {
+    const chunks: ConversationChunk[] = [];
+
+    for (const filePath of sessionIds) {
+      let content: string;
+      try {
+        content = readFileSync(filePath, "utf-8");
+      } catch {
+        continue;
+      }
+
+      const messages = parseAiderHistory(content);
+      if (messages.length === 0) continue;
+
+      // Get file mtime for timestamp
+      let fileTimestamp: number;
+      try {
+        fileTimestamp = statSync(filePath).mtimeMs;
+      } catch {
+        fileTimestamp = Date.now();
+      }
+
+      // Build chunks respecting maxTokens boundaries
+      let currentTexts: string[] = [];
+      let currentTokens = 0;
+      let chunkIndex = 0;
+
+      const flushChunk = () => {
+        if (currentTexts.length === 0) return;
+        chunkIndex++;
+        const text = currentTexts.join("\n\n");
+        chunks.push({
+          label: `Aider history (${chunkIndex})`,
+          text,
+          estimatedTokens: estimateTokens(text),
+          timestamp: fileTimestamp,
+        });
+        currentTexts = [];
+        currentTokens = 0;
+      };
+
+      for (const msg of messages) {
+        const formatted = `[${msg.role}] ${msg.text}`;
+        const msgTokens = estimateTokens(formatted);
+
+        if (currentTokens > 0 && currentTokens + msgTokens > maxTokens) {
+          flushChunk();
+        }
+
+        currentTexts.push(formatted);
+        currentTokens += msgTokens;
+      }
+
+      flushChunk();
+    }
+
+    return chunks;
+  },
+};
+
+// Auto-register on import
+registerProvider(aiderProvider);