ex-brain 0.1.0 → 0.2.0

package/src/ai/timeline-extractor.ts

@@ -0,0 +1,403 @@
+import type { ResolvedLLM } from "../settings";
+import type { TimelineEntry } from "../types";
+import { callLLM, resolveApiKey, isLLMConfigured } from "./llm-client";
+import { jsonrepair } from "jsonrepair";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface TimelineExtractionInput {
+  /** Content to extract timeline from */
+  content: string;
+  /** Source identifier */
+  source: string;
+  /** Default date if no date found */
+  defaultDate: string;
+  /** Page slug for timeline entries */
+  pageSlug: string;
+}
+
+export interface TimelineExtractionResult {
+  /** Extracted timeline entries */
+  entries: TimelineEntry[];
+  /** Whether extraction succeeded */
+  success: boolean;
+  /** Confidence of extraction */
+  confidence: number;
+}
+
+export interface EventExtraction {
+  /** Event date (ISO or YYYY-MM-DD) */
+  date: string;
+  /** Event summary */
+  summary: string;
+  /** Event detail (optional) */
+  detail?: string;
+  /** Event type classification */
+  eventType: "milestone" | "update" | "meeting" | "announcement" | "transaction" | "other";
+  /** Importance score */
+  importance: number;
+}
+
+// ---------------------------------------------------------------------------
+// Timeline Extraction
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract timeline events from unstructured content.
+ * Handles various date formats and event descriptions.
+ */
+export async function extractTimelineEvents(
+  input: TimelineExtractionInput,
+  llm: ResolvedLLM,
+): Promise<TimelineExtractionResult> {
+  if (!isLLMConfigured(llm)) {
+    // Fallback: regex-based extraction
+    return fallbackExtract(input);
+  }
+
+  const prompt = buildExtractionPrompt(input);
+  const systemPrompt = "You are a timeline extraction assistant. Extract events from unstructured text. Always output valid JSON array. Be concise and factual.";
+  const resp = await callLLM(llm, prompt, 2048, systemPrompt);
+  
+  if (!resp) {
+    return fallbackExtract(input);
+  }
+
+  const entries = parseExtractionResponse(resp, input.pageSlug);
+  
+  return {
+    entries,
+    success: entries.length > 0,
+    confidence: entries.length > 0 ? 0.85 : 0.3,
+  };
+}
+
+/**
+ * Extract timeline events from entity relations.
+ * Used when processing entity-link extraction results.
+ */
+export async function extractTimelineFromRelation(
+  relation: {
+    from: string;
+    to: string;
+    relationType: string;
+    context: string;
+  },
+  defaultDate: string,
+  pageSlug: string,
+  llm: ResolvedLLM,
+): Promise<TimelineEntry | null> {
+  // Only extract timeline for significant relation types
+  const significantTypes = ["invested_in", "acquired", "founder_of", "leader_of", "works_at"];
+  if (!significantTypes.includes(relation.relationType)) {
+    return null;
+  }
+
+  const prompt = buildRelationTimelinePrompt(relation, defaultDate);
+  const systemPrompt = "You are a timeline extraction assistant. Extract events from relationships. Always output valid JSON array.";
+  const resp = await callLLM(llm, prompt, 512, systemPrompt);
+  
+  if (!resp) return null;
+
+  const entries = parseExtractionResponse(resp, pageSlug);
+  return entries[0] ?? null;
+}
+
+// ---------------------------------------------------------------------------
+// Prompts
+// ---------------------------------------------------------------------------
+
+function buildExtractionPrompt(input: TimelineExtractionInput): string {
+  return `Extract timeline events from this content.
+
+## Content
+Source: ${input.source}
+Default Date (use if no date found): ${input.defaultDate}
+Content:
+${input.content.slice(0, 4000)}
+
+## Task
+Extract ALL significant events worth recording in a timeline. Output ONLY JSON array.
+
+Schema:
+[
+  {
+    "date": "YYYY-MM-DD (extract from content or use default)",
+    "summary": "concise one-line summary (max 80 chars)",
+    "detail": "optional markdown detail",
+    "eventType": "milestone|update|meeting|announcement|transaction|other",
+    "importance": 1-5 (5 = most important)
+  }
+]
+
+Rules:
+1. Extract explicit dates from content (formats: "Jan 15", "2024-01-15", "1月15日", "last week", "yesterday", etc.)
+2. Convert relative dates to absolute using default date as reference
+3. Include: milestones, decisions, meetings, announcements, transactions, status changes
+4. Exclude: trivial mentions, routine activities, vague references
+5. Importance 5: founding, acquisition, major funding, product launch
+6. Importance 3-4: meetings, partnerships, minor updates
+7. Importance 1-2: minor mentions, routine status
+8. Max 5 entries, prioritized by importance
+9. Empty array if no significant events
+
+Examples:
+- "River AI closed Series A yesterday" → [{date: "${input.defaultDate}", summary: "River AI closed Series A funding", eventType: "transaction", importance: 5}]
+- "We met with the team on Jan 15" → [{date: "2025-01-15", summary: "Met with team", eventType: "meeting", importance: 3}]
+- "The company was founded in 2020" → [{date: "2020-01-01", summary: "Company founded", eventType: "milestone", importance: 5}]
+
+/no_think`;
+}
+
+function buildRelationTimelinePrompt(
+  relation: { from: string; to: string; relationType: string; context: string },
+  defaultDate: string,
+): string {
+  return `Create a timeline entry for this relationship event.
+
+## Relationship
+From: ${relation.from}
+To: ${relation.to}
+Type: ${relation.relationType}
+Context: ${relation.context}
+Default Date: ${defaultDate}
+
+## Task
+Output ONLY JSON array (single entry or empty).
+
+[
+  {
+    "date": "YYYY-MM-DD",
+    "summary": "concise summary (max 80 chars)",
+    "detail": "",
+    "eventType": "milestone|update|transaction",
+    "importance": 1-5
+  }
+]
+
+Rules:
+1. Extract date from context if mentioned
+2. Summarize the relationship event factually
+3. Empty array if context is vague or lacks timing
+
+Examples:
+- "John founded the company in 2019" → [{date: "2019-01-01", summary: "${relation.from} founded ${relation.to}", importance: 5}]
+- "She joined as CEO last month" → [{date: "${defaultDate}", summary: "${relation.from} became CEO of ${relation.to}", importance: 4}]
+
+/no_think`;
+}
+
+// ---------------------------------------------------------------------------
+// Response Parsing
+// ---------------------------------------------------------------------------
+
+function parseExtractionResponse(resp: string, pageSlug: string): TimelineEntry[] {
+  const match = resp.match(/\[[\s\S]*\]/);
+  if (!match) return [];
+
+  try {
+    // Use jsonrepair to fix common LLM JSON issues
+    const repaired = jsonrepair(match[0]);
+    const parsed = JSON.parse(repaired) as unknown[];
+    const entries: TimelineEntry[] = [];
+    
+    for (const e of parsed) {
+      if (typeof e !== "object" || e === null) continue;
+      const entry = e as Record<string, unknown>;
+      
+      const date = normalizeDate(String(entry.date ?? ""));
+      if (!date) continue;
+      
+      // Get importance from the response, default to 3
+      const importance = typeof entry.importance === "number" 
+        ? Math.max(1, Math.min(5, Math.round(entry.importance))) 
+        : 3;
+      
+      entries.push({
+        pageSlug,
+        date,
+        source: "extracted",
+        summary: String(entry.summary ?? "").slice(0, 120),
+        detail: String(entry.detail ?? ""),
+        importance,
+      });
+    }
+    
+    // Sort by date descending
+    entries.sort((a, b) => b.date.localeCompare(a.date));
+    
+    return entries.slice(0, 5); // Max 5 entries per extraction
+  } catch {
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Fallback Extraction (Regex-based)
+// ---------------------------------------------------------------------------
+
+function fallbackExtract(input: TimelineExtractionInput): TimelineExtractionResult {
+  const entries: TimelineEntry[] = [];
+  const content = input.content;
+  
+  // Common date patterns
+  const datePatterns = [
+    // ISO: 2024-01-15
+    /\b(\d{4}-\d{2}-\d{2})\b/g,
+    // Chinese: 2024年1月15日, 1月15日
+    /\b(\d{4}年\d{1,2}月\d{1,2}日)\b/g,
+    /\b(\d{1,2}月\d{1,2}日)\b/g,
+    // English: Jan 15, January 15, Jan 15th
+    /\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:st|nd|rd|th)?(?:,?\s+\d{4})?)\b/gi,
+    // Relative: yesterday, last week, last month
+    /\b(yesterday|last\s+week|last\s+month|recently)\b/gi,
+  ];
+
+  // Try to find dates and extract surrounding context
+  for (const pattern of datePatterns) {
+    const matches = content.matchAll(pattern);
+    for (const match of matches) {
+      if (!match[1]) continue;
+      
+      const rawDate = match[1];
+      const normalizedDate = normalizeDate(rawDate, input.defaultDate);
+      if (!normalizedDate) continue;
+      
+      // Extract context around the date (up to 100 chars before and after)
+      const start = Math.max(0, match.index! - 100);
+      const end = Math.min(content.length, match.index! + match[0].length + 100);
+      const context = content.slice(start, end).trim();
+      
+      // Create a summary from the context
+      const summary = context.slice(0, 80).replace(/\n+/g, " ").trim();
+      
+      if (summary.length > 10) {
+        entries.push({
+          pageSlug: input.pageSlug,
+          date: normalizedDate,
+          source: input.source,
+          summary,
+          detail: "",
+        });
+      }
+    }
+  }
+
+  // Deduplicate by date + summary similarity
+  const uniqueEntries = deduplicateEntries(entries);
+  
+  return {
+    entries: uniqueEntries,
+    success: uniqueEntries.length > 0,
+    confidence: 0.4, // Lower confidence for regex fallback
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Date Normalization
+// ---------------------------------------------------------------------------
+
+function normalizeDate(raw: string, defaultDate?: string): string {
+  const trimmed = raw.trim();
+  
+  // Already ISO format
+  if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) {
+    return trimmed;
+  }
+
+  // Chinese format: 2024年1月15日
+  const chineseMatch = trimmed.match(/(\d{4})年(\d{1,2})月(\d{1,2})日/);
+  if (chineseMatch) {
+    const [, year, month, day] = chineseMatch;
+    if (year && month && day) {
+      return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`;
+    }
+  }
+
+  // Chinese format without year: 1月15日
+  const chineseNoYearMatch = trimmed.match(/(\d{1,2})月(\d{1,2})日/);
+  if (chineseNoYearMatch && defaultDate) {
+    const [, month, day] = chineseNoYearMatch;
+    if (month && day) {
+      const year = defaultDate.slice(0, 4);
+      return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`;
+    }
+  }
+
+  // English month names
+  const monthMap: Record<string, string> = {
+    jan: "01", january: "01",
+    feb: "02", february: "02",
+    mar: "03", march: "03",
+    apr: "04", april: "04",
+    may: "05",
+    jun: "06", june: "06",
+    jul: "07", july: "07",
+    aug: "08", august: "08",
+    sep: "09", september: "09",
+    oct: "10", october: "10",
+    nov: "11", november: "11",
+    dec: "12", december: "12",
+  };
+
+  const englishMatch = trimmed.match(/(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?(?:,?\s+(\d{4}))?/i);
+  if (englishMatch) {
+    const [, monthName, day, year] = englishMatch;
+    if (monthName && day) {
+      const month = monthMap[monthName.toLowerCase().slice(0, 3)];
+      if (month) {
+        const finalYear = year || (defaultDate ? defaultDate.slice(0, 4) : new Date().getFullYear().toString());
+        return `${finalYear}-${month}-${day.padStart(2, "0")}`;
+      }
+    }
+  }
+
+  // Relative dates
+  if (/yesterday/i.test(trimmed) && defaultDate) {
+    const d = new Date(defaultDate);
+    d.setDate(d.getDate() - 1);
+    return d.toISOString().slice(0, 10);
+  }
+
+  if (/last\s+week/i.test(trimmed) && defaultDate) {
+    const d = new Date(defaultDate);
+    d.setDate(d.getDate() - 7);
+    return d.toISOString().slice(0, 10);
+  }
+
+  if (/last\s+month/i.test(trimmed) && defaultDate) {
+    const d = new Date(defaultDate);
+    d.setMonth(d.getMonth() - 1);
+    return d.toISOString().slice(0, 10);
+  }
+
+  if (/recently/i.test(trimmed) && defaultDate) {
+    return defaultDate;
+  }
+
+  // Default date fallback
+  if (defaultDate) {
+    return defaultDate;
+  }
+
+  return "";
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function deduplicateEntries(entries: TimelineEntry[]): TimelineEntry[] {
+  const seen = new Map<string, TimelineEntry>();
+  
+  for (const entry of entries) {
+    const key = `${entry.date}:${entry.summary.slice(0, 50)}`;
+    if (!seen.has(key)) {
+      seen.set(key, entry);
+    }
+  }
+  
+  return Array.from(seen.values());
+}

package/src/cli.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env bun
+import { buildProgram } from "./commands";
+
+async function main(): Promise<void> {
+  const program = buildProgram();
+  await program.parseAsync(process.argv);
+  // Force exit to avoid seekdb native library segfault on cleanup
+  // (seekdb has a bug where its native cleanup crashes on process exit)
+  process.exit(0);
+}
+
+main().catch((error: unknown) => {
+  const message = error instanceof Error ? error.message : String(error);
+  console.error(`[ebrain] ${message}`);
+  process.exit(1);
+});

package/src/commands/compile-cmd.ts

@@ -0,0 +1,208 @@
+import { Command } from "commander";
+import { basename } from "node:path";
+import { normalizeLongSlug, slugify } from "../config";
+import { readMaybeStdin, readTextFile } from "../markdown/io";
+import { loadSettings } from "../settings";
+import { BrainRepository } from "../repositories/brain-repo";
+import { BrainDb } from "../db/client";
+import { createProgress, formatDuration } from "../utils/progress";
+
+function isDryRun(opts: Record<string, unknown>): boolean {
+  return Boolean(opts.dryRun);
+}
+
+async function resolveInput(
+  fileOpt: string | undefined,
+  stdin: boolean,
+): Promise<string> {
+  if (fileOpt) return readTextFile(fileOpt);
+  return readMaybeStdin().then((s) => s ?? "");
+}
+
+async function withRepo(
+  program: Command,
+  callback: (repo: BrainRepository) => Promise<void>,
+): Promise<void> {
+  const settings = await loadSettings();
+  const cliDb = program.opts().db;
+  const dbPath = cliDb ?? settings.dbPath;
+  const db = await BrainDb.connect(dbPath, settings);
+  const repo = new BrainRepository(db);
+  await callback(repo);
+  process.exit(0);
+}
+
+function print(program: Command, payload: unknown): void {
+  if (program.opts().json) {
+    console.log(JSON.stringify(payload, null, 2));
+    return;
+  }
+  console.log(JSON.stringify(payload, null, 2));
+}
+
+export function registerCompileCommands(program: Command): void {
+  // -- compile (Smart Compilation)
+  program
+    .command("compile")
+    .argument("<slug>", "page slug")
+    .argument("<info>", "new information to compile")
+    .option("--source <source>", "source of information", "user")
+    .option("--date <date>", "date of information (YYYY-MM-DD)")
+    .option("--dry-run", "preview changes without executing", false)
+    .description("Intelligently compile new information into a page's compiled truth")
+    .addHelpText(
+      "after",
+      `
+Examples:
+  ebrain compile companies/river-ai "River AI closed Series A funding" --source meeting_notes
+  ebrain compile people/john "John joined as CEO last month" --date 2025-03-01
+`,
+    )
+    .action(async (slug: string, info: string, opts: { source?: string; date?: string; dryRun?: boolean }) => {
+      if (isDryRun(opts)) {
+        print(program, {
+          dryRun: true,
+          action: "compile",
+          slug,
+          info,
+          source: opts.source ?? "user",
+          date: opts.date ?? new Date().toISOString().slice(0, 10),
+        });
+        return;
+      }
+
+      await withRepo(program, async (repo) => {
+        const settings = await loadSettings();
+        const progress = createProgress();
+        
+        progress.start(`Compiling into ${slug}...`);
+        const startTime = Date.now();
+        
+        const result = await repo.compilePage(
+          slug,
+          info,
+          opts.source ?? "user",
+          opts.date ?? new Date().toISOString().slice(0, 10),
+          settings.llm,
+        );
+        
+        const duration = formatDuration(Date.now() - startTime);
+        
+        if (result.changed) {
+          progress.succeed(`${result.changeSummary} (${duration})`);
+        } else {
+          progress.stop();
+          process.stderr.write(`No changes made (${duration})\n`);
+        }
+        
+        print(program, {
+          ok: true,
+          action: "compile",
+          slug,
+          changed: result.changed,
+          changeType: result.changeType,
+          changeSummary: result.changeSummary,
+          timelineEntriesAdded: result.timelineEntries.length,
+          confidence: result.confidence,
+        });
+      });
+    });
+
+  // -- smart-ingest (Full Intelligent Ingestion)
+  program
+    .command("smart-ingest")
+    .argument("[slug]", "page slug (optional; auto-generated if omitted)")
+    .option("--file <path>", "read content from file")
+    .option("--stdin", "read content from stdin", false)
+    .option("--type <type>", "page type", "note")
+    .option("--title <title>", "page title")
+    .option("--source <source>", "source identifier", "ingest")
+    .option("--dry-run", "preview changes without executing", false)
+    .description("Full intelligent ingestion: compile truth, extract timeline, create entity links")
+    .addHelpText(
+      "after",
+      `
+Examples:
+  ebrain smart-ingest --file meeting.md --type meeting --source "meeting_notes"
+  ebrain smart-ingest companies/river-ai --file report.md --type company
+  cat article.md | ebrain smart-ingest --stdin --type article
+`,
+    )
+    .action(async (slug: string | undefined, opts: { file?: string; stdin?: boolean; type?: string; title?: string; source?: string; dryRun?: boolean }) => {
+      const input = await resolveInput(opts.file, opts.stdin ?? false);
+      if (!input.trim()) {
+        throw new Error("empty input — provide --file <path>, --stdin, or pipe content");
+      }
+
+      let finalSlug = slug;
+      if (!finalSlug) {
+        if (opts.file) {
+          const fileName = basename(opts.file).replace(/\.[^.]+$/i, "");
+          finalSlug = normalizeLongSlug(slugify(fileName));
+        } else if (opts.title) {
+          finalSlug = normalizeLongSlug(slugify(opts.title));
+        } else {
+          const timestamp = new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "");
+          finalSlug = `ingest/${timestamp}`;
+        }
+      }
+
+      if (isDryRun(opts)) {
+        print(program, {
+          dryRun: true,
+          action: "smart-ingest",
+          slug: finalSlug,
+          type: opts.type ?? "note",
+          source: opts.source ?? "ingest",
+          contentLength: input.length,
+        });
+        return;
+      }
+
+      await withRepo(program, async (repo) => {
+        const settings = await loadSettings();
+        const progress = createProgress();
+        const startTime = Date.now();
+        
+        progress.start(`Ingesting into ${finalSlug}...`);
+        
+        const result = await repo.ingestContent(
+          finalSlug,
+          input,
+          opts.source ?? "ingest",
+          opts.type ?? "note",
+          settings.llm,
+        );
+        
+        const duration = formatDuration(Date.now() - startTime);
+        
+        const parts = [];
+        if (result.compileResult.changed) parts.push(result.compileResult.changeSummary);
+        if (result.timelineResult.entries.length > 0) parts.push(`${result.timelineResult.entries.length} timeline entries`);
+        
+        if (parts.length > 0) {
+          progress.succeed(`${parts.join(", ")} (${duration})`);
+        } else {
+          progress.stop();
+          process.stderr.write(`No changes made (${duration})\n`);
+        }
+        
+        print(program, {
+          ok: true,
+          action: "smart-ingest",
+          slug: result.page.slug,
+          compile: {
+            changed: result.compileResult.changed,
+            changeType: result.compileResult.changeType,
+            changeSummary: result.compileResult.changeSummary,
+            confidence: result.compileResult.confidence,
+          },
+          timeline: {
+            entriesAdded: result.timelineResult.entries.length,
+            confidence: result.timelineResult.confidence,
+          },
+          updatedAt: result.page.updatedAt,
+        });
+      });
+    });
+}