ei-tui 0.1.10 → 0.1.13

package/src/integrations/claude-code/importer.ts

@@ -0,0 +1,323 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { Ei_Interface, Topic, Message, ContextStatus, PersonaEntity } from "../../core/types.js";
+import type { IClaudeCodeReader, ClaudeCodeSession, ClaudeCodeMessage } from "./types.js";
+import {
+  CLAUDE_CODE_PERSONA_NAME,
+  CLAUDE_CODE_TOPIC_GROUPS,
+  MIN_SESSION_AGE_MS,
+} from "./types.js";
+import { ClaudeCodeReader } from "./reader.js";
+import {
+  queueAllScans,
+  type ExtractionContext,
+} from "../../core/orchestrators/human-extraction.js";
+
+// =============================================================================
+// Export Types
+// =============================================================================
+
+export interface ClaudeCodeImportResult {
+  sessionsProcessed: number;
+  topicsCreated: number;
+  topicsUpdated: number;
+  messagesImported: number;
+  personaCreated: boolean;
+  extractionScansQueued: number;
+}
+
+export interface ClaudeCodeImporterOptions {
+  stateManager: StateManager;
+  interface?: Ei_Interface;
+  reader?: IClaudeCodeReader;
+  signal?: AbortSignal;
+}
+
+// =============================================================================
+// Utility Functions
+// =============================================================================
+
+const TWELVE_HOURS_MS = 43_200_000;
+const CLAUDE_CODE_GROUP = "Claude Code";
+
+function convertToEiMessage(msg: ClaudeCodeMessage): Message {
+  return {
+    id: msg.id,
+    role: msg.role === "user" ? "human" : "system",
+    verbal_response: msg.content,
+    timestamp: msg.timestamp,
+    read: true,
+    context_status: "default" as ContextStatus,
+  };
+}
+
+function convertToPreMarkedEiMessage(msg: ClaudeCodeMessage): Message {
+  return {
+    ...convertToEiMessage(msg),
+    f: true,
+    r: true,
+    p: true,
+    o: true,
+  };
+}
+
+/**
+ * Ensure the single "Claude Code" persona exists.
+ * All sessions share one persona — it's a coding assistant, not a multi-agent system.
+ */
+function ensureClaudeCodePersona(
+  stateManager: StateManager,
+  eiInterface?: Ei_Interface
+): PersonaEntity {
+  const existing = stateManager.persona_getByName(CLAUDE_CODE_PERSONA_NAME);
+  if (existing) return existing;
+
+  const now = new Date().toISOString();
+  const persona: PersonaEntity = {
+    id: crypto.randomUUID(),
+    display_name: CLAUDE_CODE_PERSONA_NAME,
+    entity: "system",
+    aliases: ["claude-code", "claude code"],
+    short_description: "Claude Code — Anthropic's AI coding assistant",
+    long_description:
+      "Claude Code is an agentic coding assistant that helps with coding tasks, debugging, architecture decisions, and more.",
+    group_primary: CLAUDE_CODE_GROUP,
+    groups_visible: [CLAUDE_CODE_GROUP],
+    traits: [],
+    topics: [],
+    is_paused: false,
+    is_archived: false,
+    is_static: false,
+    heartbeat_delay_ms: TWELVE_HOURS_MS,
+    last_heartbeat: now,
+    last_updated: now,
+    last_activity: now,
+  };
+
+  stateManager.persona_add(persona);
+  eiInterface?.onPersonaAdded?.();
+  return persona;
+}
+
+// =============================================================================
+// Topic Management
+// =============================================================================
+
+function ensureSessionTopic(
+  session: ClaudeCodeSession,
+  stateManager: StateManager
+): "created" | "updated" | "unchanged" {
+  const human = stateManager.getHuman();
+  const existingTopic = human.topics.find((t) => t.id === session.id);
+
+  if (existingTopic) {
+    if (existingTopic.name !== session.title) {
+      const updatedTopic: Topic = {
+        ...existingTopic,
+        name: session.title,
+        last_updated: new Date().toISOString(),
+      };
+      stateManager.human_topic_upsert(updatedTopic);
+      return "updated";
+    }
+    return "unchanged";
+  }
+
+  const newTopic: Topic = {
+    id: session.id,
+    name: session.title,
+    description: `Claude Code session in ${session.cwd}`,
+    sentiment: 0,
+    exposure_current: 0.5,
+    exposure_desired: 0.3,
+    persona_groups: CLAUDE_CODE_TOPIC_GROUPS,
+    learned_by: CLAUDE_CODE_PERSONA_NAME,
+    last_updated: new Date().toISOString(),
+  };
+
+  stateManager.human_topic_upsert(newTopic);
+  return "created";
+}
+
+// =============================================================================
+// State Helpers
+// =============================================================================
+
+function updateProcessedState(
+  stateManager: StateManager,
+  session: ClaudeCodeSession
+): void {
+  const human = stateManager.getHuman();
+  const processedSessions = {
+    ...(human.settings?.claudeCode?.processed_sessions ?? {}),
+    [session.id]: new Date().toISOString(),
+  };
+
+  stateManager.setHuman({
+    ...human,
+    settings: {
+      ...human.settings,
+      claudeCode: {
+        ...human.settings?.claudeCode,
+        processed_sessions: processedSessions,
+      },
+    },
+  });
+}
+
+// =============================================================================
+// Main Import Function
+// =============================================================================
+
+/**
+ * Import one Claude Code session per call.
+ *
+ * Flow:
+ * 1. Ensure topics exist for all sessions (cheap, always runs).
+ * 2. Find the next unprocessed session (20+ minutes old).
+ * 3. Ensure the "Claude Code" persona exists.
+ * 4. Archive the persona and clear its messages, then write all messages
+ *    for the session — pre-marking already-imported messages [p,r,o,f]=true,
+ *    leaving new messages unmarked for extraction.
+ * 5. Queue extraction for unmarked messages.
+ * 6. Mark session processed.
+ */
+export async function importClaudeCodeSessions(
+  options: ClaudeCodeImporterOptions
+): Promise<ClaudeCodeImportResult> {
+  const { stateManager, interface: eiInterface, signal } = options;
+  const reader = options.reader ?? new ClaudeCodeReader();
+
+  const result: ClaudeCodeImportResult = {
+    sessionsProcessed: 0,
+    topicsCreated: 0,
+    topicsUpdated: 0,
+    messagesImported: 0,
+    personaCreated: false,
+    extractionScansQueued: 0,
+  };
+
+  // ─── Step 1: Ensure topics exist for ALL sessions ─────────────────────
+  const allSessions = await reader.getSessions();
+
+  for (const session of allSessions) {
+    const topicResult = ensureSessionTopic(session, stateManager);
+    if (topicResult === "created") result.topicsCreated++;
+    else if (topicResult === "updated") result.topicsUpdated++;
+  }
+
+  if (signal?.aborted) return result;
+  if (result.topicsCreated > 0 || result.topicsUpdated > 0) {
+    eiInterface?.onHumanUpdated?.();
+  }
+
+  // ─── Step 2: Find next unprocessed session ────────────────────────────
+  const human = stateManager.getHuman();
+  const processedSessions = human.settings?.claudeCode?.processed_sessions ?? {};
+  const now = Date.now();
+
+  let targetSession: ClaudeCodeSession | null = null;
+
+  // allSessions is already sorted oldest-first
+  for (const session of allSessions) {
+    const lastImported = processedSessions[session.id];
+    const sessionLastMs = new Date(session.lastMessageAt).getTime();
+    const ageMs = now - sessionLastMs;
+
+    if (ageMs < MIN_SESSION_AGE_MS) continue; // too fresh
+
+    if (!lastImported) {
+      targetSession = session;
+      break;
+    }
+
+    // Re-import if session has been updated since last import
+    if (sessionLastMs > new Date(lastImported).getTime()) {
+      targetSession = session;
+      break;
+    }
+  }
+
+  if (!targetSession) {
+    console.log("[ClaudeCode] All sessions processed, nothing new to import");
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  console.log(
+    `[ClaudeCode] Processing session: "${targetSession.title}" ` +
+      `(last message: ${targetSession.lastMessageAt})`
+  );
+
+  // ─── Step 3: Pull messages ────────────────────────────────────────────
+  const messages = await reader.getMessagesForSession(targetSession.id);
+
+  if (messages.length === 0) {
+    updateProcessedState(stateManager, targetSession);
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  // ─── Step 4: Ensure persona, archive, clear, write messages ──────────
+  const persona = ensureClaudeCodePersona(stateManager, eiInterface);
+  result.personaCreated = !stateManager.persona_getByName(CLAUDE_CODE_PERSONA_NAME);
+
+  if (!persona.is_archived) {
+    stateManager.persona_archive(persona.id);
+  }
+
+  const existingMsgs = stateManager.messages_get(persona.id);
+  if (existingMsgs.length > 0) {
+    stateManager.messages_remove(persona.id, existingMsgs.map((m) => m.id));
+  }
+
+  const cutoffIso = processedSessions[targetSession.id] ?? null;
+  const cutoffMs = cutoffIso ? new Date(cutoffIso).getTime() : null;
+  const toAnalyze: Message[] = [];
+
+  for (const msg of messages) {
+    const msgMs = new Date(msg.timestamp).getTime();
+    const isOld = cutoffMs !== null && msgMs < cutoffMs;
+    const eiMsg = isOld ? convertToPreMarkedEiMessage(msg) : convertToEiMessage(msg);
+    stateManager.messages_append(persona.id, eiMsg);
+    result.messagesImported++;
+    if (!isOld) toAnalyze.push(eiMsg);
+  }
+
+  stateManager.messages_sort(persona.id);
+  stateManager.persona_update(persona.id, {
+    last_activity: new Date().toISOString(),
+  });
+  eiInterface?.onMessageAdded?.(persona.id);
+
+  // ─── Step 5: Queue extraction for new messages ────────────────────────
+  if (toAnalyze.length > 0 && !signal?.aborted) {
+    const allInState = stateManager.messages_get(persona.id);
+    const analyzeIds = new Set(toAnalyze.map((m) => m.id));
+    const analyzeStartIndex = allInState.findIndex((m) => analyzeIds.has(m.id));
+    const contextMsgs = analyzeStartIndex > 0 ? allInState.slice(0, analyzeStartIndex) : [];
+
+    const context: ExtractionContext = {
+      personaId: persona.id,
+      personaDisplayName: persona.display_name,
+      messages_context: contextMsgs,
+      messages_analyze: toAnalyze,
+    };
+
+    queueAllScans(context, stateManager);
+    result.extractionScansQueued += 4;
+  }
+
+  result.sessionsProcessed = 1;
+
+  // ─── Step 6: Mark processed ───────────────────────────────────────────
+  updateProcessedState(stateManager, targetSession);
+
+  console.log(
+    `[ClaudeCode] Session complete: ${result.messagesImported} messages imported, ` +
+      `${result.extractionScansQueued} extraction scans queued`
+  );
+
+  return result;
+}

package/src/integrations/claude-code/index.ts

@@ -0,0 +1,10 @@
+export { ClaudeCodeReader } from "./reader.js";
+export { importClaudeCodeSessions } from "./importer.js";
+export type { ClaudeCodeImportResult, ClaudeCodeImporterOptions } from "./importer.js";
+export type {
+  IClaudeCodeReader,
+  ClaudeCodeSession,
+  ClaudeCodeMessage,
+  ClaudeCodeSettings,
+} from "./types.js";
+export { CLAUDE_CODE_PERSONA_NAME } from "./types.js";

package/src/integrations/claude-code/reader.ts

@@ -0,0 +1,238 @@
+import type {
+  IClaudeCodeReader,
+  ClaudeCodeSession,
+  ClaudeCodeMessage,
+  ClaudeCodeRecord,
+  ClaudeCodeUserRecord,
+  ClaudeCodeAssistantRecord,
+} from "./types.js";
+
+// OpenTUI polyfills window but not document — check document for real browser
+const isBrowser = typeof document !== "undefined";
+
+let _join: typeof import("path").join;
+let _readdir: typeof import("fs/promises").readdir;
+let _readFile: typeof import("fs/promises").readFile;
+let _nodeModulesLoaded = false;
+
+async function ensureNodeModules(): Promise<boolean> {
+  if (isBrowser) return false;
+  if (_nodeModulesLoaded) return true;
+
+  const PATH_MODULE = "path";
+  const FS_MODULE = "fs/promises";
+
+  const pathMod = await import(/* @vite-ignore */ PATH_MODULE);
+  const fsMod = await import(/* @vite-ignore */ FS_MODULE);
+
+  _join = pathMod.join;
+  _readdir = fsMod.readdir;
+  _readFile = fsMod.readFile;
+  _nodeModulesLoaded = true;
+  return true;
+}
+
+function getDefaultProjectsPath(): string {
+  if (!_join) return "";
+  return _join(process.env.HOME || "~", ".claude", "projects");
+}
+
+/**
+ * Derives a human-readable session title from the cwd.
+ * "/Users/flare576/Projects/Personal/ei" → "ei"
+ */
+function titleFromCwd(cwd: string): string {
+  if (!cwd) return "Unknown";
+  const parts = cwd.replace(/\\/g, "/").split("/").filter(Boolean);
+  return parts[parts.length - 1] ?? cwd;
+}
+
+/**
+ * Extracts plain text from an assistant content block array.
+ * Skips: thinking blocks, tool_use blocks, anything that isn't a text block.
+ */
+function extractAssistantText(content: ClaudeCodeAssistantRecord["message"]["content"]): string {
+  return content
+    .filter((block): block is { type: "text"; text: string } => block.type === "text")
+    .map((block) => block.text)
+    .join("\n\n")
+    .trim();
+}
+
+export class ClaudeCodeReader implements IClaudeCodeReader {
+  private readonly projectsPath?: string;
+
+  constructor(projectsPath?: string) {
+    this.projectsPath = projectsPath;
+  }
+
+  async getSessions(): Promise<ClaudeCodeSession[]> {
+    if (!(await ensureNodeModules())) return [];
+
+    const projectsDir = this.projectsPath ?? getDefaultProjectsPath();
+    const sessions: ClaudeCodeSession[] = [];
+
+    let projectDirs: string[];
+    try {
+      projectDirs = await _readdir(projectsDir);
+    } catch {
+      return [];
+    }
+
+    for (const projectDirName of projectDirs) {
+      if (projectDirName.startsWith(".")) continue;
+
+      const projectPath = _join(projectsDir, projectDirName);
+      let sessionFiles: string[];
+      try {
+        sessionFiles = await _readdir(projectPath);
+      } catch {
+        continue;
+      }
+
+      for (const fileName of sessionFiles) {
+        if (!fileName.endsWith(".jsonl")) continue;
+
+        // Skip agent sidechain sessions (named "agent-<hash>.jsonl")
+        if (fileName.startsWith("agent-")) continue;
+
+        const sessionId = fileName.replace(/\.jsonl$/, "");
+        const filePath = _join(projectPath, fileName);
+
+        const session = await this.parseSessionMeta(sessionId, filePath);
+        if (session) {
+          sessions.push(session);
+        }
+      }
+    }
+
+    return sessions.sort(
+      (a, b) => new Date(a.lastMessageAt).getTime() - new Date(b.lastMessageAt).getTime()
+    );
+  }
+
+  async getMessagesForSession(sessionId: string): Promise<ClaudeCodeMessage[]> {
+    if (!(await ensureNodeModules())) return [];
+
+    const projectsDir = this.projectsPath ?? getDefaultProjectsPath();
+    const messages: ClaudeCodeMessage[] = [];
+
+    // Find the file — it could be under any project dir
+    let projectDirs: string[];
+    try {
+      projectDirs = await _readdir(projectsDir);
+    } catch {
+      return [];
+    }
+
+    let filePath: string | null = null;
+    for (const projectDirName of projectDirs) {
+      if (projectDirName.startsWith(".")) continue;
+      const candidate = _join(projectsDir, projectDirName, `${sessionId}.jsonl`);
+      try {
+        await _readFile(candidate, "utf-8"); // throws if not found
+        filePath = candidate;
+        break;
+      } catch {
+        // not in this project dir
+      }
+    }
+
+    if (!filePath) return [];
+
+    const records = await this.readJsonl(filePath);
+
+    for (const record of records) {
+      if (record.type === "user") {
+        const r = record as ClaudeCodeUserRecord;
+        const content = typeof r.message?.content === "string" ? r.message.content : "";
+        if (!content.trim()) continue;
+
+        messages.push({
+          id: r.uuid,
+          sessionId: r.sessionId,
+          role: "user",
+          content,
+          timestamp: r.timestamp,
+        });
+      } else if (record.type === "assistant") {
+        const r = record as ClaudeCodeAssistantRecord;
+        const content = extractAssistantText(r.message?.content ?? []);
+        if (!content) continue;
+
+        messages.push({
+          id: r.uuid,
+          sessionId: r.sessionId,
+          role: "assistant",
+          content,
+          timestamp: r.timestamp,
+        });
+      }
+      // Skip: file-history-snapshot, system, summary, progress, tool_use, tool_result
+    }
+
+    return messages.sort(
+      (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+    );
+  }
+
+  // ============================================================================
+  // Private Helpers
+  // ============================================================================
+
+  private async parseSessionMeta(
+    sessionId: string,
+    filePath: string
+  ): Promise<ClaudeCodeSession | null> {
+    const records = await this.readJsonl(filePath);
+
+    let firstTimestamp: string | null = null;
+    let lastTimestamp: string | null = null;
+    let cwd = "";
+
+    for (const record of records) {
+      if (record.type !== "user" && record.type !== "assistant") continue;
+
+      const ts = record.timestamp;
+      if (!ts) continue;
+
+      if (!firstTimestamp || ts < firstTimestamp) firstTimestamp = ts;
+      if (!lastTimestamp || ts > lastTimestamp) lastTimestamp = ts;
+
+      if (!cwd && (record as ClaudeCodeUserRecord | ClaudeCodeAssistantRecord).cwd) {
+        cwd = (record as ClaudeCodeUserRecord | ClaudeCodeAssistantRecord).cwd!;
+      }
+    }
+
+    if (!firstTimestamp || !lastTimestamp) return null;
+
+    return {
+      id: sessionId,
+      cwd,
+      title: titleFromCwd(cwd),
+      firstMessageAt: firstTimestamp,
+      lastMessageAt: lastTimestamp,
+    };
+  }
+
+  private async readJsonl(filePath: string): Promise<ClaudeCodeRecord[]> {
+    let text: string;
+    try {
+      text = await _readFile(filePath, "utf-8");
+    } catch {
+      return [];
+    }
+
+    const records: ClaudeCodeRecord[] = [];
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        records.push(JSON.parse(trimmed) as ClaudeCodeRecord);
+      } catch {
+        // skip malformed lines
+      }
+    }
+    return records;
+  }
+}