ei-tui 1.6.1 → 1.6.3

package/src/integrations/codex/types.ts

@@ -0,0 +1,117 @@
+/**
+ * Codex Integration Types
+ *
+ * Codex Desktop / CLI stores thread metadata in ~/.codex/state_*.sqlite and
+ * per-thread rollout JSONL files under ~/.codex/sessions/YYYY/MM/DD/.
+ * Ei imports only visible user/agent event messages and skips tool chatter,
+ * prompt scaffolding, token-count events, and system/developer payloads.
+ */
+
+// ============================================================================
+// Reader Interface
+// ============================================================================
+
+export interface CodexMessageWindow {
+  message: CodexMessage;
+  before: CodexMessage[];
+  after: CodexMessage[];
+  session: CodexSession;
+}
+
+export interface ICodexReader {
+  getSessions(): Promise<CodexSession[]>;
+  getMessageById(sessionId: string, messageId: string, before?: number, after?: number): Promise<CodexMessageWindow | null>;
+  isAvailable(): Promise<boolean>;
+}
+
+// ============================================================================
+// Raw Storage Types
+// ============================================================================
+
+export interface CodexThreadRow {
+  id: string;
+  rollout_path?: string | null;
+  created_at?: number | null;
+  updated_at?: number | null;
+  created_at_ms?: number | null;
+  updated_at_ms?: number | null;
+  cwd?: string | null;
+  title?: string | null;
+  first_user_message?: string | null;
+  source?: string | null;
+  thread_source?: string | null;
+  agent_nickname?: string | null;
+  agent_role?: string | null;
+  agent_path?: string | null;
+  archived?: number | boolean | null;
+}
+
+export interface CodexRolloutRecord {
+  timestamp?: string;
+  type?: string;
+  payload?: Record<string, unknown>;
+}
+
+// ============================================================================
+// Cleaned Session / Message Types
+// ============================================================================
+
+export interface CodexMessage {
+  /** Stable synthetic id derived from JSONL line number, e.g. "evt_42" */
+  id: string;
+  sessionId: string;
+  role: "user" | "assistant";
+  content: string;
+  timestamp: string;
+}
+
+export interface CodexSession {
+  id: string;
+  title: string;
+  cwd: string;
+  source?: string;
+  threadSource?: string;
+  agentNickname?: string;
+  agentRole?: string;
+  agentPath?: string;
+  rolloutPath: string;
+  firstMessageAt: string;
+  lastMessageAt: string;
+  messages: CodexMessage[];
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** The single persona name for all Codex sessions */
+export const CODEX_PERSONA_NAME = "Codex";
+
+/** Topic groups assigned to Codex session topics */
+export const CODEX_TOPIC_GROUPS = ["General", "Coding", "Codex"];
+
+/**
+ * Minimum session age before import.
+ * Mirrors Claude Code / Cursor's 20-minute rule so active sessions can settle.
+ */
+export { MIN_SESSION_AGE_MS } from "../constants.js";
+
+// ============================================================================
+// Human Settings Shape
+// ============================================================================
+
+/**
+ * Stored under human.settings.codex
+ *
+ * WARNING: ADDING A NEW FIELD HERE?
+ * If it is runtime-managed (not user-editable), also preserve it in
+ * settingsFromYAML() in tui/src/util/yaml-settings.ts or /settings will wipe it.
+ */
+export interface CodexSettings {
+  integration?: boolean;
+  polling_interval_ms?: number;
+  extraction_model?: string;
+  last_sync?: string;
+  extraction_point?: string;
+  processed_sessions?: Record<string, string>;
+}

package/src/integrations/constants.ts

@@ -0,0 +1,3 @@
+export const MIN_SESSION_AGE_MS = 20 * 60 * 1_000;
+
+export const TWELVE_HOURS_MS = 43_200_000;

package/src/integrations/cursor/importer.ts

@@ -1,15 +1,14 @@
 import type { StateManager } from "../../core/state-manager.js";
-import type { Ei_Interface, Message, ContextStatus, PersonaEntity, PersonaTrait } from "../../core/types.js";
+import type { Ei_Interface, Message, PersonaEntity, PersonaTrait } from "../../core/types.js";
 import { DEFAULT_SEED_TRAITS } from "../../core/constants/seed-traits.js";
 import type { ICursorReader, CursorSession, CursorMessage } from "./types.js";
-import {
-  CURSOR_PERSONA_NAME,
-  MIN_SESSION_AGE_MS,
-} from "./types.js";
+import { CURSOR_PERSONA_NAME } from "./types.js";
+import { MIN_SESSION_AGE_MS, TWELVE_HOURS_MS } from "../constants.js";
 import { CursorReader } from "./reader.js";
 import { isProcessRunning } from "../process-check.js";
 import { getMachineId } from "../machine-id.js";
 import { qualifyCursorMessage } from "../../core/utils/message-id.js";
+import { convertToEiMessage, convertToPreMarkedEiMessage } from "../shared/message-converter.js";
 import {
   queueAllScans,
   type ExtractionContext,
@@ -33,29 +32,12 @@ export interface CursorImporterOptions {
   signal?: AbortSignal;
 }
 
-const TWELVE_HOURS_MS = 43_200_000;
 const CURSOR_GROUP = "Cursor";
 
-function convertToEiMessage(msg: CursorMessage, sessionId: string): Message {
-  return {
-    id: qualifyCursorMessage(getMachineId(), sessionId, msg.id),
-    role: msg.type === 1 ? "human" : "system",
-    content: msg.text,
-    timestamp: msg.timestamp,
-    read: true,
-    context_status: "default" as ContextStatus,
-    external: true,
-  };
-}
+const qualify = qualifyCursorMessage;
 
-function convertToPreMarkedEiMessage(msg: CursorMessage, sessionId: string): Message {
-  return {
-    ...convertToEiMessage(msg, sessionId),
-    f: true,
-    t: true,
-    p: true,
-    e: true,
-  };
+function normalizeCursorMessage(msg: CursorMessage) {
+  return { id: msg.id, role: (msg.type === 1 ? "user" : "assistant") as "user" | "assistant", content: msg.text, timestamp: msg.timestamp };
 }
 
 function ensureCursorPersona(
@@ -209,7 +191,8 @@ export async function importCursorSessions(
   for (const msg of messages) {
     const msgMs = new Date(msg.timestamp).getTime();
     const isOld = cutoffMs !== null && msgMs < cutoffMs;
-    const eiMsg = isOld ? convertToPreMarkedEiMessage(msg, targetSession.id) : convertToEiMessage(msg, targetSession.id);
+    const normalized = normalizeCursorMessage(msg);
+    const eiMsg = isOld ? convertToPreMarkedEiMessage(normalized, targetSession.id, qualify) : convertToEiMessage(normalized, targetSession.id, qualify);
     stateManager.messages_append(persona.id, eiMsg);
     result.messagesImported++;
     if (!isOld) toAnalyze.push(eiMsg);

package/src/integrations/cursor/types.ts

@@ -116,7 +116,7 @@ export const CURSOR_TOPIC_GROUPS = ["General", "Coding", "Cursor"];
  * Minimum session age before we import it.
  * Mirrors ClaudeCode's 20-minute rule — gives the session time to "settle."
  */
-export const MIN_SESSION_AGE_MS = 20 * 60 * 1000;
+export { MIN_SESSION_AGE_MS } from "../constants.js";
 
 // ============================================================================
 // Human Settings Shape

package/src/integrations/opencode/reader-factory.ts

@@ -11,19 +11,19 @@ export async function createOpenCodeReader(basePath?: string): Promise<IOpenCode
   if (existsSync(dbPath)) {
     try {
       const { SqliteReader } = await import("./sqlite-reader.js");
-      console.log("[OpenCode] Using SQLite reader");
+      console.error("[OpenCode] Using SQLite reader");
       return new SqliteReader(dbPath);
     } catch {
-      console.log("[OpenCode] SQLite not available, falling back to JSON reader");
+      console.error("[OpenCode] SQLite not available, falling back to JSON reader");
     }
   }
 
   if (existsSync(storagePath)) {
-    console.log("[OpenCode] Using JSON reader (legacy)");
+    console.error("[OpenCode] Using JSON reader (legacy)");
     return new JsonReader(storagePath);
   }
 
-  console.log("[OpenCode] No OpenCode data found");
+  console.error("[OpenCode] No OpenCode data found");
   return new JsonReader(storagePath);
 }
 

package/src/integrations/pi/importer.ts

@@ -0,0 +1,235 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { Ei_Interface, Message, PersonaEntity, PersonaTrait } from "../../core/types.js";
+import { DEFAULT_SEED_TRAITS } from "../../core/constants/seed-traits.js";
+import {
+  queueAllScans,
+  type ExtractionContext,
+} from "../../core/orchestrators/human-extraction.js";
+import {
+  queuePersonRewritePhase,
+  queueTopicRewritePhase,
+} from "../../core/orchestrators/ceremony.js";
+import { qualifyPiMessage } from "../../core/utils/message-id.js";
+import { convertToEiMessage, convertToPreMarkedEiMessage } from "../shared/message-converter.js";
+import { getMachineId } from "../machine-id.js";
+import { isProcessRunning } from "../process-check.js";
+import { PiReader } from "./reader.js";
+import {
+  PI_PERSONA_NAME,
+  type PiSession,
+  type IPiReader,
+} from "./types.js";
+import { MIN_SESSION_AGE_MS, TWELVE_HOURS_MS } from "../constants.js";
+
+export interface PiImportResult {
+  sessionsProcessed: number;
+  messagesImported: number;
+  personaCreated: boolean;
+  extractionScansQueued: number;
+}
+
+export interface PiImporterOptions {
+  stateManager: StateManager;
+  interface?: Ei_Interface;
+  reader?: IPiReader;
+  signal?: AbortSignal;
+}
+
+const PI_GROUP = "Pi";
+
+const qualify = qualifyPiMessage;
+
+function ensurePiPersona(
+  stateManager: StateManager,
+  eiInterface?: Ei_Interface
+): PersonaEntity {
+  const existing = stateManager.persona_getByName(PI_PERSONA_NAME);
+  if (existing) return existing;
+
+  const now = new Date().toISOString();
+  const seedTraits: PersonaTrait[] = DEFAULT_SEED_TRAITS.map((t) => ({
+    id: crypto.randomUUID(),
+    name: t.name,
+    description: t.description,
+    sentiment: t.sentiment,
+    strength: t.strength,
+    last_updated: now,
+  }));
+
+  const persona: PersonaEntity = {
+    id: crypto.randomUUID(),
+    display_name: PI_PERSONA_NAME,
+    entity: "system",
+    aliases: ["pi", "pi coding agent", "omp", "oh-my-pi"],
+    short_description: "Pi - minimal terminal coding harness",
+    long_description:
+      "Pi is a minimal terminal coding harness. Covers both vanilla Pi (earendil-works/pi) and the oh-my-pi fork (omp), which share the same JSONL session format.",
+    group_primary: PI_GROUP,
+    groups_visible: [PI_GROUP],
+    traits: seedTraits,
+    topics: [],
+    is_paused: false,
+    is_archived: false,
+    is_static: false,
+    heartbeat_delay_ms: TWELVE_HOURS_MS,
+    last_heartbeat: now,
+    last_updated: now,
+  };
+
+  stateManager.persona_add(persona);
+  eiInterface?.onPersonaAdded?.();
+  return persona;
+}
+
+function updateProcessedState(stateManager: StateManager, session: PiSession): void {
+  const human = stateManager.getHuman();
+  const lastMessageMs = new Date(session.lastMessageAt).getTime();
+  const extractionPoint = human.settings?.pi?.extraction_point;
+  const currentPointMs = extractionPoint ? new Date(extractionPoint).getTime() : 0;
+  const newPointMs = Math.max(currentPointMs, lastMessageMs);
+
+  const processedSessions = {
+    ...(human.settings?.pi?.processed_sessions ?? {}),
+    [session.id]: new Date().toISOString(),
+  };
+
+  stateManager.setHuman({
+    ...human,
+    settings: {
+      ...human.settings,
+      pi: {
+        ...human.settings?.pi,
+        extraction_point: new Date(newPointMs).toISOString(),
+        processed_sessions: processedSessions,
+      },
+    },
+  });
+}
+
+async function isPiRunning(): Promise<boolean> {
+  return (
+    (await isProcessRunning("pi")) ||
+    (await isProcessRunning("omp"))
+  );
+}
+
+export async function importPiSessions(options: PiImporterOptions): Promise<PiImportResult> {
+  const { stateManager, interface: eiInterface, signal } = options;
+  const reader = options.reader ?? new PiReader();
+
+  const result: PiImportResult = {
+    sessionsProcessed: 0,
+    messagesImported: 0,
+    personaCreated: false,
+    extractionScansQueued: 0,
+  };
+
+  const allSessions = await reader.getSessions();
+  if (signal?.aborted) return result;
+
+  const human = stateManager.getHuman();
+  const processedSessions = human.settings?.pi?.processed_sessions ?? {};
+  const now = Date.now();
+  const toolRunning = await isPiRunning();
+
+  let targetSession: PiSession | null = null;
+
+  for (const session of allSessions) {
+    const sessionLastMs = new Date(session.lastMessageAt).getTime();
+    const ageMs = now - sessionLastMs;
+
+    if (ageMs < MIN_SESSION_AGE_MS && toolRunning) continue;
+
+    const lastImported = processedSessions[session.id];
+    if (lastImported && sessionLastMs <= new Date(lastImported).getTime()) continue;
+
+    targetSession = session;
+    break;
+  }
+
+  if (!targetSession) {
+    console.log("[Pi] All sessions processed, nothing new to import");
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  console.log(
+    `[Pi] Processing session: "${targetSession.title}" ` +
+      `(last message: ${targetSession.lastMessageAt})`
+  );
+
+  const messages = targetSession.messages;
+  if (messages.length === 0) {
+    updateProcessedState(stateManager, targetSession);
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  const personaExistedBefore = stateManager.persona_getByName(PI_PERSONA_NAME) !== null;
+  const persona = ensurePiPersona(stateManager, eiInterface);
+  result.personaCreated = !personaExistedBefore;
+
+  if (!personaExistedBefore) {
+    stateManager.persona_archive(persona.id);
+  } else {
+    const existingMsgs = stateManager.messages_get(persona.id);
+    const externalIds = existingMsgs.filter((m) => m.external === true).map((m) => m.id);
+    if (externalIds.length > 0) {
+      stateManager.messages_remove(persona.id, externalIds);
+    }
+  }
+
+  const cutoffIso = processedSessions[targetSession.id] ?? null;
+  const cutoffMs = cutoffIso ? new Date(cutoffIso).getTime() : null;
+  const toAnalyze: Message[] = [];
+
+  for (const msg of messages) {
+    const msgMs = new Date(msg.timestamp).getTime();
+    const isOld = cutoffMs !== null && msgMs < cutoffMs;
+    const eiMsg = isOld
+      ? convertToPreMarkedEiMessage(msg, targetSession.id, qualify)
+      : convertToEiMessage(msg, targetSession.id, qualify);
+
+    stateManager.messages_append(persona.id, eiMsg);
+    result.messagesImported++;
+    if (!isOld) toAnalyze.push(eiMsg);
+  }
+
+  stateManager.messages_sort(persona.id);
+  eiInterface?.onMessageAdded?.(persona.id);
+
+  if (toAnalyze.length > 0 && !signal?.aborted) {
+    const allInState = stateManager.messages_get(persona.id);
+    const analyzeIds = new Set(toAnalyze.map((m) => m.id));
+    const analyzeStartIndex = allInState.findIndex((m) => analyzeIds.has(m.id));
+    const contextMsgs = analyzeStartIndex > 0 ? allInState.slice(0, analyzeStartIndex) : [];
+
+    const context: ExtractionContext = {
+      personaId: persona.id,
+      channelDisplayName: persona.display_name,
+      messages_context: contextMsgs,
+      messages_analyze: toAnalyze,
+      sources: [`pi:${getMachineId()}:${targetSession.id}`],
+    };
+
+    queuePersonRewritePhase(stateManager);
+    queueTopicRewritePhase(stateManager);
+    queueAllScans(context, stateManager, {
+      extraction_model: human.settings?.pi?.extraction_model,
+      external_filter: "only",
+    });
+    result.extractionScansQueued += 4;
+  }
+
+  result.sessionsProcessed = 1;
+  updateProcessedState(stateManager, targetSession);
+
+  console.log(
+    `[Pi] Session complete: ${result.messagesImported} messages imported, ` +
+      `${result.extractionScansQueued} extraction scans queued`
+  );
+
+  return result;
+}

package/src/integrations/pi/index.ts

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export * from "./reader.js";
+export * from "./importer.js";