ei-tui 1.6.0 → 1.6.2

package/src/integrations/codex/importer.ts

@@ -0,0 +1,258 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { ContextStatus, Ei_Interface, Message, PersonaEntity, PersonaTrait } from "../../core/types.js";
+import { DEFAULT_SEED_TRAITS } from "../../core/constants/seed-traits.js";
+import {
+  queueAllScans,
+  type ExtractionContext,
+} from "../../core/orchestrators/human-extraction.js";
+import {
+  queuePersonRewritePhase,
+  queueTopicRewritePhase,
+} from "../../core/orchestrators/ceremony.js";
+import { qualifyCodexMessage } from "../../core/utils/message-id.js";
+import { getMachineId } from "../machine-id.js";
+import { isProcessRunning } from "../process-check.js";
+import { CodexReader } from "./reader.js";
+import {
+  CODEX_PERSONA_NAME,
+  MIN_SESSION_AGE_MS,
+  type CodexMessage,
+  type CodexSession,
+  type ICodexReader,
+} from "./types.js";
+
+export interface CodexImportResult {
+  sessionsProcessed: number;
+  messagesImported: number;
+  personaCreated: boolean;
+  extractionScansQueued: number;
+}
+
+export interface CodexImporterOptions {
+  stateManager: StateManager;
+  interface?: Ei_Interface;
+  reader?: ICodexReader;
+  signal?: AbortSignal;
+}
+
+const TWELVE_HOURS_MS = 43_200_000;
+const CODEX_GROUP = "Codex";
+
+function convertToEiMessage(msg: CodexMessage, sessionId: string): Message {
+  return {
+    id: qualifyCodexMessage(getMachineId(), sessionId, msg.id),
+    role: msg.role === "user" ? "human" : "system",
+    content: msg.content,
+    timestamp: msg.timestamp,
+    read: true,
+    context_status: "default" as ContextStatus,
+    external: true,
+  };
+}
+
+function convertToPreMarkedEiMessage(msg: CodexMessage, sessionId: string): Message {
+  return {
+    ...convertToEiMessage(msg, sessionId),
+    f: true,
+    t: true,
+    p: true,
+    e: true,
+  };
+}
+
+function ensureCodexPersona(
+  stateManager: StateManager,
+  eiInterface?: Ei_Interface
+): PersonaEntity {
+  const existing = stateManager.persona_getByName(CODEX_PERSONA_NAME);
+  if (existing) return existing;
+
+  const now = new Date().toISOString();
+  const seedTraits: PersonaTrait[] = DEFAULT_SEED_TRAITS.map((t) => ({
+    id: crypto.randomUUID(),
+    name: t.name,
+    description: t.description,
+    sentiment: t.sentiment,
+    strength: t.strength,
+    last_updated: now,
+  }));
+
+  const persona: PersonaEntity = {
+    id: crypto.randomUUID(),
+    display_name: CODEX_PERSONA_NAME,
+    entity: "system",
+    aliases: ["codex", "codex cli", "codex desktop", "openai codex"],
+    short_description: "Codex - OpenAI coding agent environment",
+    long_description:
+      "Codex is OpenAI's coding agent environment for working with local codebases, terminal commands, tools, and implementation tasks.",
+    group_primary: CODEX_GROUP,
+    groups_visible: [CODEX_GROUP],
+    traits: seedTraits,
+    topics: [],
+    is_paused: false,
+    is_archived: false,
+    is_static: false,
+    heartbeat_delay_ms: TWELVE_HOURS_MS,
+    last_heartbeat: now,
+    last_updated: now,
+  };
+
+  stateManager.persona_add(persona);
+  eiInterface?.onPersonaAdded?.();
+  return persona;
+}
+
+function updateProcessedState(
+  stateManager: StateManager,
+  session: CodexSession
+): void {
+  const human = stateManager.getHuman();
+  const lastMessageMs = new Date(session.lastMessageAt).getTime();
+  const extractionPoint = human.settings?.codex?.extraction_point;
+  const currentPointMs = extractionPoint ? new Date(extractionPoint).getTime() : 0;
+  const newPointMs = Math.max(currentPointMs, lastMessageMs);
+
+  const processedSessions = {
+    ...(human.settings?.codex?.processed_sessions ?? {}),
+    [session.id]: new Date().toISOString(),
+  };
+
+  stateManager.setHuman({
+    ...human,
+    settings: {
+      ...human.settings,
+      codex: {
+        ...human.settings?.codex,
+        extraction_point: new Date(newPointMs).toISOString(),
+        processed_sessions: processedSessions,
+      },
+    },
+  });
+}
+
+async function isCodexRunning(): Promise<boolean> {
+  return (await isProcessRunning("Codex")) || (await isProcessRunning("codex"));
+}
+
+export async function importCodexSessions(
+  options: CodexImporterOptions
+): Promise<CodexImportResult> {
+  const { stateManager, interface: eiInterface, signal } = options;
+  const reader = options.reader ?? new CodexReader();
+
+  const result: CodexImportResult = {
+    sessionsProcessed: 0,
+    messagesImported: 0,
+    personaCreated: false,
+    extractionScansQueued: 0,
+  };
+
+  const allSessions = await reader.getSessions();
+  if (signal?.aborted) return result;
+
+  const human = stateManager.getHuman();
+  const processedSessions = human.settings?.codex?.processed_sessions ?? {};
+  const now = Date.now();
+  const toolRunning = await isCodexRunning();
+
+  let targetSession: CodexSession | null = null;
+
+  for (const session of allSessions) {
+    const sessionLastMs = new Date(session.lastMessageAt).getTime();
+    const ageMs = now - sessionLastMs;
+
+    if (ageMs < MIN_SESSION_AGE_MS && toolRunning) continue;
+
+    const lastImported = processedSessions[session.id];
+    if (lastImported && sessionLastMs <= new Date(lastImported).getTime()) continue;
+
+    targetSession = session;
+    break;
+  }
+
+  if (!targetSession) {
+    console.log("[Codex] All sessions processed, nothing new to import");
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  console.log(
+    `[Codex] Processing session: "${targetSession.title}" ` +
+      `(last message: ${targetSession.lastMessageAt})`
+  );
+
+  const messages = targetSession.messages;
+  if (messages.length === 0) {
+    updateProcessedState(stateManager, targetSession);
+    return result;
+  }
+
+  if (signal?.aborted) return result;
+
+  const personaExistedBefore = stateManager.persona_getByName(CODEX_PERSONA_NAME) !== null;
+  const persona = ensureCodexPersona(stateManager, eiInterface);
+  result.personaCreated = !personaExistedBefore;
+
+  if (!personaExistedBefore) {
+    stateManager.persona_archive(persona.id);
+  } else {
+    const existingMsgs = stateManager.messages_get(persona.id);
+    const externalIds = existingMsgs.filter((m) => m.external === true).map((m) => m.id);
+    if (externalIds.length > 0) {
+      stateManager.messages_remove(persona.id, externalIds);
+    }
+  }
+
+  const cutoffIso = processedSessions[targetSession.id] ?? null;
+  const cutoffMs = cutoffIso ? new Date(cutoffIso).getTime() : null;
+  const toAnalyze: Message[] = [];
+
+  for (const msg of messages) {
+    const msgMs = new Date(msg.timestamp).getTime();
+    const isOld = cutoffMs !== null && msgMs < cutoffMs;
+    const eiMsg = isOld
+      ? convertToPreMarkedEiMessage(msg, targetSession.id)
+      : convertToEiMessage(msg, targetSession.id);
+
+    stateManager.messages_append(persona.id, eiMsg);
+    result.messagesImported++;
+    if (!isOld) toAnalyze.push(eiMsg);
+  }
+
+  stateManager.messages_sort(persona.id);
+  eiInterface?.onMessageAdded?.(persona.id);
+
+  if (toAnalyze.length > 0 && !signal?.aborted) {
+    const allInState = stateManager.messages_get(persona.id);
+    const analyzeIds = new Set(toAnalyze.map((m) => m.id));
+    const analyzeStartIndex = allInState.findIndex((m) => analyzeIds.has(m.id));
+    const contextMsgs = analyzeStartIndex > 0 ? allInState.slice(0, analyzeStartIndex) : [];
+
+    const context: ExtractionContext = {
+      personaId: persona.id,
+      channelDisplayName: persona.display_name,
+      messages_context: contextMsgs,
+      messages_analyze: toAnalyze,
+      sources: [`codex:${getMachineId()}:${targetSession.id}`],
+    };
+
+    queuePersonRewritePhase(stateManager);
+    queueTopicRewritePhase(stateManager);
+    queueAllScans(context, stateManager, {
+      extraction_model: human.settings?.codex?.extraction_model,
+      external_filter: "only",
+    });
+    result.extractionScansQueued += 4;
+  }
+
+  result.sessionsProcessed = 1;
+  updateProcessedState(stateManager, targetSession);
+
+  console.log(
+    `[Codex] Session complete: ${result.messagesImported} messages imported, ` +
+      `${result.extractionScansQueued} extraction scans queued`
+  );
+
+  return result;
+}

package/src/integrations/codex/index.ts

@@ -0,0 +1,11 @@
+export { CodexReader } from "./reader.js";
+export { importCodexSessions } from "./importer.js";
+export type { CodexImportResult, CodexImporterOptions } from "./importer.js";
+export type {
+  CodexMessage,
+  CodexMessageWindow,
+  CodexSession,
+  CodexSettings,
+  ICodexReader,
+} from "./types.js";
+export { CODEX_PERSONA_NAME } from "./types.js";

package/src/integrations/codex/reader.ts

@@ -0,0 +1,241 @@
+import type {
+  CodexMessage,
+  CodexMessageWindow,
+  CodexRolloutRecord,
+  CodexSession,
+  CodexThreadRow,
+  ICodexReader,
+} from "./types.js";
+
+const isBrowser = typeof document !== "undefined";
+
+let _join: typeof import("path").join;
+let _basename: typeof import("path").basename;
+let _readFile: typeof import("fs/promises").readFile;
+let _readdir: typeof import("fs/promises").readdir;
+let _stat: typeof import("fs/promises").stat;
+let _nodeModulesLoaded = false;
+
+async function ensureNodeModules(): Promise<boolean> {
+  if (isBrowser) return false;
+  if (_nodeModulesLoaded) return true;
+
+  const PATH_MODULE = "path";
+  const FS_MODULE = "fs/promises";
+
+  const pathMod = await import(/* @vite-ignore */ PATH_MODULE);
+  const fsMod = await import(/* @vite-ignore */ FS_MODULE);
+
+  _join = pathMod.join;
+  _basename = pathMod.basename;
+  _readFile = fsMod.readFile;
+  _readdir = fsMod.readdir;
+  _stat = fsMod.stat;
+  _nodeModulesLoaded = true;
+  return true;
+}
+
+function getDefaultCodexHome(): string {
+  if (!_join) return "";
+  return process.env.CODEX_HOME || _join(process.env.HOME || "~", ".codex");
+}
+
+function titleFromCwd(cwd: string): string {
+  if (!cwd) return "Codex Session";
+  const parts = cwd.replace(/\\/g, "/").split("/").filter(Boolean);
+  return parts[parts.length - 1] ?? cwd;
+}
+
+function timestampFromMs(value: number | null | undefined): string | null {
+  if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null;
+  return new Date(value).toISOString();
+}
+
+function extractEventMessage(record: CodexRolloutRecord, lineIndex: number, sessionId: string): CodexMessage | null {
+  if (record.type !== "event_msg") return null;
+
+  const payload = record.payload ?? {};
+  const payloadType = payload.type;
+  if (payloadType !== "user_message" && payloadType !== "agent_message") return null;
+
+  const rawMessage = payload.message;
+  if (typeof rawMessage !== "string" || rawMessage.trim() === "") return null;
+
+  const timestamp = typeof record.timestamp === "string" && record.timestamp.trim()
+    ? record.timestamp
+    : new Date(0).toISOString();
+
+  return {
+    id: `evt_${lineIndex + 1}`,
+    sessionId,
+    role: payloadType === "user_message" ? "user" : "assistant",
+    content: rawMessage.trim(),
+    timestamp,
+  };
+}
+
+export function parseCodexRolloutMessages(text: string, sessionId: string): CodexMessage[] {
+  const messages: CodexMessage[] = [];
+  const lines = text.split("\n");
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (!trimmed) continue;
+
+    let record: CodexRolloutRecord;
+    try {
+      record = JSON.parse(trimmed) as CodexRolloutRecord;
+    } catch {
+      continue;
+    }
+
+    const message = extractEventMessage(record, i, sessionId);
+    if (message) messages.push(message);
+  }
+
+  return messages.sort(
+    (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+  );
+}
+
+export class CodexReader implements ICodexReader {
+  private readonly codexHome?: string;
+
+  constructor(codexHome?: string) {
+    this.codexHome = codexHome;
+  }
+
+  async isAvailable(): Promise<boolean> {
+    if (!(await ensureNodeModules())) return false;
+    const dbPath = await this.findStateDbPath();
+    if (!dbPath) return false;
+
+    try {
+      return (await _stat(dbPath)).isFile();
+    } catch {
+      return false;
+    }
+  }
+
+  async getSessions(): Promise<CodexSession[]> {
+    if (!(await ensureNodeModules())) return [];
+
+    const dbPath = await this.findStateDbPath();
+    if (!dbPath) return [];
+
+    let db: import("bun:sqlite").Database;
+    try {
+      const { Database } = await import(/* @vite-ignore */ "bun:sqlite");
+      db = new Database(dbPath, { readonly: true });
+    } catch (err) {
+      console.warn("[CodexReader] failed to open state DB:", err);
+      return [];
+    }
+
+    let rows: CodexThreadRow[];
+    try {
+      rows = db.query("SELECT * FROM threads WHERE rollout_path IS NOT NULL AND rollout_path != ''").all() as CodexThreadRow[];
+    } catch (err) {
+      console.warn("[CodexReader] failed to read threads table:", err);
+      db.close();
+      return [];
+    }
+
+    db.close();
+
+    const sessions: CodexSession[] = [];
+    for (const row of rows) {
+      if (!row.id || !row.rollout_path) continue;
+      const session = await this.sessionFromThreadRow(row);
+      if (session) sessions.push(session);
+    }
+
+    return sessions.sort(
+      (a, b) => new Date(a.lastMessageAt).getTime() - new Date(b.lastMessageAt).getTime()
+    );
+  }
+
+  async getMessageById(
+    sessionId: string,
+    messageId: string,
+    before = 0,
+    after = 0
+  ): Promise<CodexMessageWindow | null> {
+    const sessions = await this.getSessions();
+    const session = sessions.find((s) => s.id === sessionId);
+    if (!session) return null;
+
+    const idx = session.messages.findIndex((m) => m.id === messageId);
+    if (idx === -1) return null;
+
+    return {
+      message: session.messages[idx],
+      before: session.messages.slice(Math.max(0, idx - before), idx),
+      after: session.messages.slice(idx + 1, idx + 1 + after),
+      session,
+    };
+  }
+
+  private async findStateDbPath(): Promise<string | null> {
+    const base = this.codexHome ?? getDefaultCodexHome();
+    let entries: string[];
+    try {
+      entries = await _readdir(base);
+    } catch {
+      return null;
+    }
+
+    const stateDbs = entries
+      .map((name) => {
+        const match = name.match(/^state_(\d+)\.sqlite$/);
+        return match ? { name, version: Number(match[1]) } : null;
+      })
+      .filter((entry): entry is { name: string; version: number } => entry !== null)
+      .sort((a, b) => b.version - a.version);
+
+    if (stateDbs.length > 0) return _join(base, stateDbs[0].name);
+    if (entries.includes("state.sqlite")) return _join(base, "state.sqlite");
+    return null;
+  }
+
+  private async sessionFromThreadRow(row: CodexThreadRow): Promise<CodexSession | null> {
+    const rolloutPath = row.rollout_path;
+    if (!rolloutPath) return null;
+
+    const messages = await this.readMessages(row.id, rolloutPath);
+    if (messages.length === 0) return null;
+
+    const first = messages[0];
+    const last = messages[messages.length - 1];
+    const title = row.title?.trim()
+      || row.first_user_message?.trim()
+      || titleFromCwd(row.cwd ?? "");
+
+    return {
+      id: row.id,
+      title,
+      cwd: row.cwd ?? "",
+      source: row.source ?? undefined,
+      threadSource: row.thread_source ?? undefined,
+      agentNickname: row.agent_nickname ?? undefined,
+      agentRole: row.agent_role ?? undefined,
+      agentPath: row.agent_path ?? undefined,
+      rolloutPath,
+      firstMessageAt: first.timestamp || timestampFromMs(row.created_at_ms) || timestampFromMs(row.created_at) || new Date(0).toISOString(),
+      lastMessageAt: last.timestamp || timestampFromMs(row.updated_at_ms) || timestampFromMs(row.updated_at) || new Date(0).toISOString(),
+      messages,
+    };
+  }
+
+  private async readMessages(sessionId: string, rolloutPath: string): Promise<CodexMessage[]> {
+    let text: string;
+    try {
+      text = await _readFile(rolloutPath, "utf-8");
+    } catch (err) {
+      console.warn(`[CodexReader] skipping missing rollout ${_basename(rolloutPath)}:`, err);
+      return [];
+    }
+
+    return parseCodexRolloutMessages(text, sessionId);
+  }
+}

package/src/integrations/codex/types.ts

@@ -0,0 +1,117 @@
+/**
+ * Codex Integration Types
+ *
+ * Codex Desktop / CLI stores thread metadata in ~/.codex/state_*.sqlite and
+ * per-thread rollout JSONL files under ~/.codex/sessions/YYYY/MM/DD/.
+ * Ei imports only visible user/agent event messages and skips tool chatter,
+ * prompt scaffolding, token-count events, and system/developer payloads.
+ */
+
+// ============================================================================
+// Reader Interface
+// ============================================================================
+
+export interface CodexMessageWindow {
+  message: CodexMessage;
+  before: CodexMessage[];
+  after: CodexMessage[];
+  session: CodexSession;
+}
+
+export interface ICodexReader {
+  getSessions(): Promise<CodexSession[]>;
+  getMessageById(sessionId: string, messageId: string, before?: number, after?: number): Promise<CodexMessageWindow | null>;
+  isAvailable(): Promise<boolean>;
+}
+
+// ============================================================================
+// Raw Storage Types
+// ============================================================================
+
+export interface CodexThreadRow {
+  id: string;
+  rollout_path?: string | null;
+  created_at?: number | null;
+  updated_at?: number | null;
+  created_at_ms?: number | null;
+  updated_at_ms?: number | null;
+  cwd?: string | null;
+  title?: string | null;
+  first_user_message?: string | null;
+  source?: string | null;
+  thread_source?: string | null;
+  agent_nickname?: string | null;
+  agent_role?: string | null;
+  agent_path?: string | null;
+  archived?: number | boolean | null;
+}
+
+export interface CodexRolloutRecord {
+  timestamp?: string;
+  type?: string;
+  payload?: Record<string, unknown>;
+}
+
+// ============================================================================
+// Cleaned Session / Message Types
+// ============================================================================
+
+export interface CodexMessage {
+  /** Stable synthetic id derived from JSONL line number, e.g. "evt_42" */
+  id: string;
+  sessionId: string;
+  role: "user" | "assistant";
+  content: string;
+  timestamp: string;
+}
+
+export interface CodexSession {
+  id: string;
+  title: string;
+  cwd: string;
+  source?: string;
+  threadSource?: string;
+  agentNickname?: string;
+  agentRole?: string;
+  agentPath?: string;
+  rolloutPath: string;
+  firstMessageAt: string;
+  lastMessageAt: string;
+  messages: CodexMessage[];
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** The single persona name for all Codex sessions */
+export const CODEX_PERSONA_NAME = "Codex";
+
+/** Topic groups assigned to Codex session topics */
+export const CODEX_TOPIC_GROUPS = ["General", "Coding", "Codex"];
+
+/**
+ * Minimum session age before import.
+ * Mirrors Claude Code / Cursor's 20-minute rule so active sessions can settle.
+ */
+export const MIN_SESSION_AGE_MS = 20 * 60 * 1000;
+
+// ============================================================================
+// Human Settings Shape
+// ============================================================================
+
+/**
+ * Stored under human.settings.codex
+ *
+ * WARNING: ADDING A NEW FIELD HERE?
+ * If it is runtime-managed (not user-editable), also preserve it in
+ * settingsFromYAML() in tui/src/util/yaml-settings.ts or /settings will wipe it.
+ */
+export interface CodexSettings {
+  integration?: boolean;
+  polling_interval_ms?: number;
+  extraction_model?: string;
+  last_sync?: string;
+  extraction_point?: string;
+  processed_sessions?: Record<string, string>;
+}

package/src/integrations/opencode/reader-factory.ts

@@ -11,19 +11,19 @@ export async function createOpenCodeReader(basePath?: string): Promise<IOpenCode
   if (existsSync(dbPath)) {
     try {
       const { SqliteReader } = await import("./sqlite-reader.js");
-      console.log("[OpenCode] Using SQLite reader");
+      console.error("[OpenCode] Using SQLite reader");
       return new SqliteReader(dbPath);
     } catch {
-      console.log("[OpenCode] SQLite not available, falling back to JSON reader");
+      console.error("[OpenCode] SQLite not available, falling back to JSON reader");
     }
   }
 
   if (existsSync(storagePath)) {
-    console.log("[OpenCode] Using JSON reader (legacy)");
+    console.error("[OpenCode] Using JSON reader (legacy)");
     return new JsonReader(storagePath);
   }
 
-  console.log("[OpenCode] No OpenCode data found");
+  console.error("[OpenCode] No OpenCode data found");
   return new JsonReader(storagePath);
 }
 

package/src/integrations/slack/importer.ts

@@ -27,7 +27,6 @@ export interface SlackImportResult {
 function ensureSlackPersona(stateManager: StateManager, eiInterface: Ei_Interface): PersonaEntity {
   const existing = stateManager.persona_getAll().find(p => p.display_name === "Slack");
   if (existing) {
-    if (existing.is_archived) stateManager.persona_unarchive(existing.id);
     return existing;
   }
   const persona: PersonaEntity = {

package/src/prompts/response/index.ts

@@ -11,6 +11,7 @@ import type { ResponsePromptData, PromptOutput } from "./types.js";
 import { formatCurrentTime } from "../../core/format-utils.js";
 import {
   buildIdentitySection,
+  buildNotesSection,
   buildGuidelinesSection,
   buildTraitsSection,
   buildTopicsSection,
@@ -44,6 +45,7 @@ Your role is unique among personas:
   - Consider their traits when building your responses more than the current conversation history
 - You encourage human-to-human connection when appropriate`;
 
+  const notesSection = buildNotesSection(data.persona.notes);
   const guidelines = buildGuidelinesSection("ei");
   const yourTraits = buildTraitsSection(data.persona.traits, "Your Personality");
   const yourTopics = buildTopicsSection(data.persona.topics, "Your Interests");
@@ -63,7 +65,7 @@ Your role is unique among personas:
     : "";
 
   return `${identity}
-
+${notesSection ? `\n${notesSection}` : ""}
 ${guidelines}
 
 ${yourTraits}
@@ -93,6 +95,7 @@ ${conversationState}
  */
 function buildStandardSystemPrompt(data: ResponsePromptData): string {
   const identity = buildIdentitySection(data.persona);
+  const notesSection = buildNotesSection(data.persona.notes);
   const guidelines = buildGuidelinesSection(data.persona.name);
   const yourTraits = buildTraitsSection(data.persona.traits, "Your Personality");
   const yourTopics = buildTopicsSection(data.persona.topics, "Your Interests");
@@ -111,7 +114,7 @@ function buildStandardSystemPrompt(data: ResponsePromptData): string {
     : "";
 
   return `${identity}
-
+${notesSection ? `\n${notesSection}` : ""}
 ${guidelines}
 
 ${yourTraits}