ei-tui 0.9.4 → 1.0.1

package/tui/src/context/ei.tsx

@@ -14,7 +14,13 @@ import { Processor } from "../../../src/core/processor.js";
 import { FileStorage } from "../storage/file.js";
 import { remoteSync } from "../../../src/storage/remote.js";
 import { logger, clearLog, interceptConsole } from "../util/logger.js";
-import { E2E_SKIP_LOCAL_DETECT } from "../util/e2e-flags.js";
+import { E2E_SKIP_LOCAL_DETECT, E2E_SKIP_CLOUD_DETECT } from "../util/e2e-flags.js";
+import {
+  detectProviders,
+  buildProviderAccounts,
+  ALL_PROVIDER_NAMES,
+} from "../util/provider-detection.js";
+import type { ProviderDetectionStatus } from "../util/provider-detection.js";
 import { ConflictOverlay } from "../components/ConflictOverlay.js";
 import type {
   Ei_Interface,
@@ -28,8 +34,6 @@ import type {
   Topic,
   Person,
   Quote,
-  ProviderAccount,
-  ProviderType,
   StateConflictData,
   StateConflictResolution,
   ContextStatus,
@@ -109,6 +113,8 @@ export interface EiContextValue {
   }>;
   showWelcomeOverlay: () => boolean;
   dismissWelcomeOverlay: () => void;
+  detectedProviders: () => ProviderDetectionStatus[];
+  firstBootDefaultModel: () => string | undefined;
   deleteMessages: (personaId: string, messageIds: string[]) => Promise<void>;
   setMessageContextStatus: (personaId: string, messageId: string, status: ContextStatus) => Promise<void>;
   deleteRoomMessages: (roomId: string, messageIds: string[]) => Promise<void>;
@@ -146,6 +152,9 @@ export interface EiContextValue {
   humanRoomMessagePending: () => boolean;
   getArchivedRooms: () => RoomSummary[];
   generatePersonaPreview: (name: string, description: string, relationship?: string, personaId?: string) => Promise<import('../../../src/prompts/generation/types.js').PersonaGenerationResult>;
+  importDocument: (filePath: string) => Promise<import('../../../src/integrations/document/types.js').DocumentImportResult>;
+  getUnsourcePreview: (sourceTag: string) => import('../../../src/integrations/document/unsource.js').UnsourcePreview;
+  executeUnsource: (preview: import('../../../src/integrations/document/unsource.js').UnsourcePreview) => Promise<import('../../../src/integrations/document/unsource.js').UnsourceResult>;
 }
 const EiContext = createContext<EiContextValue>();
 
@@ -168,6 +177,8 @@ export const EiProvider: ParentComponent = (props) => {
   const [contextBoundarySignal, setContextBoundarySignal] = createSignal<string | undefined>(undefined);
   const [quotesVersion, setQuotesVersion] = createSignal(0);
   const [showWelcomeOverlay, setShowWelcomeOverlay] = createSignal(false);
+  const [detectedProviders, setDetectedProviders] = createSignal<ProviderDetectionStatus[]>([]);
+  const [firstBootDefaultModel, setFirstBootDefaultModel] = createSignal<string | undefined>(undefined);
   const [conflictData, setConflictData] = createSignal<StateConflictData | null>(null);
 
   let processor: Processor | null = null;
@@ -175,6 +186,7 @@ export const EiProvider: ParentComponent = (props) => {
   let readTimer: Timer | null = null;
   let dwelledPersona: string | null = null;
   let syncConfiguredFromEnv = false;
+  let eiDataPath = "";
 
   const showNotification = (message: string, level: "error" | "warn" | "info") => {
     if (notificationTimer) clearTimeout(notificationTimer);
@@ -320,6 +332,32 @@ export const EiProvider: ParentComponent = (props) => {
     return processor.generatePersonaPreview(name, description, relationship, personaId);
   };
 
+  const importDocument = async (filePath: string) => {
+    if (!processor) throw new Error("Processor not ready");
+    const { readFile } = await import("node:fs/promises");
+    const { basename } = await import("node:path");
+    const { homedir } = await import("node:os");
+    const expandedPath = filePath === "~" || filePath.startsWith("~/")
+      ? homedir() + filePath.slice(1)
+      : filePath.replace(/^\$HOME(?=\/|$)/, homedir());
+    const content = await readFile(expandedPath, "utf-8");
+    const filename = basename(expandedPath);
+    return processor.importDocument(content, filename);
+  };
+
+  const getUnsourcePreview = (sourceTag: string) => {
+    if (!processor) throw new Error("Processor not ready");
+    return processor.getUnsourcePreview(sourceTag);
+  };
+
+  const executeUnsource = async (preview: import('../../../src/integrations/document/unsource.js').UnsourcePreview) => {
+    if (!processor) throw new Error("Processor not ready");
+    const result = await processor.executeUnsource(preview);
+    const { writeUnsourceInvoice } = await import("../../../src/integrations/document/invoice.js");
+    await writeUnsourceInvoice(preview, result, eiDataPath);
+    return result;
+  };
+
   const archivePersona = async (personaId: string) => {
     if (!processor) return;
     await processor.archivePersona(personaId);
@@ -748,64 +786,40 @@ export const EiProvider: ParentComponent = (props) => {
       try {
         const human = await processor!.getHuman();
         const hasAccounts = human.settings?.accounts && human.settings.accounts.length > 0;
-        if (!hasAccounts && E2E_SKIP_LOCAL_DETECT) {
-          logger.info("E2E_SKIP_LOCAL_DETECT active, skipping local LLM check");
-          setShowWelcomeOverlay(true);
-        } else if (!hasAccounts) {
-          logger.info("No LLM accounts configured, checking for local LLMs...");
-
-          const candidates = [
-            { name: "LMStudio", url: "http://127.0.0.1:1234/v1" },
-            { name: "Ollama", url: "http://127.0.0.1:11434/v1" },
-          ];
-
-          const detected = await Promise.all(
-            candidates.map(async (candidate) => {
-              try {
-                const response = await fetch(`${candidate.url}/models`, {
-                  method: "GET",
-                  signal: AbortSignal.timeout(3000),
-                });
-                return response.ok ? candidate : null;
-              } catch {
-                return null;
-              }
-            })
-          );
-
-          const found = detected.filter(Boolean) as typeof candidates;
-
-          if (found.length > 0) {
-            const accounts: ProviderAccount[] = found.map((candidate) => {
-              const defaultModelId = crypto.randomUUID();
-              return {
-                id: crypto.randomUUID(),
-                name: candidate.name,
-                type: "llm" as ProviderType,
-                url: candidate.url,
-                enabled: true,
-                created_at: new Date().toISOString(),
-                default_model: defaultModelId,
-                models: [{ id: defaultModelId, name: "default" }],
-              };
-            });
-            const firstDefaultModelId = accounts[0].default_model!;
-            const currentHuman = await processor!.getHuman();
-            await processor!.updateHuman({
-              settings: {
-                ...currentHuman.settings,
-                accounts,
-                default_model: firstDefaultModelId,
-              },
-            });
-            const names = found.map((c) => c.name).join(" and ");
-            showNotification(`${names} detected and configured!`, "info");
-            logger.info(`Auto-configured: ${names}`);
-          } else {
-            logger.info("No local LLMs found, showing welcome overlay");
-            setShowWelcomeOverlay(true);
-          }
+        if (hasAccounts) return;
+
+        const { detected, statuses } = await detectProviders({
+          skipLocalDetect: E2E_SKIP_LOCAL_DETECT,
+          skipCloudDetect: E2E_SKIP_CLOUD_DETECT,
+        });
+
+        const allStatuses: ProviderDetectionStatus[] = ALL_PROVIDER_NAMES.map((name) => {
+          const found = statuses.find((s) => s.name === name);
+          return found ?? { name, detected: false };
+        });
+        setDetectedProviders(allStatuses);
+
+        if (detected.length > 0) {
+          const accounts = buildProviderAccounts(detected);
+          const topProvider = detected[0];
+          const defaultModel = `${topProvider.name}:${topProvider.selected.extractionModel}`;
+          setFirstBootDefaultModel(defaultModel);
+          const currentHuman = await processor!.getHuman();
+          await processor!.updateHuman({
+            settings: {
+              ...currentHuman.settings,
+              accounts,
+              default_model: defaultModel,
+            },
+          });
+          const names = detected.map((d) => d.name).join(" and ");
+          showNotification(`${names} detected and configured!`, "info");
+          logger.info(`Auto-configured: ${names}`);
+        } else {
+          logger.info("No LLM providers found, showing welcome overlay");
         }
+
+        setShowWelcomeOverlay(true);
       } catch (err: any) {
         logger.warn(`LLM detection failed: ${err?.message || err}`);
       }
@@ -826,6 +840,7 @@ export const EiProvider: ParentComponent = (props) => {
     logger.info("Ei TUI bootstrap starting");
     try {
       const storage = new FileStorage(Bun.env.EI_DATA_PATH);
+      eiDataPath = storage.getDataPath();
       // Pre-configure remoteSync from env vars BEFORE processor.start()
       // so the processor's sync decision tree can detect remote state
       const syncUsername = Bun.env.EI_SYNC_USERNAME;
@@ -969,6 +984,8 @@ export const EiProvider: ParentComponent = (props) => {
     searchHumanData,
     showWelcomeOverlay,
     dismissWelcomeOverlay: () => setShowWelcomeOverlay(false),
+    detectedProviders,
+    firstBootDefaultModel,
     deleteMessages,
     setMessageContextStatus,
     deleteRoomMessages,
@@ -1006,6 +1023,9 @@ export const EiProvider: ParentComponent = (props) => {
     humanRoomMessagePending,
     getArchivedRooms,
     generatePersonaPreview,
+    importDocument,
+    getUnsourcePreview,
+    executeUnsource,
   };
   return (
     <Switch>

package/tui/src/index.tsx

@@ -30,6 +30,20 @@ if (!lockResult.acquired) {
 // Release lock when the app exits (keyboard context calls process.exit(0) on normal quit)
 process.on("exit", () => { void lock.release(); });
 
+// Validate state.json is parseable before handing off to the app.
+// A corrupt file must never silently wipe all data — exit cleanly with recovery instructions.
+try {
+  await storage.load();
+} catch (e) {
+  await lock.release();
+  process.stderr.write(
+    `\nEi cannot start: state.json failed to load.\n\n` +
+    `  ${e instanceof Error ? e.message : String(e)}\n\n` +
+    `Fix the file manually, restore from a backup, or delete it to start fresh (all data will be lost).\n\n`
+  );
+  process.exit(1);
+}
+
 render(App, {
   exitOnCtrlC: false,
   targetFps: 30,

package/tui/src/storage/file.ts

@@ -59,15 +59,21 @@ export class FileStorage implements Storage {
   async load(): Promise<StorageState | null> {
     const filePath = join(this.dataPath, STATE_FILE);
     const file = Bun.file(filePath);
-    
+
     if (await file.exists()) {
+      let text: string;
       try {
-        const text = await file.text();
-        if (text) {
+        text = await file.text();
+      } catch (e) {
+        throw new Error(`STORAGE_READ_FAILED: Could not read ${filePath}: ${e instanceof Error ? e.message : String(e)}`);
+      }
+
+      if (text) {
+        try {
           return decodeAllEmbeddings(JSON.parse(text) as StorageState);
+        } catch (e) {
+          throw new Error(`STORAGE_PARSE_FAILED: ${filePath} exists but could not be parsed as JSON. Your data is intact — fix the file manually or restore from a backup in ${join(this.dataPath, "backups")}.\n  Parse error: ${e instanceof Error ? e.message : String(e)}`);
         }
-      } catch {
-        return null;
       }
     }
 

package/tui/src/util/e2e-flags.ts

@@ -3,11 +3,12 @@
  *
  * Use prime-power bits so combinations are unambiguous:
  *   1  — skip local LLM auto-detect (fetch to :1234/:11434)
- *   2  — (reserved for next scenario)
- *   3  — flags 1 + 2 combined
+ *   2  — skip cloud provider auto-detect (env var → /models checks)
+ *   3  — flags 1 + 2 combined (skip all auto-detect)
  *
  * Production code should never set this. Tests pass it via env in test.use({ env: { EI_E2E_MODE: "1" } }).
  */
 const E2E_MODE = parseInt(process.env.EI_E2E_MODE ?? "0", 10);
 
-export const E2E_SKIP_LOCAL_DETECT = (E2E_MODE & 1) !== 0;
+export const E2E_SKIP_LOCAL_DETECT  = (E2E_MODE & 1) !== 0;
+export const E2E_SKIP_CLOUD_DETECT  = (E2E_MODE & 2) !== 0;

package/tui/src/util/help-content.ts

@@ -89,6 +89,26 @@ ROOM COMMANDS
         /capture topic <name>   Re-scan all messages for a specific topic.
 
 EXTENDED COMMANDS
+    /reflect
+        Review a persona's pending reflection — a proposed identity update
+        generated by Ei after observing patterns in your conversations.
+        A badge appears on the persona pill when one is ready.
+        /reflect generate   Write current + proposed YAML files to disk
+        /reflect update     Read edited proposed.yaml back into Ei
+        /reflect apply      Apply the proposed identity to the persona
+        /reflect dismiss    Discard without changing anything
+
+    /import <path>
+        Import a document (txt, md, pdf, etc.) into Ei. Ei segments it,
+        extracts knowledge, and attributes it to the "Emmett" persona.
+        /import ~/notes/journal.md
+        /import /path/to/report.pdf
+
+    /unsource <source_tag>
+        Remove all facts, topics, etc. extracted from a previously imported
+        document. Use the source tag shown when the import completed.
+        /unsource my-journal-2024
+
     /tools
         Manage tool providers — enable or disable tools per persona.
 

package/tui/src/util/logger.ts

@@ -22,7 +22,7 @@ const LOG_LEVELS: Record<LogLevel, number> = {
   error: 3,
 };
 
-const currentLevel: LogLevel = (Bun.env.EI_LOG_LEVEL as LogLevel) || "debug";
+const currentLevel: LogLevel = (Bun.env.EI_LOG_LEVEL as LogLevel) || "warn";
 
 function shouldLog(level: LogLevel): boolean {
   return LOG_LEVELS[level] >= LOG_LEVELS[currentLevel];

package/tui/src/util/provider-detection.ts

@@ -0,0 +1,251 @@
+import type { ProviderType } from "../../../src/core/types.js";
+import type { ProviderAccount, ModelConfig } from "../../../src/core/types.js";
+
+export interface LocalProviderConfig {
+  name: string;
+  url: string;
+  priority: number;
+}
+
+export interface CloudProviderConfig {
+  name: string;
+  envVar: string;
+  url: string;
+  priority: number;
+}
+
+export interface SelectedModels {
+  extractionModel: string;
+  chatModel: string;
+  bonusModel?: string;
+}
+
+export interface ProviderDetectionResult {
+  name: string;
+  url: string;
+  apiKey?: string;
+  modelIds: string[];
+  selected: SelectedModels;
+  status: "detected" | "failed";
+}
+
+export interface ProviderDetectionStatus {
+  name: string;
+  detected: boolean;
+}
+
+export interface DetectProvidersOptions {
+  skipLocalDetect?: boolean;
+  skipCloudDetect?: boolean;
+  env?: Record<string, string | undefined>;
+}
+
+export const LOCAL_PROVIDERS: ReadonlyArray<LocalProviderConfig> = [
+  { name: "LMStudio", url: "http://127.0.0.1:1234/v1",  priority: 1 },
+  { name: "Ollama",   url: "http://127.0.0.1:11434/v1", priority: 2 },
+];
+
+export const CLOUD_PROVIDERS: ReadonlyArray<CloudProviderConfig> = [
+  { name: "Anthropic", envVar: "ANTHROPIC_API_KEY", url: "https://api.anthropic.com/v1",                             priority: 3 },
+  { name: "OpenAI",    envVar: "OPENAI_API_KEY",    url: "https://api.openai.com/v1",                                priority: 4 },
+  { name: "Groq",      envVar: "GROQ_API_KEY",      url: "https://api.groq.com/openai/v1",                          priority: 5 },
+  { name: "Mistral",   envVar: "MISTRAL_API_KEY",   url: "https://api.mistral.ai/v1",                               priority: 6 },
+  { name: "Gemini",    envVar: "GEMINI_API_KEY",    url: "https://generativelanguage.googleapis.com/v1beta/openai", priority: 7 },
+];
+
+export const ALL_PROVIDER_NAMES: ReadonlyArray<string> = [
+  ...LOCAL_PROVIDERS.map((p) => p.name),
+  ...CLOUD_PROVIDERS.map((p) => p.name),
+];
+
+function latestMatch(modelIds: string[], pattern: string): string | undefined {
+  const matches = modelIds.filter((id) => id.toLowerCase().includes(pattern));
+  if (matches.length === 0) return undefined;
+  return [...matches].sort().reverse()[0];
+}
+
+export function selectModelsForProvider(
+  providerName: string,
+  modelIds: string[]
+): SelectedModels {
+  const name = providerName.toLowerCase();
+
+  if (name === "groq") {
+    return {
+      extractionModel: "llama-3.1-8b-instant",
+      chatModel: "llama-3.3-70b-versatile",
+    };
+  }
+
+  if (modelIds.length === 0) {
+    return { extractionModel: "default", chatModel: "default" };
+  }
+
+  if (name === "anthropic") {
+    return {
+      extractionModel: latestMatch(modelIds, "haiku") ?? modelIds[0],
+      chatModel:       latestMatch(modelIds, "sonnet") ?? modelIds[0],
+      bonusModel:      latestMatch(modelIds, "opus"),
+    };
+  }
+
+  if (name === "openai") {
+    const gpt4oNonMini = modelIds.filter(
+      (id) => id.toLowerCase().includes("gpt-4o") && !id.toLowerCase().includes("mini")
+    );
+    return {
+      extractionModel: latestMatch(modelIds, "mini") ?? modelIds[0],
+      chatModel: gpt4oNonMini.length > 0
+        ? [...gpt4oNonMini].sort().reverse()[0]
+        : modelIds[0],
+    };
+  }
+
+  if (name === "mistral") {
+    return {
+      extractionModel: latestMatch(modelIds, "small") ?? modelIds[0],
+      chatModel:       latestMatch(modelIds, "large") ?? modelIds[0],
+    };
+  }
+
+  if (name === "gemini") {
+    return {
+      extractionModel: latestMatch(modelIds, "flash") ?? modelIds[0],
+      chatModel:       latestMatch(modelIds, "pro")   ?? modelIds[0],
+    };
+  }
+
+  return { extractionModel: modelIds[0], chatModel: modelIds[0] };
+}
+
+type FetchFn = (url: string, init?: RequestInit) => Promise<Response>;
+
+function buildAuthHeaders(url: string, apiKey: string | undefined): Record<string, string> {
+  if (!apiKey) return {};
+  if (url.includes("api.anthropic.com")) {
+    return {
+      "x-api-key": apiKey,
+      "anthropic-version": "2023-06-01",
+    };
+  }
+  return { "Authorization": `Bearer ${apiKey}` };
+}
+
+async function probeModels(
+  url: string,
+  apiKey: string | undefined,
+  fetchFn: FetchFn
+): Promise<string[] | null> {
+  try {
+    const headers = buildAuthHeaders(url, apiKey);
+    const response = await fetchFn(`${url}/models`, {
+      method: "GET",
+      headers,
+      signal: AbortSignal.timeout(3000),
+    });
+    if (!response.ok) return null;
+    const json = await response.json() as { data?: Array<{ id: string }> };
+    return (json.data ?? []).map((m) => m.id).filter(Boolean);
+  } catch {
+    return null;
+  }
+}
+
+export async function detectProviders(
+  options: DetectProvidersOptions = {},
+  fetchFn: FetchFn = fetch
+): Promise<{
+  detected: ProviderDetectionResult[];
+  statuses: ProviderDetectionStatus[];
+}> {
+  const env = options.env ?? (process.env as Record<string, string | undefined>);
+  const detected: ProviderDetectionResult[] = [];
+  const statuses: ProviderDetectionStatus[] = [];
+
+  const localResults = await Promise.all(
+    LOCAL_PROVIDERS.map(async (provider) => {
+      if (options.skipLocalDetect) return { provider, modelIds: null };
+      const modelIds = await probeModels(provider.url, undefined, fetchFn);
+      return { provider, modelIds };
+    })
+  );
+
+  for (const { provider, modelIds } of localResults) {
+    const ok = modelIds !== null;
+    statuses.push({ name: provider.name, detected: ok });
+    if (ok) {
+      detected.push({
+        name: provider.name,
+        url: provider.url,
+        modelIds: modelIds!,
+        selected: selectModelsForProvider(provider.name, modelIds!),
+        status: "detected",
+      });
+    }
+  }
+
+  const cloudResults = await Promise.all(
+    CLOUD_PROVIDERS.map(async (provider) => {
+      const apiKey = env[provider.envVar];
+      if (!apiKey) return { provider, apiKey: undefined, modelIds: null };
+      if (options.skipCloudDetect) return { provider, apiKey, modelIds: null };
+      const modelIds = await probeModels(provider.url, apiKey, fetchFn);
+      return { provider, apiKey, modelIds };
+    })
+  );
+
+  for (const { provider, apiKey, modelIds } of cloudResults) {
+    const ok = modelIds !== null;
+    statuses.push({ name: provider.name, detected: ok });
+    if (ok) {
+      detected.push({
+        name: provider.name,
+        url: provider.url,
+        apiKey,
+        modelIds: modelIds!,
+        selected: selectModelsForProvider(provider.name, modelIds!),
+        status: "detected",
+      });
+    }
+  }
+
+  return { detected, statuses };
+}
+
+export function buildProviderAccounts(
+  detected: ProviderDetectionResult[]
+): ProviderAccount[] {
+  return detected.map((d) => {
+    const makeModel = (modelName: string): ModelConfig => ({
+      id: crypto.randomUUID(),
+      name: modelName,
+    });
+
+    const seenNames = new Set<string>();
+    const models: ModelConfig[] = [];
+
+    const pushIfNew = (name: string) => {
+      if (!seenNames.has(name)) {
+        seenNames.add(name);
+        models.push(makeModel(name));
+      }
+    };
+
+    pushIfNew(d.selected.chatModel);
+    pushIfNew(d.selected.extractionModel);
+    if (d.selected.bonusModel) pushIfNew(d.selected.bonusModel);
+    for (const id of d.modelIds) pushIfNew(id);
+
+    return {
+      id: crypto.randomUUID(),
+      name: d.name,
+      type: "llm" as ProviderType,
+      url: d.url,
+      api_key: d.apiKey,
+      enabled: true,
+      created_at: new Date().toISOString(),
+      default_model: d.selected.chatModel,
+      models,
+    };
+  });
+}

package/tui/src/util/yaml-human.ts

@@ -191,6 +191,12 @@ export function humanToYAML(
       })
       .replace(/^(\s+)(identifiers:)/mg, (_, indent, _key) => {
         return `${indent}${personComment}\n${indent}identifiers:`;
+      })
+      .replace(/^( +)(sources:\n(?:\1  - .+\n)*)/mg, (match, indent, block) => {
+        return block
+          .split('\n')
+          .map(line => line.trim() ? `${indent}# [read-only] ${line}` : line)
+          .join('\n');
       });
 
   const serializeSection = (key: "facts" | "topics" | "people", items: unknown[] | undefined): string => {
@@ -199,7 +205,7 @@ export function humanToYAML(
       return `${key}:\n${stub}`;
     }
     const ordered = (items as object[]).map(canonicalFieldOrder);
-    const itemsYaml = YAML.stringify(ordered, { lineWidth: 0 })
+    const itemsYaml = YAML.stringify(ordered, { lineWidth: 0, aliasDuplicateObjects: false })
       .split('\n')
       .map(line => `  ${line}`)
       .join('\n')