ei-tui 0.9.4 → 1.0.0

package/tui/src/commands/unsource.tsx

@@ -0,0 +1,115 @@
+import type { Command } from "./registry";
+import { ConfirmOverlay } from "../components/ConfirmOverlay";
+import { PersonaListOverlay } from "../components/PersonaListOverlay";
+
+export const unsourceCommand: Command = {
+  name: "unsource",
+  aliases: [],
+  description: "Remove knowledge extracted from a specific document source",
+  usage: "/unsource <sourceTag>",
+
+  async execute(args, ctx) {
+    if (args.length === 0) {
+      const human = await ctx.ei.getHuman();
+      const docs = human.settings?.document?.processed_documents ?? {};
+      const sources = Object.keys(docs);
+
+      if (sources.length === 0) {
+        ctx.showNotification("No imported documents found. Use /import first.", "warn");
+        return;
+      }
+
+      const items = sources.map(f => ({
+        id: `import:document:${f}`,
+        display_name: `import:document:${f}`,
+        aliases: [] as string[],
+        is_paused: false,
+        is_archived: false,
+        unread_count: 0,
+        has_pending_update: false,
+      }));
+
+      ctx.showOverlay((hideOverlay) => (
+        <PersonaListOverlay
+          personas={items}
+          activePersonaId={null}
+          title="Select source to unsource"
+          onSelect={async (sourceTag) => {
+            hideOverlay();
+            await unsourceCommand.execute([sourceTag], ctx);
+          }}
+          onDismiss={hideOverlay}
+        />
+      ), ctx.renderer);
+      return;
+    }
+
+    const rawArg = args.join(" ").trim();
+
+    let sourceTag = rawArg;
+    if (!rawArg.includes(":")) {
+      const human = await ctx.ei.getHuman();
+      const docs = human.settings?.document?.processed_documents ?? {};
+      const allSources = Object.keys(docs).map(f => `import:document:${f}`);
+      const matches = allSources.filter(s => s.endsWith(rawArg) || s.includes(rawArg));
+      if (matches.length === 1) {
+        sourceTag = matches[0];
+      } else if (matches.length > 1) {
+        ctx.showNotification(`Ambiguous: "${rawArg}" matches multiple sources. Use /unsource with no args to pick.`, "warn");
+        return;
+      }
+    }
+
+    const preview = ctx.ei.getUnsourcePreview(sourceTag);
+
+    const totalDelete =
+      preview.toDelete.facts.length +
+      preview.toDelete.topics.length +
+      preview.toDelete.people.length;
+    const totalStrip =
+      preview.toStrip.facts.length +
+      preview.toStrip.topics.length +
+      preview.toStrip.people.length;
+
+    if (
+      totalDelete === 0 &&
+      preview.toDelete.quotes.length === 0 &&
+      totalStrip === 0
+    ) {
+      ctx.showNotification(`No knowledge found for source: ${sourceTag}`, "warn");
+      return;
+    }
+
+    const confirmed = await new Promise<boolean>((resolve) => {
+      const msg = [
+        `Unsource: ${sourceTag}`,
+        "",
+        `Delete: ${preview.toDelete.facts.length} facts, ${preview.toDelete.topics.length} topics, ${preview.toDelete.people.length} people, ${preview.toDelete.quotes.length} quotes`,
+        `Strip source: ${preview.toStrip.facts.length} facts, ${preview.toStrip.topics.length} topics, ${preview.toStrip.people.length} people`,
+        "",
+        "This cannot be undone. Proceed? [y/N]",
+      ].join("\n");
+
+      ctx.showOverlay((hideOverlay, _hideForEditor) => (
+        <ConfirmOverlay
+          message={msg}
+          onConfirm={() => { hideOverlay(); resolve(true); }}
+          onCancel={() => { hideOverlay(); resolve(false); }}
+        />
+      ), ctx.renderer);
+    });
+
+    if (!confirmed) {
+      ctx.showNotification("Cancelled", "info");
+      return;
+    }
+
+    const result = await ctx.ei.executeUnsource(preview);
+    const deletedTotal = result.deleted.facts + result.deleted.topics + result.deleted.people + result.deleted.quotes;
+    const strippedTotal = result.stripped.facts + result.stripped.topics + result.stripped.people;
+    ctx.showNotification(
+      `Unsourced ${sourceTag}: deleted ${deletedTotal} items, stripped source from ${strippedTotal} items`,
+      "info"
+    );
+  },
+};

package/tui/src/components/PromptInput.tsx

@@ -30,6 +30,8 @@ import { activateCommand } from "../commands/activate.js";
 import { reflectCommand } from "../commands/reflect.js";
 import { silenceCommand } from "../commands/silence.js";
 import { captureCommand } from "../commands/capture.js";
+import { importCommand } from "../commands/import.js";
+import { unsourceCommand } from "../commands/unsource.js";
 import { openCYPEditor } from "../util/cyp-editor.js";
 import { useOverlay } from "../context/overlay";
 import { CommandSuggest } from "./CommandSuggest";
@@ -86,6 +88,8 @@ export function PromptInput() {
   registerCommand(activateCommand);
   registerCommand(silenceCommand);
   registerCommand(captureCommand);
+  registerCommand(importCommand);
+  registerCommand(unsourceCommand);
   registerCommand(authCommand);
   registerCommand(pauseCommand);
   registerCommand(resumeCommand);

package/tui/src/components/WelcomeOverlay.tsx

@@ -1,11 +1,16 @@
 import { useKeyboard } from "@opentui/solid";
-import { onMount, onCleanup } from "solid-js";
+import { onMount, onCleanup, For } from "solid-js";
 import { useKeyboardNav } from "../context/keyboard.js";
+import type { ProviderDetectionStatus } from "../util/provider-detection.js";
 
 interface WelcomeOverlayProps {
   onDismiss: () => void;
+  detectedProviders: ProviderDetectionStatus[];
+  defaultModel?: string;
 }
 
+const COLUMNS = 3;
+
 export function WelcomeOverlay(props: WelcomeOverlayProps) {
   const { setOverlayActive } = useKeyboardNav();
   onMount(() => setOverlayActive(true));
@@ -16,6 +21,17 @@ export function WelcomeOverlay(props: WelcomeOverlayProps) {
     props.onDismiss();
   });
 
+  const hasAny = () => props.detectedProviders.some((p) => p.detected);
+
+  const rows = () => {
+    const items = props.detectedProviders;
+    const out: ProviderDetectionStatus[][] = [];
+    for (let i = 0; i < items.length; i += COLUMNS) {
+      out.push(items.slice(i, i + COLUMNS));
+    }
+    return out;
+  };
+
   return (
     <box
       position="absolute"
@@ -35,44 +51,54 @@ export function WelcomeOverlay(props: WelcomeOverlayProps) {
         padding={2}
         flexDirection="column"
       >
-        <text fg="#eee8d5">
-          Welcome to Ei!
-        </text>
+        <text fg="#eee8d5">Welcome to Ei!</text>
         <text> </text>
 
-        <text fg="#dc322f">
-          No LLM provider detected.
-        </text>
-        <text> </text>
+        <box visible={hasAny()} flexDirection="column">
+          <text fg="#93a1a1">Detected providers:</text>
+          <text> </text>
 
-        <text fg="#93a1a1">
-          To get started, you need a local LLM running or a provider configured.
-        </text>
-        <text> </text>
+          <For each={rows()}>
+            {(row) => (
+              <box flexDirection="row">
+                <For each={row}>
+                  {(provider) => (
+                    <box width={22} flexDirection="row">
+                      <text fg="#93a1a1">{provider.name}:</text>
+                      <text> </text>
+                      <text fg={provider.detected ? "#859900" : "#dc322f"}>
+                        {provider.detected ? "[✓]" : "[✗]"}
+                      </text>
+                    </box>
+                  )}
+                </For>
+              </box>
+            )}
+          </For>
 
-        <text fg="#93a1a1">
-          Options:
-        </text>
-        <text fg="#93a1a1">
-          1. Start LMStudio (port 1234) or Ollama (port 11434)
-        </text>
-        <text fg="#93a1a1">
-          2. Run /provider new to configure a cloud provider
-        </text>
-        <text> </text>
+          <text> </text>
+          <box visible={!!props.defaultModel} flexDirection="row">
+            <text fg="#657b83">Default model: </text>
+            <text fg="#eee8d5">{props.defaultModel ?? ""}</text>
+          </box>
+          <text> </text>
+          <text fg="#93a1a1">To chat with a smarter model, try: /provider</text>
+          <text fg="#93a1a1">To change your default, use: /settings</text>
+          <text fg="#93a1a1">See /help for... well, Help!</text>
+        </box>
 
-        <text fg="#657b83">
-          Once configured, restart Ei or run /provider new to add your provider.
-        </text>
-        <text> </text>
+        <box visible={!hasAny()} flexDirection="column">
+          <text fg="#dc322f">No LLM provider detected.</text>
+          <text> </text>
+          <text fg="#93a1a1">Start LMStudio (port 1234) or Ollama (port 11434), or</text>
+          <text fg="#93a1a1">set one of: ANTHROPIC_API_KEY, OPENAI_API_KEY, GROQ_API_KEY,</text>
+          <text fg="#93a1a1">MISTRAL_API_KEY, GEMINI_API_KEY and restart.</text>
+        </box>
 
-        <text fg="#586e75">
-          Press any key to dismiss
-        </text>
         <text> </text>
-        <text fg="#2a2a3e">
-          Ei - 永 (ei) - eternal
-        </text>
+        <text fg="#586e75">Press any key to continue</text>
+        <text> </text>
+        <text fg="#2a2a3e">Ei - 永 (ei) - eternal</text>
       </box>
     </box>
   );

package/tui/src/context/ei.tsx

@@ -14,7 +14,13 @@ import { Processor } from "../../../src/core/processor.js";
 import { FileStorage } from "../storage/file.js";
 import { remoteSync } from "../../../src/storage/remote.js";
 import { logger, clearLog, interceptConsole } from "../util/logger.js";
-import { E2E_SKIP_LOCAL_DETECT } from "../util/e2e-flags.js";
+import { E2E_SKIP_LOCAL_DETECT, E2E_SKIP_CLOUD_DETECT } from "../util/e2e-flags.js";
+import {
+  detectProviders,
+  buildProviderAccounts,
+  ALL_PROVIDER_NAMES,
+} from "../util/provider-detection.js";
+import type { ProviderDetectionStatus } from "../util/provider-detection.js";
 import { ConflictOverlay } from "../components/ConflictOverlay.js";
 import type {
   Ei_Interface,
@@ -28,8 +34,6 @@ import type {
   Topic,
   Person,
   Quote,
-  ProviderAccount,
-  ProviderType,
   StateConflictData,
   StateConflictResolution,
   ContextStatus,
@@ -109,6 +113,8 @@ export interface EiContextValue {
   }>;
   showWelcomeOverlay: () => boolean;
   dismissWelcomeOverlay: () => void;
+  detectedProviders: () => ProviderDetectionStatus[];
+  firstBootDefaultModel: () => string | undefined;
   deleteMessages: (personaId: string, messageIds: string[]) => Promise<void>;
   setMessageContextStatus: (personaId: string, messageId: string, status: ContextStatus) => Promise<void>;
   deleteRoomMessages: (roomId: string, messageIds: string[]) => Promise<void>;
@@ -146,6 +152,9 @@ export interface EiContextValue {
   humanRoomMessagePending: () => boolean;
   getArchivedRooms: () => RoomSummary[];
   generatePersonaPreview: (name: string, description: string, relationship?: string, personaId?: string) => Promise<import('../../../src/prompts/generation/types.js').PersonaGenerationResult>;
+  importDocument: (filePath: string) => Promise<import('../../../src/integrations/document/types.js').DocumentImportResult>;
+  getUnsourcePreview: (sourceTag: string) => import('../../../src/integrations/document/unsource.js').UnsourcePreview;
+  executeUnsource: (preview: import('../../../src/integrations/document/unsource.js').UnsourcePreview) => Promise<import('../../../src/integrations/document/unsource.js').UnsourceResult>;
 }
 const EiContext = createContext<EiContextValue>();
 
@@ -168,6 +177,8 @@ export const EiProvider: ParentComponent = (props) => {
   const [contextBoundarySignal, setContextBoundarySignal] = createSignal<string | undefined>(undefined);
   const [quotesVersion, setQuotesVersion] = createSignal(0);
   const [showWelcomeOverlay, setShowWelcomeOverlay] = createSignal(false);
+  const [detectedProviders, setDetectedProviders] = createSignal<ProviderDetectionStatus[]>([]);
+  const [firstBootDefaultModel, setFirstBootDefaultModel] = createSignal<string | undefined>(undefined);
   const [conflictData, setConflictData] = createSignal<StateConflictData | null>(null);
 
   let processor: Processor | null = null;
@@ -175,6 +186,7 @@ export const EiProvider: ParentComponent = (props) => {
   let readTimer: Timer | null = null;
   let dwelledPersona: string | null = null;
   let syncConfiguredFromEnv = false;
+  let eiDataPath = "";
 
   const showNotification = (message: string, level: "error" | "warn" | "info") => {
     if (notificationTimer) clearTimeout(notificationTimer);
@@ -320,6 +332,32 @@ export const EiProvider: ParentComponent = (props) => {
     return processor.generatePersonaPreview(name, description, relationship, personaId);
   };
 
+  const importDocument = async (filePath: string) => {
+    if (!processor) throw new Error("Processor not ready");
+    const { readFile } = await import("node:fs/promises");
+    const { basename } = await import("node:path");
+    const { homedir } = await import("node:os");
+    const expandedPath = filePath === "~" || filePath.startsWith("~/")
+      ? homedir() + filePath.slice(1)
+      : filePath.replace(/^\$HOME(?=\/|$)/, homedir());
+    const content = await readFile(expandedPath, "utf-8");
+    const filename = basename(expandedPath);
+    return processor.importDocument(content, filename);
+  };
+
+  const getUnsourcePreview = (sourceTag: string) => {
+    if (!processor) throw new Error("Processor not ready");
+    return processor.getUnsourcePreview(sourceTag);
+  };
+
+  const executeUnsource = async (preview: import('../../../src/integrations/document/unsource.js').UnsourcePreview) => {
+    if (!processor) throw new Error("Processor not ready");
+    const result = await processor.executeUnsource(preview);
+    const { writeUnsourceInvoice } = await import("../../../src/integrations/document/invoice.js");
+    await writeUnsourceInvoice(preview, result, eiDataPath);
+    return result;
+  };
+
   const archivePersona = async (personaId: string) => {
     if (!processor) return;
     await processor.archivePersona(personaId);
@@ -748,64 +786,40 @@ export const EiProvider: ParentComponent = (props) => {
       try {
         const human = await processor!.getHuman();
         const hasAccounts = human.settings?.accounts && human.settings.accounts.length > 0;
-        if (!hasAccounts && E2E_SKIP_LOCAL_DETECT) {
-          logger.info("E2E_SKIP_LOCAL_DETECT active, skipping local LLM check");
-          setShowWelcomeOverlay(true);
-        } else if (!hasAccounts) {
-          logger.info("No LLM accounts configured, checking for local LLMs...");
-
-          const candidates = [
-            { name: "LMStudio", url: "http://127.0.0.1:1234/v1" },
-            { name: "Ollama", url: "http://127.0.0.1:11434/v1" },
-          ];
-
-          const detected = await Promise.all(
-            candidates.map(async (candidate) => {
-              try {
-                const response = await fetch(`${candidate.url}/models`, {
-                  method: "GET",
-                  signal: AbortSignal.timeout(3000),
-                });
-                return response.ok ? candidate : null;
-              } catch {
-                return null;
-              }
-            })
-          );
-
-          const found = detected.filter(Boolean) as typeof candidates;
-
-          if (found.length > 0) {
-            const accounts: ProviderAccount[] = found.map((candidate) => {
-              const defaultModelId = crypto.randomUUID();
-              return {
-                id: crypto.randomUUID(),
-                name: candidate.name,
-                type: "llm" as ProviderType,
-                url: candidate.url,
-                enabled: true,
-                created_at: new Date().toISOString(),
-                default_model: defaultModelId,
-                models: [{ id: defaultModelId, name: "default" }],
-              };
-            });
-            const firstDefaultModelId = accounts[0].default_model!;
-            const currentHuman = await processor!.getHuman();
-            await processor!.updateHuman({
-              settings: {
-                ...currentHuman.settings,
-                accounts,
-                default_model: firstDefaultModelId,
-              },
-            });
-            const names = found.map((c) => c.name).join(" and ");
-            showNotification(`${names} detected and configured!`, "info");
-            logger.info(`Auto-configured: ${names}`);
-          } else {
-            logger.info("No local LLMs found, showing welcome overlay");
-            setShowWelcomeOverlay(true);
-          }
+        if (hasAccounts) return;
+
+        const { detected, statuses } = await detectProviders({
+          skipLocalDetect: E2E_SKIP_LOCAL_DETECT,
+          skipCloudDetect: E2E_SKIP_CLOUD_DETECT,
+        });
+
+        const allStatuses: ProviderDetectionStatus[] = ALL_PROVIDER_NAMES.map((name) => {
+          const found = statuses.find((s) => s.name === name);
+          return found ?? { name, detected: false };
+        });
+        setDetectedProviders(allStatuses);
+
+        if (detected.length > 0) {
+          const accounts = buildProviderAccounts(detected);
+          const topProvider = detected[0];
+          const defaultModel = `${topProvider.name}:${topProvider.selected.extractionModel}`;
+          setFirstBootDefaultModel(defaultModel);
+          const currentHuman = await processor!.getHuman();
+          await processor!.updateHuman({
+            settings: {
+              ...currentHuman.settings,
+              accounts,
+              default_model: defaultModel,
+            },
+          });
+          const names = detected.map((d) => d.name).join(" and ");
+          showNotification(`${names} detected and configured!`, "info");
+          logger.info(`Auto-configured: ${names}`);
+        } else {
+          logger.info("No LLM providers found, showing welcome overlay");
         }
+
+        setShowWelcomeOverlay(true);
       } catch (err: any) {
         logger.warn(`LLM detection failed: ${err?.message || err}`);
       }
@@ -826,6 +840,7 @@ export const EiProvider: ParentComponent = (props) => {
     logger.info("Ei TUI bootstrap starting");
     try {
       const storage = new FileStorage(Bun.env.EI_DATA_PATH);
+      eiDataPath = storage.getDataPath();
       // Pre-configure remoteSync from env vars BEFORE processor.start()
       // so the processor's sync decision tree can detect remote state
       const syncUsername = Bun.env.EI_SYNC_USERNAME;
@@ -969,6 +984,8 @@ export const EiProvider: ParentComponent = (props) => {
     searchHumanData,
     showWelcomeOverlay,
     dismissWelcomeOverlay: () => setShowWelcomeOverlay(false),
+    detectedProviders,
+    firstBootDefaultModel,
     deleteMessages,
     setMessageContextStatus,
     deleteRoomMessages,
@@ -1006,6 +1023,9 @@ export const EiProvider: ParentComponent = (props) => {
     humanRoomMessagePending,
     getArchivedRooms,
     generatePersonaPreview,
+    importDocument,
+    getUnsourcePreview,
+    executeUnsource,
   };
   return (
     <Switch>

package/tui/src/index.tsx

@@ -30,6 +30,20 @@ if (!lockResult.acquired) {
 // Release lock when the app exits (keyboard context calls process.exit(0) on normal quit)
 process.on("exit", () => { void lock.release(); });
 
+// Validate state.json is parseable before handing off to the app.
+// A corrupt file must never silently wipe all data — exit cleanly with recovery instructions.
+try {
+  await storage.load();
+} catch (e) {
+  await lock.release();
+  process.stderr.write(
+    `\nEi cannot start: state.json failed to load.\n\n` +
+    `  ${e instanceof Error ? e.message : String(e)}\n\n` +
+    `Fix the file manually, restore from a backup, or delete it to start fresh (all data will be lost).\n\n`
+  );
+  process.exit(1);
+}
+
 render(App, {
   exitOnCtrlC: false,
   targetFps: 30,

package/tui/src/storage/file.ts

@@ -59,15 +59,21 @@ export class FileStorage implements Storage {
   async load(): Promise<StorageState | null> {
     const filePath = join(this.dataPath, STATE_FILE);
     const file = Bun.file(filePath);
-    
+
     if (await file.exists()) {
+      let text: string;
       try {
-        const text = await file.text();
-        if (text) {
+        text = await file.text();
+      } catch (e) {
+        throw new Error(`STORAGE_READ_FAILED: Could not read ${filePath}: ${e instanceof Error ? e.message : String(e)}`);
+      }
+
+      if (text) {
+        try {
           return decodeAllEmbeddings(JSON.parse(text) as StorageState);
+        } catch (e) {
+          throw new Error(`STORAGE_PARSE_FAILED: ${filePath} exists but could not be parsed as JSON. Your data is intact — fix the file manually or restore from a backup in ${join(this.dataPath, "backups")}.\n  Parse error: ${e instanceof Error ? e.message : String(e)}`);
         }
-      } catch {
-        return null;
       }
     }
 

package/tui/src/util/e2e-flags.ts

@@ -3,11 +3,12 @@
  *
  * Use prime-power bits so combinations are unambiguous:
  *   1  — skip local LLM auto-detect (fetch to :1234/:11434)
- *   2  — (reserved for next scenario)
- *   3  — flags 1 + 2 combined
+ *   2  — skip cloud provider auto-detect (env var → /models checks)
+ *   3  — flags 1 + 2 combined (skip all auto-detect)
  *
  * Production code should never set this. Tests pass it via env in test.use({ env: { EI_E2E_MODE: "1" } }).
  */
 const E2E_MODE = parseInt(process.env.EI_E2E_MODE ?? "0", 10);
 
-export const E2E_SKIP_LOCAL_DETECT = (E2E_MODE & 1) !== 0;
+export const E2E_SKIP_LOCAL_DETECT  = (E2E_MODE & 1) !== 0;
+export const E2E_SKIP_CLOUD_DETECT  = (E2E_MODE & 2) !== 0;

package/tui/src/util/help-content.ts

@@ -89,6 +89,26 @@ ROOM COMMANDS
         /capture topic <name>   Re-scan all messages for a specific topic.
 
 EXTENDED COMMANDS
+    /reflect
+        Review a persona's pending reflection — a proposed identity update
+        generated by Ei after observing patterns in your conversations.
+        A badge appears on the persona pill when one is ready.
+        /reflect generate   Write current + proposed YAML files to disk
+        /reflect update     Read edited proposed.yaml back into Ei
+        /reflect apply      Apply the proposed identity to the persona
+        /reflect dismiss    Discard without changing anything
+
+    /import <path>
+        Import a document (txt, md, pdf, etc.) into Ei. Ei segments it,
+        extracts knowledge, and attributes it to the "Emmett" persona.
+        /import ~/notes/journal.md
+        /import /path/to/report.pdf
+
+    /unsource <source_tag>
+        Remove all facts, topics, etc. extracted from a previously imported
+        document. Use the source tag shown when the import completed.
+        /unsource my-journal-2024
+
     /tools
         Manage tool providers — enable or disable tools per persona.