ei-tui 1.1.0 → 1.2.0

package/src/prompts/synthesis/index.ts

@@ -0,0 +1,101 @@
+import type { SynthesisPromptData } from "./types.js";
+import type { PromptOutput } from "../response/types.js";
+
+export type { SynthesisPromptData, EnrichedTopic, EnrichedPerson } from "./types.js";
+
+export function buildSynthesisPrompt(data: SynthesisPromptData): PromptOutput {
+  const hasEntityMap = data.loadedEntityNames !== undefined;
+
+  const system = `You are synthesizing a knowledge document from a personal knowledge base called Ei.
+
+Your goal is to produce a well-structured markdown document that a human could share with a teammate, hand to their future self, or use as a reference. Write as if you are distilling what someone actually knows — not restating a list of facts, but synthesizing relationships, context, and meaning.
+
+## What you have been given
+
+Everything below is complete as provided — do not use tools to re-fetch records already present here. Only use tools to fill genuine gaps not covered by the data below.
+
+- **Facts**: Ground-truth statements.
+- **Topics**: Areas of interest, work, or concern with descriptions.
+- **People**: Individuals with relationship context.
+- **Quotes**: Verbatim things said, with a \`message_id\`. Use \`fetch_message\` with the \`message_id\` if you want the surrounding conversation for additional context.${hasEntityMap ? `
+- **Quote links**: Each quote lists the entities it was extracted from. Entities marked \`(not loaded)\` were referenced by that quote but are not present in this payload — use \`fetch_memory\` with the entity ID to retrieve them if the gap is relevant to your synthesis.` : ""}
+
+## Output
+
+Write clean, structured markdown. Use headings. Synthesize — do not just restate the bullets. Where the data tells a story or shows a pattern, say so. Where something is uncertain or a work-in-progress, reflect that. Aim for the document a thoughtful person would write after reviewing all of this, not a formatted dump.`;
+
+  const lines: string[] = [`# ${data.subject}`, ""];
+
+  const formatQuoteLinks = (dataItemIds: string[]): string | null => {
+    if (!hasEntityMap || dataItemIds.length === 0) return null;
+    const labels = dataItemIds.map(id => {
+      const name = data.loadedEntityNames!.get(id);
+      return name ? `[id:${id}] ${name}` : `[id:${id}] (not loaded)`;
+    });
+    return `  _Links: ${labels.join(", ")}_`;
+  };
+
+  if (data.facts.length > 0) {
+    lines.push("## Facts");
+    for (const fact of data.facts) {
+      lines.push(`- [id:${fact.id}] **${fact.name}**: ${fact.description}`);
+    }
+    lines.push("");
+  }
+
+  if (data.topics.length > 0) {
+    lines.push("## Topics");
+    for (const { topic, quotes } of data.topics) {
+      const categoryTag = topic.category ? ` _(${topic.category})_` : "";
+      lines.push(`### [id:${topic.id}] ${topic.name}${categoryTag}`);
+      lines.push(topic.description);
+      if (quotes.length > 0) {
+        lines.push("");
+        lines.push("**Related quotes:**");
+        for (const q of quotes) {
+          const attribution = q.channel ? `${q.speaker} in ${q.channel}` : q.speaker;
+          lines.push(`- [message_id:${q.message_id ?? "none"}] "${q.text}" — ${attribution}`);
+          const linkLine = formatQuoteLinks(q.data_item_ids);
+          if (linkLine) lines.push(linkLine);
+        }
+      }
+      lines.push("");
+    }
+  }
+
+  if (data.people.length > 0) {
+    lines.push("## People");
+    for (const { person, quotes } of data.people) {
+      lines.push(`### [id:${person.id}] ${person.name}`);
+      lines.push(`_${person.relationship}_`);
+      lines.push("");
+      lines.push(person.description);
+      if (quotes.length > 0) {
+        lines.push("");
+        lines.push("**Related quotes:**");
+        for (const q of quotes) {
+          const attribution = q.channel ? `${q.speaker} in ${q.channel}` : q.speaker;
+          lines.push(`- [message_id:${q.message_id ?? "none"}] "${q.text}" — ${attribution}`);
+          const linkLine = formatQuoteLinks(q.data_item_ids);
+          if (linkLine) lines.push(linkLine);
+        }
+      }
+      lines.push("");
+    }
+  }
+
+  if (data.standaloneQuotes.length > 0) {
+    lines.push("## Additional Quotes");
+    for (const q of data.standaloneQuotes) {
+      const attribution = q.channel ? `${q.speaker} in ${q.channel}` : q.speaker;
+      lines.push(`- [message_id:${q.message_id ?? "none"}] "${q.text}" — ${attribution}`);
+      const linkLine = formatQuoteLinks(q.data_item_ids);
+      if (linkLine) lines.push(linkLine);
+    }
+    lines.push("");
+  }
+
+  const user = lines.join("\n");
+
+  return { system, user };
+}

package/src/prompts/synthesis/types.ts

@@ -0,0 +1,26 @@
+import type { Fact, Topic, Person, Quote } from "../../core/types.js";
+
+export interface EnrichedTopic {
+  topic: Topic;
+  quotes: Quote[];
+}
+
+export interface EnrichedPerson {
+  person: Person;
+  quotes: Quote[];
+}
+
+export interface SynthesisPromptData {
+  subject: string;
+  facts: Fact[];
+  topics: EnrichedTopic[];
+  people: EnrichedPerson[];
+  standaloneQuotes: Quote[];
+  /**
+   * Map of entity ID → display name for all entities included in this payload.
+   * Used to annotate quote links: IDs present in a quote's data_item_ids but
+   * absent from this map are rendered as "(not loaded)" — a signal to the LLM
+   * that a related record exists and can be fetched via fetch_memory.
+   */
+  loadedEntityNames?: Map<string, string>;
+}

package/tui/src/commands/generate.tsx

@@ -0,0 +1,98 @@
+import type { Command } from "./registry";
+import { ConfirmOverlay } from "../components/ConfirmOverlay";
+import { GeneratedDocsOverlay } from "../components/GeneratedDocsOverlay";
+
+async function doGenerate(subject: string, ctx: Parameters<Command["execute"]>[1]): Promise<void> {
+  const { model, isRewriteModel } = ctx.ei.checkGenerationModel();
+
+  if (!isRewriteModel) {
+    const confirmed = await new Promise<boolean>((resolve) => {
+      const msg = [
+        `Generating with your default model (${model}).`,
+        "A high-capability model (Opus-class) is recommended.",
+        "Set one via /settings → rewrite_model, or continue anyway?",
+      ].join("\n");
+
+      ctx.showOverlay((hideOverlay) => (
+        <ConfirmOverlay
+          message={msg}
+          onConfirm={() => { hideOverlay(); resolve(true); }}
+          onCancel={() => { hideOverlay(); resolve(false); }}
+        />
+      ), ctx.renderer);
+    });
+
+    if (!confirmed) {
+      ctx.showNotification("Cancelled", "info");
+      return;
+    }
+  }
+
+  ctx.showNotification(`Generating knowledge document about: ${subject.slice(0, 60)}`, "info");
+
+  try {
+    await ctx.ei.generateDocument(subject);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    ctx.showNotification(`Generation failed: ${message}`, "error");
+  }
+}
+
+async function writeDoc(slug: string, ctx: Parameters<Command["execute"]>[1]): Promise<void> {
+  const outPath = await ctx.ei.writeGeneratedDocument(slug);
+  if (!outPath) {
+    ctx.showNotification(`No content found for document "${slug}"`, "error");
+    return;
+  }
+  ctx.showNotification(`Written to ${outPath}`, "info");
+}
+
+export const generateCommand: Command = {
+  name: "generate",
+  aliases: [],
+  description: "Generate a knowledge document | /generate <subject> to create, /generate to manage",
+  usage: "/generate [subject description]",
+
+  async execute(args, ctx) {
+    if (args.length === 0) {
+      const human = await ctx.ei.getHuman();
+      const docs = human.settings?.document?.processed_documents ?? {};
+      const generated = Object.entries(docs)
+        .filter(([, r]) => r.type === "generated" && r.subject)
+        .sort(([, a], [, b]) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
+        .map(([slug, r]) => ({ slug, subject: r.subject!, created_at: r.created_at }));
+
+      if (generated.length === 0) {
+        ctx.showNotification(
+          "No generated documents yet. Use /generate <subject description> to create one.",
+          "info"
+        );
+        return;
+      }
+
+      ctx.showOverlay((hideOverlay) => (
+        <GeneratedDocsOverlay
+          docs={generated}
+          onWrite={async (doc) => {
+            hideOverlay();
+            await writeDoc(doc.slug, ctx);
+          }}
+          onReRun={async (doc) => {
+            hideOverlay();
+            ctx.showNotification(`Re-running generation for: ${doc.subject.slice(0, 60)}`, "info");
+            try {
+              await ctx.ei.reRunDocument(doc.slug);
+            } catch (err) {
+              const message = err instanceof Error ? err.message : String(err);
+              ctx.showNotification(`Re-run failed: ${message}`, "error");
+            }
+          }}
+          onDismiss={hideOverlay}
+        />
+      ), ctx.renderer);
+      return;
+    }
+
+    await doGenerate(args.join(" "), ctx);
+  },
+};

package/tui/src/commands/unsource.tsx

@@ -19,15 +19,19 @@ export const unsourceCommand: Command = {
         return;
       }
 
-      const items = sources.map(f => ({
-        id: `import:document:${f}`,
-        display_name: `import:document:${f}`,
-        aliases: [] as string[],
-        is_paused: false,
-        is_archived: false,
-        unread_count: 0,
-        has_pending_update: false,
-      }));
+      const items = sources.map(f => {
+        const prefix = docs[f]?.type === "generated" ? "generate:document:" : "import:document:";
+        const tag = `${prefix}${f}`;
+        return {
+          id: tag,
+          display_name: tag,
+          aliases: [] as string[],
+          is_paused: false,
+          is_archived: false,
+          unread_count: 0,
+          has_pending_update: false,
+        };
+      });
 
       ctx.showOverlay((hideOverlay) => (
         <PersonaListOverlay
@@ -50,7 +54,10 @@ export const unsourceCommand: Command = {
     if (!rawArg.includes(":")) {
       const human = await ctx.ei.getHuman();
       const docs = human.settings?.document?.processed_documents ?? {};
-      const allSources = Object.keys(docs).map(f => `import:document:${f}`);
+      const allSources = Object.keys(docs).map(f => {
+        const prefix = docs[f]?.type === "generated" ? "generate:document:" : "import:document:";
+        return `${prefix}${f}`;
+      });
       const matches = allSources.filter(s => s.endsWith(rawArg) || s.includes(rawArg));
       if (matches.length === 1) {
         sourceTag = matches[0];

package/tui/src/components/GeneratedDocsOverlay.tsx

@@ -0,0 +1,136 @@
+import { useKeyboard } from "@opentui/solid";
+import { For, createSignal, createMemo, onMount, onCleanup } from "solid-js";
+import type { KeyEvent } from "@opentui/core";
+import { useKeyboardNav } from "../context/keyboard.js";
+
+export interface GeneratedDocItem {
+  slug: string;
+  subject: string;
+  created_at: string;
+}
+
+interface GeneratedDocsOverlayProps {
+  docs: GeneratedDocItem[];
+  onWrite: (doc: GeneratedDocItem) => void;
+  onReRun: (doc: GeneratedDocItem) => void;
+  onDismiss: () => void;
+}
+
+export function GeneratedDocsOverlay(props: GeneratedDocsOverlayProps) {
+  const { setOverlayActive } = useKeyboardNav();
+  onMount(() => setOverlayActive(true));
+  onCleanup(() => setOverlayActive(false));
+
+  const [selectedIndex, setSelectedIndex] = createSignal(0);
+
+  const clampedIndex = createMemo(() =>
+    Math.min(selectedIndex(), Math.max(0, props.docs.length - 1))
+  );
+
+  useKeyboard((event: KeyEvent) => {
+    const key = event.name;
+    const listLength = props.docs.length;
+
+    if (key === "j" || key === "down") {
+      event.preventDefault();
+      setSelectedIndex((prev) => Math.min(prev + 1, listLength - 1));
+      return;
+    }
+
+    if (key === "k" || key === "up") {
+      event.preventDefault();
+      setSelectedIndex((prev) => Math.max(prev - 1, 0));
+      return;
+    }
+
+    if (key === "return" || key === "w") {
+      event.preventDefault();
+      if (listLength > 0) props.onWrite(props.docs[clampedIndex()]);
+      return;
+    }
+
+    if (key === "r") {
+      event.preventDefault();
+      if (listLength > 0) props.onReRun(props.docs[clampedIndex()]);
+      return;
+    }
+
+    if (key === "escape") {
+      event.preventDefault();
+      props.onDismiss();
+      return;
+    }
+  });
+
+  const formatDate = (iso: string) => {
+    try {
+      return new Date(iso).toLocaleDateString(undefined, {
+        year: "numeric",
+        month: "short",
+        day: "numeric",
+      });
+    } catch {
+      return iso.slice(0, 10);
+    }
+  };
+
+  const truncate = (s: string, max: number) =>
+    s.length > max ? s.slice(0, max - 3) + "..." : s;
+
+  return (
+    <box
+      position="absolute"
+      width="100%"
+      height="100%"
+      left={0}
+      top={0}
+      backgroundColor="#000000"
+      alignItems="center"
+      justifyContent="center"
+    >
+      <box
+        width={72}
+        height="80%"
+        backgroundColor="#1a1a2e"
+        borderStyle="single"
+        borderColor="#586e75"
+        padding={2}
+        flexDirection="column"
+      >
+        <text fg="#eee8d5" marginBottom={1}>
+          Generated Documents
+        </text>
+
+        <scrollbox height="100%">
+          <For each={props.docs}>
+            {(doc, index) => {
+              const isSelected = () => clampedIndex() === index();
+              const label = () =>
+                `  ${truncate(doc.subject, 48)}    ${formatDate(doc.created_at)}`;
+
+              return (
+                <box
+                  backgroundColor={isSelected() ? "#2d3748" : "transparent"}
+                  paddingLeft={1}
+                  paddingRight={1}
+                >
+                  <text fg={isSelected() ? "#eee8d5" : "#839496"}>
+                    {label()}
+                  </text>
+                </box>
+              );
+            }}
+          </For>
+        </scrollbox>
+
+        <text> </text>
+        <text fg="#586e75">
+          j/k: navigate | Enter/w: write file | r: re-run | Esc: cancel
+        </text>
+        <text fg="#dc322f">
+          ⚠ Re-running replaces the existing file — write first to keep it
+        </text>
+      </box>
+    </box>
+  );
+}

package/tui/src/components/PromptInput.tsx

@@ -32,6 +32,7 @@ import { silenceCommand } from "../commands/silence.js";
 import { captureCommand } from "../commands/capture.js";
 import { importCommand } from "../commands/import.js";
 import { unsourceCommand } from "../commands/unsource.js";
+import { generateCommand } from "../commands/generate.js";
 import { openCYPEditor } from "../util/cyp-editor.js";
 import { useOverlay } from "../context/overlay";
 import { CommandSuggest } from "./CommandSuggest";
@@ -90,6 +91,7 @@ export function PromptInput() {
   registerCommand(captureCommand);
   registerCommand(importCommand);
   registerCommand(unsourceCommand);
+  registerCommand(generateCommand);
   registerCommand(authCommand);
   registerCommand(pauseCommand);
   registerCommand(resumeCommand);

package/tui/src/context/ei.tsx

@@ -13,7 +13,7 @@ import { createStore } from "solid-js/store";
 import { Processor } from "../../../src/core/processor.js";
 import { FileStorage } from "../storage/file.js";
 import { remoteSync } from "../../../src/storage/remote.js";
-import { logger, clearLog, interceptConsole } from "../util/logger.js";
+import { logger, rotateLog, interceptConsole } from "../util/logger.js";
 import { E2E_SKIP_LOCAL_DETECT, E2E_SKIP_CLOUD_DETECT } from "../util/e2e-flags.js";
 import {
   detectProviders,
@@ -155,6 +155,10 @@ export interface EiContextValue {
   importDocument: (filePath: string) => Promise<import('../../../src/integrations/document/types.js').DocumentImportResult>;
   getUnsourcePreview: (sourceTag: string) => import('../../../src/integrations/document/unsource.js').UnsourcePreview;
   executeUnsource: (preview: import('../../../src/integrations/document/unsource.js').UnsourcePreview) => Promise<import('../../../src/integrations/document/unsource.js').UnsourceResult>;
+  generateDocument: (subject: string) => Promise<{ slug: string }>;
+  reRunDocument: (slug: string) => Promise<{ slug: string }>;
+  writeGeneratedDocument: (slug: string) => Promise<string | null>;
+  checkGenerationModel: () => { model: string; isRewriteModel: boolean };
 }
 const EiContext = createContext<EiContextValue>();
 
@@ -358,6 +362,34 @@ export const EiProvider: ParentComponent = (props) => {
     return result;
   };
 
+  const generateDocument = async (subject: string): Promise<{ slug: string }> => {
+    if (!processor) throw new Error("Processor not ready");
+    return processor.generateDocument(subject);
+  };
+
+  const reRunDocument = async (slug: string): Promise<{ slug: string }> => {
+    if (!processor) throw new Error("Processor not ready");
+    return processor.reRunDocument(slug);
+  };
+
+  const writeGeneratedDocument = async (slug: string): Promise<string | null> => {
+    if (!processor) throw new Error("Processor not ready");
+    const content = await processor.getGeneratedDocumentContent(slug);
+    if (!content) return null;
+    const { join } = await import("node:path");
+    const { mkdirSync, writeFileSync } = await import("node:fs");
+    const dir = join(eiDataPath, "docs");
+    mkdirSync(dir, { recursive: true });
+    const outPath = join(dir, `${slug}.md`);
+    writeFileSync(outPath, content);
+    return outPath;
+  };
+
+  const checkGenerationModel = (): { model: string; isRewriteModel: boolean } => {
+    if (!processor) throw new Error("Processor not ready");
+    return processor.checkGenerationModel();
+  };
+
   const archivePersona = async (personaId: string) => {
     if (!processor) return;
     await processor.archivePersona(personaId);
@@ -835,7 +867,7 @@ export const EiProvider: ParentComponent = (props) => {
     await finishBootstrap();
   };
   async function bootstrap() {
-    clearLog();
+    rotateLog();
     interceptConsole();
     logger.info("Ei TUI bootstrap starting");
     try {
@@ -909,6 +941,17 @@ export const EiProvider: ParentComponent = (props) => {
         onRoomMessageProcessing: (roomId) => {
           if (roomId === store.activeRoomId) setStore("isRoomProcessing", true);
         },
+        onDocumentGenerated: async (slug) => {
+          const { join } = await import("node:path");
+          const { mkdirSync, writeFileSync } = await import("node:fs");
+          const content = await processor!.getGeneratedDocumentContent(slug);
+          if (content) {
+            const dir = join(eiDataPath, "docs");
+            mkdirSync(dir, { recursive: true });
+            writeFileSync(join(dir, `${slug}.md`), content);
+            showNotification(`Document ready: ${join(dir, `${slug}.md`)}`, "info");
+          }
+        },
       };
       processor = new Processor(eiInterface);
       logger.debug("Processor created, calling start()");
@@ -1026,6 +1069,10 @@ export const EiProvider: ParentComponent = (props) => {
     importDocument,
     getUnsourcePreview,
     executeUnsource,
+    generateDocument,
+    reRunDocument,
+    writeGeneratedDocument,
+    checkGenerationModel,
   };
   return (
     <Switch>

package/tui/src/util/logger.ts

@@ -1,8 +1,10 @@
 /** File-based logger for TUI debugging. Usage: tail -f $EI_DATA_PATH/tui.log */
 
-import { appendFileSync, mkdirSync } from "node:fs";
+import { appendFileSync, mkdirSync, existsSync, readdirSync, unlinkSync, renameSync } from "node:fs";
 import { join } from "node:path";
 
+const MAX_ROLLED_LOGS = 10;
+
 function getDataPath(): string {
   if (Bun.env.EI_DATA_PATH) return Bun.env.EI_DATA_PATH;
   const xdgData = Bun.env.XDG_DATA_HOME || join(Bun.env.HOME || "~", ".local", "share");
@@ -59,16 +61,34 @@ export const logger = {
   error: (message: string, data?: unknown) => writeLogSync("error", message, data),
 };
 
-export function clearLog(): void {
+export function rotateLog(): void {
   try {
     const logPath = getLogPath();
     const dataDir = logPath.substring(0, logPath.lastIndexOf("/"));
     mkdirSync(dataDir, { recursive: true });
+
+    if (existsSync(logPath)) {
+      const ts = new Date().toISOString().replace(/[:.]/g, "-").replace("T", "_").slice(0, 19);
+      renameSync(logPath, join(dataDir, `tui-${ts}.log`));
+    }
+
+    const rolled = readdirSync(dataDir)
+      .filter(f => /^tui-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log$/.test(f))
+      .sort();
+    for (const old of rolled.slice(0, Math.max(0, rolled.length - MAX_ROLLED_LOGS))) {
+      unlinkSync(join(dataDir, old));
+    }
+
     const header = `--- TUI Started at ${new Date().toISOString()} ---\n`;
     Bun.write(logPath, header);
   } catch {}
 }
 
+/** @deprecated Use rotateLog() instead */
+export function clearLog(): void {
+  rotateLog();
+}
+
 export function interceptConsole(): void {
   const originalLog = console.log.bind(console);
   const originalWarn = console.warn.bind(console);

package/tui/src/util/provider-detection.ts

@@ -158,7 +158,7 @@ export async function detectProviders(
   detected: ProviderDetectionResult[];
   statuses: ProviderDetectionStatus[];
 }> {
-  const env = options.env ?? (process.env as Record<string, string | undefined>);
+  const env = options.env ?? (Bun.env as Record<string, string | undefined>);
   const detected: ProviderDetectionResult[] = [];
   const statuses: ProviderDetectionStatus[] = [];
 
@@ -236,12 +236,15 @@ export function buildProviderAccounts(
     if (d.selected.bonusModel) pushIfNew(d.selected.bonusModel);
     for (const id of d.modelIds) pushIfNew(id);
 
+    const cloudConfig = CLOUD_PROVIDERS.find((p) => p.name === d.name);
+    const apiKey = cloudConfig ? `$${cloudConfig.envVar}` : d.apiKey;
+
     return {
       id: crypto.randomUUID(),
       name: d.name,
       type: "llm" as ProviderType,
       url: d.url,
-      api_key: d.apiKey,
+      api_key: apiKey,
       enabled: true,
       created_at: new Date().toISOString(),
       default_model: d.selected.chatModel,

package/tui/src/util/yaml-provider.ts

@@ -35,12 +35,6 @@ export interface ProviderYAMLResult {
   _delete: boolean;
 }
 
-function resolveEnvVar(value: string | undefined): string | undefined {
-  if (!value || !value.startsWith("$")) return value;
-  const varName = value.slice(1);
-  return process.env[varName] || value;
-}
-
 const PLACEHOLDER_PROVIDER_NAME = "My Provider";
 const PLACEHOLDER_PROVIDER_URL = "https://api.example.com/v1";
 const PLACEHOLDER_PROVIDER_API_KEY = "your-api-key-or-$ENVAR";
@@ -120,7 +114,7 @@ export function newProviderFromYAML(yamlContent: string): ProviderAccount {
     name: data.name,
     type: (data.type === "storage" ? "storage" : "llm") as ProviderType,
     url: data.url,
-    api_key: resolveEnvVar(data.api_key),
+    api_key: data.api_key,
     default_model: data.default_model,
     token_limit: data.token_limit ?? undefined,
     extra_headers: data.extra_headers && Object.keys(data.extra_headers).length > 0 ? data.extra_headers : undefined,
@@ -227,7 +221,7 @@ export function providerFromYAML(yamlContent: string, original: ProviderAccount)
     name: data.name,
     type: (data.type === "storage" ? "storage" : "llm") as ProviderType,
     url: data.url,
-    api_key: resolveEnvVar(data.api_key),
+    api_key: data.api_key,
     default_model: data.default_model,
     token_limit: data.token_limit ?? undefined,
     extra_headers: data.extra_headers && Object.keys(data.extra_headers).length > 0 ? data.extra_headers : undefined,