ei-tui 0.3.1 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.3.1",
+  "version": "0.3.6",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/handlers/rewrite.ts

@@ -36,7 +36,15 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
 
   const subjects = response.parsed as RewriteScanResult | undefined;
   if (!subjects || !Array.isArray(subjects) || subjects.length === 0) {
-    console.log(`[handleRewriteScan] No extra subjects found for ${itemType} "${itemId}" — item is cohesive`);
+    console.log(`[handleRewriteScan] No extra subjects found for ${itemType} "${itemId}" — marking rewrite_checked`);
+    const human = state.getHuman();
+    if (itemType === "topic") {
+      const topic = human.topics.find(t => t.id === itemId);
+      if (topic) state.human_topic_upsert({ ...topic, rewrite_checked: true });
+    } else if (itemType === "person") {
+      const person = human.people.find(p => p.id === itemId);
+      if (person) state.human_person_upsert({ ...person, rewrite_checked: true });
+    }
     return;
   }
 
@@ -246,5 +254,14 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
     newCount++;
   }
 
+  const updatedHuman = state.getHuman();
+  if (itemType === "topic") {
+    const original = updatedHuman.topics.find(t => t.id === itemId);
+    if (original) state.human_topic_upsert({ ...original, rewrite_checked: true });
+  } else if (itemType === "person") {
+    const original = updatedHuman.people.find(p => p.id === itemId);
+    if (original) state.human_person_upsert({ ...original, rewrite_checked: true });
+  }
+
   console.log(`[handleRewriteRewrite] Complete for ${itemType} "${itemId}": ${existingCount} existing updated, ${newCount} new created`);
 }

package/src/core/orchestrators/ceremony.ts

@@ -617,12 +617,12 @@ export function queueRewritePhase(state: StateManager): void {
   const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
 
   for (const topic of human.topics) {
-    if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+    if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !topic.rewrite_checked) {
       itemsToScan.push({ item: topic, type: "topic" });
     }
   }
   for (const person of human.people) {
-    if ((person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+    if ((person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !person.rewrite_checked) {
       itemsToScan.push({ item: person, type: "person" });
     }
   }

package/src/core/orchestrators/human-extraction.ts

@@ -313,19 +313,26 @@ export async function queueTopicMatch(
         }));
 
       console.log(`[queueTopicMatch] Embedding search: ${topicsWithEmbeddings.length} topics → ${topKItems.length} candidates`);
+      if (topKItems.length > 0) state.embedding_setWarning(false);
     } catch (err) {
-      console.error(`[queueTopicMatch] Embedding search failed, falling back to all topics:`, err);
+      console.error(`[queueTopicMatch] Embedding search failed, falling back to recent topics:`, err);
+      state.embedding_setWarning(true);
     }
   }
 
   if (topKItems.length === 0) {
-    console.log(`[queueTopicMatch] No embeddings available, using all topics`);
-    topKItems = human.topics.map(t => ({
+    const sorted = [...human.topics].sort((a, b) => {
+      const aDate = a.last_mentioned ?? a.last_updated;
+      const bDate = b.last_mentioned ?? b.last_updated;
+      return bDate.localeCompare(aDate);
+    });
+    topKItems = sorted.slice(0, EMBEDDING_TOP_K).map(t => ({
       id: t.id,
       name: t.name,
       description: t.description,
       category: t.category,
     }));
+    console.log(`[queueTopicMatch] No embedding matches, using ${topKItems.length} most-recent topics`);
   }
 
   const prompt = buildTopicMatchPrompt({
@@ -388,19 +395,26 @@ export async function queuePersonMatch(
         }));
 
       console.log(`[queuePersonMatch] Embedding search: ${peopleWithEmbeddings.length} people → ${topKItems.length} candidates`);
+      if (topKItems.length > 0) state.embedding_setWarning(false);
     } catch (err) {
-      console.error(`[queuePersonMatch] Embedding search failed, falling back to all people:`, err);
+      console.error(`[queuePersonMatch] Embedding search failed, falling back to recent people:`, err);
+      state.embedding_setWarning(true);
     }
   }
 
   if (topKItems.length === 0) {
-    console.log(`[queuePersonMatch] No embeddings available, using all people`);
-    topKItems = human.people.map(p => ({
+    const sorted = [...human.people].sort((a, b) => {
+      const aDate = a.last_mentioned ?? a.last_updated;
+      const bDate = b.last_mentioned ?? b.last_updated;
+      return bDate.localeCompare(aDate);
+    });
+    topKItems = sorted.slice(0, EMBEDDING_TOP_K).map(p => ({
       id: p.id,
       name: p.name,
       description: p.description,
       relationship: p.relationship,
     }));
+    console.log(`[queuePersonMatch] No embedding matches, using ${topKItems.length} most-recent people`);
   }
 
   const prompt = buildPersonMatchPrompt({

package/src/core/processor.ts

@@ -453,6 +453,30 @@ export class Processor {
       });
     }
 
+    // web_fetch tool
+    if (!this.stateManager.tools_getByName("web_fetch")) {
+      this.stateManager.tools_add({
+        id: crypto.randomUUID(),
+        provider_id: "ei",
+        name: "web_fetch",
+        display_name: "Web Fetch",
+        description:
+          "Fetch content from a URL and return the text. Useful for reading web pages, documentation, or public APIs. HTML is stripped to plain text. Only available in the TUI.",
+        input_schema: {
+          type: "object",
+          properties: {
+            url: { type: "string", description: "The URL to fetch (http or https only)" },
+          },
+          required: ["url"],
+        },
+        runtime: "node",
+        builtin: true,
+        enabled: true,
+        created_at: now,
+        max_calls_per_interaction: 3,
+      });
+    }
+
     // --- Tavily Search provider ---
     if (!this.stateManager.tools_getProviderById("tavily")) {
       const tavilyProvider: ToolProvider = {

package/src/core/queue-manager.ts

@@ -26,6 +26,7 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
       : "idle",
     pending_count: sm.queue_length(),
     dlq_count: sm.queue_dlqLength(),
+    embedding_warning: sm.embedding_getWarning() || undefined,
   };
 }
 

package/src/core/state-manager.ts

@@ -30,6 +30,7 @@ export class StateManager {
   private persistenceState = new PersistenceState();
   private providers: ToolProvider[] = [];
   private tools: ToolDefinition[] = [];
+  private embeddingWarning = false;
 
   async initialize(storage: Storage): Promise<void> {
     this.persistenceState.setStorage(storage);
@@ -479,6 +480,14 @@ export class StateManager {
     return this.queueState.dlqLength();
   }
 
+  embedding_setWarning(warned: boolean): void {
+    this.embeddingWarning = warned;
+  }
+
+  embedding_getWarning(): boolean {
+    return this.embeddingWarning;
+  }
+
   queue_getDLQItems(): LLMRequest[] {
     return this.queueState.getDLQItems();
   }

package/src/core/tools/builtin/web-fetch.ts

@@ -0,0 +1,73 @@
+import type { ToolExecutor } from "../types.js";
+
+const MAX_CHARS = 20000;
+
+export const webFetchExecutor: ToolExecutor = {
+  name: "web_fetch",
+
+  async execute(args: Record<string, unknown>): Promise<string> {
+    const url = typeof args.url === "string" ? args.url.trim() : "";
+    if (!url) {
+      return JSON.stringify({ error: "Missing required argument: url" });
+    }
+
+    let parsedUrl: URL;
+    try {
+      parsedUrl = new URL(url);
+    } catch {
+      return JSON.stringify({ error: `Invalid URL: ${url}` });
+    }
+
+    if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+      return JSON.stringify({ error: "Only http and https URLs are supported" });
+    }
+
+    try {
+      console.log(`[web_fetch] fetching ${url}`);
+      const response = await fetch(url, {
+        headers: {
+          "User-Agent": "Ei/1.0 (AI companion; +https://github.com/Flare576/ei)",
+          "Accept": "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8",
+        },
+      });
+
+      if (!response.ok) {
+        return JSON.stringify({ error: `HTTP ${response.status}: ${response.statusText}`, url });
+      }
+
+      const contentType = response.headers.get("content-type") ?? "";
+      let text = await response.text();
+
+      // Strip HTML noise for readability
+      if (contentType.includes("text/html")) {
+        text = text
+          .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+          .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+          .replace(/<[^>]+>/g, " ")
+          .replace(/&nbsp;/g, " ")
+          .replace(/&amp;/g, "&")
+          .replace(/&lt;/g, "<")
+          .replace(/&gt;/g, ">")
+          .replace(/&quot;/g, '"')
+          .replace(/\s+/g, " ")
+          .trim();
+      }
+
+      const truncated = text.length > MAX_CHARS;
+      if (truncated) text = text.slice(0, MAX_CHARS);
+
+      console.log(`[web_fetch] ${url} => ${text.length} chars${truncated ? " (truncated)" : ""}`);
+
+      return JSON.stringify({
+        url,
+        content_type: contentType,
+        content: text,
+        ...(truncated ? { truncated: true, note: `Content truncated to ${MAX_CHARS} characters` } : {}),
+      });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[web_fetch] failed for ${url}: ${msg}`);
+      return JSON.stringify({ error: `Fetch failed: ${msg}`, url });
+    }
+  },
+};

package/src/core/tools/index.ts

@@ -11,6 +11,7 @@ import type { ToolCall, ToolResult, ToolExecutor } from "./types.js";
 import { tavilyWebSearchExecutor, tavilyNewsSearchExecutor } from "./builtin/web-search.js";
 import { currentlyPlayingExecutor } from "./builtin/currently-playing.js";
 import { likedSongsExecutor } from "./builtin/spotify-liked-songs.js";
+import { webFetchExecutor } from "./builtin/web-fetch.js";
 // file-read and list-directory are Node-only — imported lazily via registerFileReadExecutor() to avoid
 // file-read and list-directory are Node-only — imported lazily via registerFileReadExecutor() to avoid
 
@@ -37,6 +38,7 @@ registerExecutor(tavilyWebSearchExecutor);
 registerExecutor(tavilyNewsSearchExecutor);
 registerExecutor(currentlyPlayingExecutor);
 registerExecutor(likedSongsExecutor);
+registerExecutor(webFetchExecutor);
 // file_read and list_directory are registered lazily via registerFileReadExecutor() — Node/TUI only.
 
 /**

package/src/core/types/data-items.ts

@@ -15,6 +15,7 @@ export interface DataItemBase {
   last_changed_by?: string;      // Persona ID that most recently updated this item (stable UUID)
   persona_groups?: string[];
   embedding?: number[];
+  rewrite_checked?: boolean;     // True after rewrite scan finds no changes. Cleared automatically when extraction upserts a fresh item.
 }
 
 export interface Fact extends DataItemBase {

package/src/core/types/integrations.ts

@@ -69,6 +69,8 @@ export interface QueueStatus {
   pending_count: number;
   dlq_count: number;
   current_operation?: string;
+  /** True when the embedding service failed and topic/person matching fell back to recent items. */
+  embedding_warning?: boolean;
 }
 
 export interface EiError {

package/src/prompts/response/index.ts

@@ -51,7 +51,12 @@ Your role is unique among personas:
   const priorities = buildPrioritiesSection(data.persona, data.human);
   const responseFormat = buildResponseFormatSection();
   const toolsSection = (data.tools && data.tools.length > 0) ? buildToolsSection() : "";
-  const currentTime = new Date().toISOString();
+  const currentTime = new Date().toLocaleString('en-US', {
+    weekday: 'long', year: 'numeric', month: 'long',
+    day: 'numeric', hour: 'numeric', minute: '2-digit',
+    timeZoneName: 'short',
+  });
+  const conversationState = getConversationStateText(data.delay_ms);
 
   return `${identity}
 
@@ -70,6 +75,7 @@ ${priorities}
 ${responseFormat}${toolsSection ? `\n\n${toolsSection}` : ""}
 
 Current time: ${currentTime}
+${conversationState}
 
 ## Final Instructions
 - NEVER repeat or echo the user's message in your response. Start directly with your own words.
@@ -94,7 +100,12 @@ function buildStandardSystemPrompt(data: ResponsePromptData): string {
   const priorities = buildPrioritiesSection(data.persona, data.human);
   const responseFormat = buildResponseFormatSection();
   const toolsSection = (data.tools && data.tools.length > 0) ? buildToolsSection() : "";
-  const currentTime = new Date().toISOString();
+  const currentTime = new Date().toLocaleString('en-US', {
+    weekday: 'long', year: 'numeric', month: 'long',
+    day: 'numeric', hour: 'numeric', minute: '2-digit',
+    timeZoneName: 'short',
+  });
+  const conversationState = getConversationStateText(data.delay_ms);
 
   return `${identity}
 
@@ -112,18 +123,15 @@ ${priorities}
 ${responseFormat}${toolsSection ? `\n\n${toolsSection}` : ""}
 
 Current time: ${currentTime}
+${conversationState}
 
 ## Final Instructions
 - NEVER repeat or echo the user's message in your response. Start directly with your own words.
 - Format your response as specified in the Response Format section above.`
 }
 
-function buildUserPrompt(data: ResponsePromptData): string {
-  const conversationState = getConversationStateText(data.delay_ms);
-  
-  return `${conversationState}
-
-${RESPONSE_FORMAT_INSTRUCTION}`;
+function buildUserPrompt(): string {
+  return RESPONSE_FORMAT_INSTRUCTION;
 }
 
 /**
@@ -151,7 +159,7 @@ export function buildResponsePrompt(data: ResponsePromptData): PromptOutput {
     ? buildEiSystemPrompt(data)
     : buildStandardSystemPrompt(data);
   
-  const user = buildUserPrompt(data);
+  const user = buildUserPrompt();
 
   return { system, user };
 }

package/tui/src/context/keyboard.tsx

@@ -33,7 +33,7 @@ export const KeyboardProvider: ParentComponent = (props) => {
   const [focusedPanel, setFocusedPanel] = createSignal<Panel>("input");
   const [sidebarVisible, setSidebarVisible] = createSignal(true);
   const renderer = useRenderer();
-  const { queueStatus, abortCurrentOperation, resumeQueue, personas, activePersonaId, selectPersona, saveAndExit, showNotification, messages, recallPendingMessages, cleanupTimers } = useEi();
+  const { queueStatus, abortCurrentOperation, resumeQueue, pauseQueue, personas, activePersonaId, selectPersona, saveAndExit, showNotification, messages, recallPendingMessages, cleanupTimers } = useEi();
   
   let messageScrollRef: ScrollBoxRenderable | null = null;
   let textareaRef: TextareaRenderable | null = null;
@@ -129,6 +129,9 @@ export const KeyboardProvider: ParentComponent = (props) => {
       } else if (status.state === "paused") {
         logger.info("Escape pressed - resuming queue");
         void resumeQueue();
+      } else {
+        logger.info("Escape pressed - pausing queue");
+        pauseQueue();
       }
       return;
     }