ei-tui 0.1.4 → 0.1.6

package/README.md

@@ -92,7 +92,11 @@ More information can be found in the [Web Readme](web/README.md)
 
 ### TUI
 
-```
+```bash
+# Install Bun (if you don't have it)
+curl -fsSL https://bun.sh/install | bash
+
+# Install Ei
 npm install -g ei-tui
 ```
 
@@ -126,7 +130,7 @@ This project is separated into five (5) logical parts:
 
 ## Requirements
 
-- [Bun](https://bun.sh) runtime (>=1.0.0)
+- [Bun](https://bun.sh) runtime (>=1.0.0) — install with `curl -fsSL https://bun.sh/install | bash`
 - A local LLM (LM Studio, Ollama, etc.) OR API access to a cloud provider (Anthropic, OpenAI, Bedrock, your uncle's LLM farm, etc.)
 
 ## LM Studio Setup

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "ei-tui",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "author": "Flare576",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/flare576/ei.git"
+    "url": "git+https://github.com/Flare576/ei.git"
   },
   "engines": {
     "bun": ">=1.0.0"
@@ -64,4 +64,4 @@
     "solid-js": "1.9.9",
     "yaml": "^2.8.2"
   }
-}
+}

package/src/cli/README.md

@@ -1,4 +1,6 @@
 # The CLI
+
+> For installation, see the [TUI README](../../tui/README.md#installation).
 ```sh
 ei                             # Start the TUI
 ei "query string"              # Return up to 10 results across all types

package/src/core/handlers/index.ts

@@ -23,6 +23,7 @@ import type {
   PersonaTopicMatchResult,
   PersonaTopicUpdateResult,
 } from "../../prompts/persona/types.js";
+import type { PersonaResponseResult } from "../../prompts/response/index.js";
 
 import type { 
   PersonaExpireResult, 
@@ -106,20 +107,66 @@ function handlePersonaResponse(response: LLMResponse, state: StateManager): void
   // the messages were "seen" and processed
   state.messages_markPendingAsRead(personaId);
 
+  // Structured JSON path: queue-processor parsed valid JSON into `parsed`
+  if (response.parsed !== undefined) {
+    const result = response.parsed as PersonaResponseResult;
+
+    if (!result.should_respond) {
+      const reason = result.reason;
+      if (reason) {
+        console.log(`[handlePersonaResponse] ${personaDisplayName} chose silence: ${reason}`);
+        const silentMessage: Message = {
+          id: crypto.randomUUID(),
+          role: "system",
+          silence_reason: reason,
+          timestamp: new Date().toISOString(),
+          read: false,
+          context_status: ContextStatus.Default,
+        };
+        state.messages_append(personaId, silentMessage);
+      } else {
+        console.log(`[handlePersonaResponse] ${personaDisplayName} chose not to respond (no reason given)`);
+      }
+      return;
+    }
+
+    // Build message with structured fields
+    const verbal = result.verbal_response || undefined;
+    const action = result.action_response || undefined;
+
+    if (!verbal && !action) {
+      console.log(`[handlePersonaResponse] ${personaDisplayName} JSON had should_respond=true but no content fields`);
+      return;
+    }
+
+    const message: Message = {
+      id: crypto.randomUUID(),
+      role: "system",
+      verbal_response: verbal,
+      action_response: action,
+      timestamp: new Date().toISOString(),
+      read: false,
+      context_status: ContextStatus.Default,
+    };
+    state.messages_append(personaId, message);
+    console.log(`[handlePersonaResponse] Appended structured response to ${personaDisplayName}`);
+    return;
+  }
+
+  // Legacy plain-text fallback
   if (!response.content) {
-    console.log(`[handlePersonaResponse] No content in response (${personaDisplayName} chose not to respond)`);
+    console.log(`[handlePersonaResponse] ${personaDisplayName} chose not to respond (no reason given)`);
     return;
   }
 
   const message: Message = {
     id: crypto.randomUUID(),
     role: "system",
-    content: response.content,
+    verbal_response: response.content ?? undefined,
     timestamp: new Date().toISOString(),
     read: false,
     context_status: ContextStatus.Default,
   };
-
   state.messages_append(personaId, message);
   console.log(`[handlePersonaResponse] Appended response to ${personaDisplayName}`);
 }
@@ -150,7 +197,7 @@ function handleHeartbeatCheck(response: LLMResponse, state: StateManager): void
     const message: Message = {
       id: crypto.randomUUID(),
       role: "system",
-      content: result.message,
+      verbal_response: result.message,
       timestamp: now,
       read: false,
       context_status: ContextStatus.Default,
@@ -179,10 +226,10 @@ function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
     return;
   }
 
-  const sendMessage = (content: string) => state.messages_append("ei", {
+  const sendMessage = (verbal_response: string) => state.messages_append("ei", {
     id: crypto.randomUUID(),
     role: "system",
-    content,
+    verbal_response,
     timestamp: now,
     read: false,
     context_status: ContextStatus.Default,
@@ -731,6 +778,18 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
   console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
 }
 
+/**
+ * Returns the combined display text of a message for quote indexing.
+ * Mirrors the rendering logic used in the frontends.
+ */
+function getMessageText(message: Message): string {
+  const parts: string[] = [];
+  if (message.action_response) parts.push(`_${message.action_response}_`);
+  if (message.verbal_response) parts.push(message.verbal_response);
+  return parts.join('\n\n');
+}
+
+
 async function validateAndStoreQuotes(
   candidates: Array<{ text: string; reason: string }> | undefined,
   messages: Message[],
@@ -744,22 +803,53 @@ async function validateAndStoreQuotes(
   for (const candidate of candidates) {
     let found = false;
     for (const message of messages) {
-      const start = message.content.indexOf(candidate.text);
+      const msgText = getMessageText(message);
+      const start = msgText.indexOf(candidate.text);
       if (start !== -1) {
         const end = start + candidate.text.length;
         
+        // Check for ANY overlapping quote in this message (not just exact match)
         const existing = state.human_quote_getForMessage(message.id);
-        const existingQuote = existing.find(q => q.start === start && q.end === end);
+        const overlapping = existing.find(q =>
+          q.start !== null && q.end !== null &&
+          start < q.end && end > q.start  // ranges overlap
+        );
         
-        if (existingQuote) {
-          if (!existingQuote.data_item_ids.includes(dataItemId)) {
-            state.human_quote_update(existingQuote.id, {
-              data_item_ids: [...existingQuote.data_item_ids, dataItemId],
-            });
-            console.log(`[extraction] Linked existing quote to "${dataItemId}": "${candidate.text.slice(0, 30)}..."`);
-          } else {
-            console.log(`[extraction] Quote already linked to "${dataItemId}": "${candidate.text.slice(0, 30)}..."`);
+        if (overlapping) {
+          // Merge: expand to the union of both ranges
+          const mergedStart = Math.min(start, overlapping.start!);
+          const mergedEnd = Math.max(end, overlapping.end!);
+          const mergedText = msgText.slice(mergedStart, mergedEnd);
+          
+          // Merge data_item_ids and persona_groups (deduplicated)
+          const mergedDataItemIds = overlapping.data_item_ids.includes(dataItemId)
+            ? overlapping.data_item_ids
+            : [...overlapping.data_item_ids, dataItemId];
+          const group = personaGroup || "General";
+          const mergedGroups = overlapping.persona_groups.includes(group)
+            ? overlapping.persona_groups
+            : [...overlapping.persona_groups, group];
+          
+          // Only recompute embedding if the text actually changed
+          let embedding = overlapping.embedding;
+          if (mergedText !== overlapping.text) {
+            try {
+              const embeddingService = getEmbeddingService();
+              embedding = await embeddingService.embed(mergedText);
+            } catch (err) {
+              console.warn(`[extraction] Failed to recompute embedding for merged quote: "${mergedText.slice(0, 30)}..."`, err);
+            }
           }
+          
+          state.human_quote_update(overlapping.id, {
+            start: mergedStart,
+            end: mergedEnd,
+            text: mergedText,
+            data_item_ids: mergedDataItemIds,
+            persona_groups: mergedGroups,
+            embedding,
+          });
+          console.log(`[extraction] Merged overlapping quote: "${mergedText.slice(0, 50)}..." (${mergedStart}-${mergedEnd})`);
           found = true;
           break;
         }

package/src/core/orchestrators/ceremony.ts

@@ -1,4 +1,4 @@
-import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic } from "../types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import { applyDecayToValue } from "../utils/index.js";
 import {
@@ -384,12 +384,12 @@ export function queueExplorePhase(personaId: string, state: StateManager): void
   });
 }
 
-function extractConversationThemes(messages: { content: string; role: string }[]): string[] {
+function extractConversationThemes(messages: Message[]): string[] {
   const humanMessages = messages.filter(m => m.role === "human");
   if (humanMessages.length === 0) return [];
   
   const words = humanMessages
-    .map(m => m.content.toLowerCase())
+    .map(m => (m.verbal_response ?? '').toLowerCase())
     .join(" ")
     .split(/\s+/)
     .filter(w => w.length > 4);

package/src/core/orchestrators/extraction-chunker.ts

@@ -12,7 +12,7 @@ function estimateTokens(text: string): number {
 }
 
 function estimateMessageTokens(messages: Message[]): number {
-  return messages.reduce((sum, msg) => sum + estimateTokens(msg.content) + 4, 0);
+  return messages.reduce((sum, msg) => sum + estimateTokens(msg.verbal_response ?? '') + 4, 0);
 }
 
 function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
@@ -20,7 +20,7 @@ function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
   let tokens = 0;
 
   for (let i = messages.length - 1; i >= 0; i--) {
-    const msgTokens = estimateTokens(messages[i].content) + 4;
+    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
     if (tokens + msgTokens > maxTokens) break;
     result.unshift(messages[i]);
     tokens += msgTokens;
@@ -39,7 +39,7 @@ function pullMessagesFromStart(
   let i = startIndex;
 
   while (i < messages.length) {
-    const msgTokens = estimateTokens(messages[i].content) + 4;
+    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
     if (tokens + msgTokens > maxTokens && pulled.length > 0) break;
     pulled.push(messages[i]);
     tokens += msgTokens;

package/src/core/processor.ts

@@ -56,6 +56,7 @@ import {
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
 import { getEmbeddingService, findTopK, needsEmbeddingUpdate, needsQuoteEmbeddingUpdate, computeDataItemEmbedding, computeQuoteEmbedding } from "./embedding-service.js";
 import { ContextStatus as ContextStatusEnum } from "./types.js";
+import { buildChatMessageContent } from "../prompts/message-utils.js";
 
 // =============================================================================
 // EMBEDDING STRIPPING - Remove embeddings from data items before returning to FE
@@ -124,6 +125,7 @@ export class Processor {
   private currentRequest: LLMRequest | null = null;
   private isTUI = false;
   private lastOpenCodeSync = 0;
+  private lastDLQTrim = 0;
   private openCodeImportInProgress = false;
   private pendingConflict: StateConflictData | null = null;
 
@@ -232,7 +234,7 @@ export class Processor {
     const welcomeMessage: Message = {
       id: crypto.randomUUID(),
       role: "system",
-      content: EI_WELCOME_MESSAGE,
+      verbal_response: EI_WELCOME_MESSAGE,
       timestamp: new Date().toISOString(),
       read: false,
       context_status: ContextStatusEnum.Always,
@@ -334,31 +336,36 @@ export class Processor {
       await this.checkScheduledTasks();
 
       if (this.queueProcessor.getState() === "idle") {
-        const request = this.stateManager.queue_peekHighest();
-        if (request) {
-          const personaId = request.data.personaId as string | undefined;
-          const personaDisplayName = request.data.personaDisplayName as string | undefined;
-          const personaSuffix = personaDisplayName ? ` [${personaDisplayName}]` : "";
-          console.log(`[Processor ${this.instanceId}] processing request: ${request.next_step}${personaSuffix}`);
-          this.currentRequest = request;
-
-          if (personaId && request.next_step === LLMNextStep.HandlePersonaResponse) {
-            this.interface.onMessageProcessing?.(personaId);
+        const retryAfter = this.stateManager.queue_nextItemRetryAfter();
+        const isBackingOff = retryAfter !== null && retryAfter > new Date().toISOString();
+
+        if (!isBackingOff) {
+          const request = this.stateManager.queue_claimHighest();
+          if (request) {
+            const personaId = request.data.personaId as string | undefined;
+            const personaDisplayName = request.data.personaDisplayName as string | undefined;
+            const personaSuffix = personaDisplayName ? ` [${personaDisplayName}]` : "";
+            console.log(`[Processor ${this.instanceId}] processing request: ${request.next_step}${personaSuffix}`);
+            this.currentRequest = request;
+
+            if (personaId && request.next_step === LLMNextStep.HandlePersonaResponse) {
+              this.interface.onMessageProcessing?.(personaId);
+            }
+
+            this.queueProcessor.start(request, async (response) => {
+              this.currentRequest = null;
+              await this.handleResponse(response);
+              const nextState = this.stateManager.queue_isPaused() ? "paused" : "idle";
+              // the processor state is set in the caller, so this needs a bit of delay
+              setTimeout(() => this.interface.onQueueStateChanged?.(nextState), 0);
+            }, {
+              accounts: this.stateManager.getHuman().settings?.accounts,
+              messageFetcher: (pName) => this.fetchMessagesForLLM(pName),
+              rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
+            });
+
+            this.interface.onQueueStateChanged?.("busy");
           }
-
-          this.queueProcessor.start(request, async (response) => {
-            this.currentRequest = null;
-            await this.handleResponse(response);
-            const nextState = this.stateManager.queue_isPaused() ? "paused" : "idle";
-            // the processor state is set in the caller, so this needs a bit of delay
-            setTimeout(() => this.interface.onQueueStateChanged?.(nextState), 0);
-          }, {
-            accounts: this.stateManager.getHuman().settings?.accounts,
-            messageFetcher: (pName) => this.fetchMessagesForLLM(pName),
-            rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
-          });
-
-          this.interface.onQueueStateChanged?.("busy");
         }
       }
 
@@ -407,6 +414,15 @@ export class Processor {
         }
       }
     }
+    // DLQ rolloff — once per day
+    const MS_PER_DAY = 86_400_000;
+    if (now - this.lastDLQTrim >= MS_PER_DAY) {
+      this.lastDLQTrim = now;
+      const trimmed = this.stateManager.queue_trimDLQ();
+      if (trimmed > 0) {
+        console.log(`[Processor] DLQ trim: removed ${trimmed} expired items`);
+      }
+    }
   }
 
   private async checkAndSyncOpenCode(human: HumanEntity, now: number): Promise<void> {
@@ -484,10 +500,17 @@ export class Processor {
       contextWindowHours
     );
     
-    return filteredHistory.map((m) => ({
-      role: m.role === "human" ? "user" : "assistant",
-      content: m.content,
-    })) as import("./types.js").ChatMessage[];
+    return filteredHistory
+      .reduce<import("./types.js").ChatMessage[]>((acc, m) => {
+        const content = buildChatMessageContent(m);
+        if (content.length > 0) {
+          acc.push({
+            role: m.role === "human" ? "user" : "assistant",
+            content,
+          });
+        }
+        return acc;
+      }, []);
   }
 
   private async queueHeartbeatCheck(personaId: string): Promise<void> {
@@ -542,7 +565,7 @@ export class Processor {
     const items: EiHeartbeatItem[] = [];
 
     const unverifiedFacts = human.facts
-      .filter(f => f.validated === ValidationLevel.None && f.learned_by !== "ei")
+      .filter(f => f.validated === ValidationLevel.None && f.learned_by !== "Ei")
       .slice(0, 5);
     for (const fact of unverifiedFacts) {
       const quote = human.quotes.find(q => q.data_item_ids.includes(fact.id));
@@ -932,7 +955,7 @@ export class Processor {
       .map(m => m.id);
     if (pendingIds.length === 0) return "";
     const removed = this.stateManager.messages_remove(personaId, pendingIds);
-    const recalledContent = removed.map(m => m.content).join("\n\n");
+    const recalledContent = removed.map(m => m.verbal_response ?? '').join("\n\n");
     this.interface.onMessageAdded?.(personaId);
     this.interface.onMessageRecalled?.(personaId, recalledContent);
     return recalledContent;
@@ -953,7 +976,7 @@ export class Processor {
     const message: Message = {
       id: crypto.randomUUID(),
       role: "human",
-      content,
+      verbal_response: content,
       timestamp: new Date().toISOString(),
       read: false,
       context_status: "default" as ContextStatus,
@@ -1469,13 +1492,31 @@ export class Processor {
     return {
       state: this.stateManager.queue_isPaused()
         ? "paused"
-        : this.queueProcessor.getState() === "busy"
+        : this.stateManager.queue_hasProcessingItem()
           ? "busy"
           : "idle",
       pending_count: this.stateManager.queue_length(),
+      dlq_count: this.stateManager.queue_dlqLength(),
     };
   }
 
+  pauseQueue(): void {
+    this.stateManager.queue_pause();
+    this.queueProcessor.abort();
+  }
+
+  getQueueActiveItems(): LLMRequest[] {
+    return this.stateManager.queue_getAllActiveItems();
+  }
+
+  getDLQItems(): LLMRequest[] {
+    return this.stateManager.queue_getDLQItems();
+  }
+
+  updateQueueItem(id: string, updates: Partial<LLMRequest>): boolean {
+    return this.stateManager.queue_updateItem(id, updates);
+  }
+
   async clearQueue(): Promise<number> {
     this.queueProcessor.abort();
     return this.stateManager.queue_clear();

package/src/core/queue-processor.ts

@@ -1,5 +1,5 @@
-import type { LLMRequest, LLMResponse, LLMRequestType, ProviderAccount, ChatMessage, Message } from "./types.js";
-import { callLLMRaw, parseJSONResponse, cleanResponseContent } from "./llm-client.js";
+import { LLMRequest, LLMResponse, LLMRequestType, ProviderAccount, ChatMessage, Message } from "./types.js";
+import { callLLMRaw, parseJSONResponse } from "./llm-client.js";
 import { hydratePromptPlaceholders } from "../prompts/message-utils.js";
 
 type QueueProcessorState = "idle" | "busy";
@@ -133,9 +133,8 @@ export class QueueProcessor {
   ): LLMResponse {
     switch (request.type) {
       case "json" as LLMRequestType:
-        return this.handleJSONResponse(request, content, finishReason);
       case "response" as LLMRequestType:
-        return this.handleConversationResponse(request, content, finishReason);
+        return this.handleJSONResponse(request, content, finishReason);
       case "raw" as LLMRequestType:
       default:
         return {
@@ -172,26 +171,4 @@ export class QueueProcessor {
     }
   }
 
-  private handleConversationResponse(
-    request: LLMRequest,
-    content: string,
-    finishReason: string | null
-  ): LLMResponse {
-    const cleaned = cleanResponseContent(content);
-    
-    const noMessagePatterns = [
-      /^no\s*(new\s*)?(message|response)/i,
-      /^nothing\s+to\s+(say|add)/i,
-      /^\[no\s+message\]/i,
-    ];
-    
-    const isNoMessage = noMessagePatterns.some((p) => p.test(cleaned));
-    
-    return {
-      request,
-      success: true,
-      content: isNoMessage ? null : cleaned,
-      finish_reason: finishReason ?? undefined,
-    };
   }
-}

package/src/core/state/personas.ts

@@ -1,5 +1,17 @@
 import type { PersonaEntity, Message, ContextStatus } from "../types.js";
 
+/**
+ * Migration: If a persisted message has the old `content` field but no `verbal_response`,
+ * move content → verbal_response. Runs on every load (no-op for already-migrated data).
+ */
+function migrateMessage(msg: Message & { content?: string }): Message {
+  if (msg.content !== undefined && msg.verbal_response === undefined) {
+    const { content, ...rest } = msg;
+    return { ...rest, verbal_response: content };
+  }
+  return msg;
+}
+
 export interface PersonaData {
   entity: PersonaEntity;
   messages: Message[];
@@ -13,7 +25,7 @@ export class PersonaState {
     this.personas = new Map(
       Object.entries(personas).map(([id, data]) => [
         id,
-        { entity: data.entity, messages: data.messages },
+        { entity: data.entity, messages: data.messages.map(migrateMessage) },
       ])
     );
   }