ei-tui 0.1.3 → 0.1.5

package/src/core/processor.ts

@@ -2,6 +2,7 @@ import {
   LLMRequestType,
   LLMPriority,
   LLMNextStep,
+  ValidationLevel,
   RESERVED_PERSONA_NAMES,
   isReservedPersonaName,
   type LLMRequest,
@@ -35,9 +36,12 @@ import {
   buildResponsePrompt,
   buildPersonaTraitExtractionPrompt,
   buildHeartbeatCheckPrompt,
+  buildEiHeartbeatPrompt,
   type ResponsePromptData,
   type PersonaTraitExtractionPromptData,
   type HeartbeatCheckPromptData,
+  type EiHeartbeatPromptData,
+  type EiHeartbeatItem,
 } from "../prompts/index.js";
 import { 
   orchestratePersonaGeneration,
@@ -52,6 +56,7 @@ import {
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
 import { getEmbeddingService, findTopK, needsEmbeddingUpdate, needsQuoteEmbeddingUpdate, computeDataItemEmbedding, computeQuoteEmbedding } from "./embedding-service.js";
 import { ContextStatus as ContextStatusEnum } from "./types.js";
+import { buildChatMessageContent } from "../prompts/message-utils.js";
 
 // =============================================================================
 // EMBEDDING STRIPPING - Remove embeddings from data items before returning to FE
@@ -120,6 +125,7 @@ export class Processor {
   private currentRequest: LLMRequest | null = null;
   private isTUI = false;
   private lastOpenCodeSync = 0;
+  private lastDLQTrim = 0;
   private openCodeImportInProgress = false;
   private pendingConflict: StateConflictData | null = null;
 
@@ -228,7 +234,7 @@ export class Processor {
     const welcomeMessage: Message = {
       id: crypto.randomUUID(),
       role: "system",
-      content: EI_WELCOME_MESSAGE,
+      verbal_response: EI_WELCOME_MESSAGE,
       timestamp: new Date().toISOString(),
       read: false,
       context_status: ContextStatusEnum.Always,
@@ -330,31 +336,36 @@ export class Processor {
       await this.checkScheduledTasks();
 
       if (this.queueProcessor.getState() === "idle") {
-        const request = this.stateManager.queue_peekHighest();
-        if (request) {
-          const personaId = request.data.personaId as string | undefined;
-          const personaDisplayName = request.data.personaDisplayName as string | undefined;
-          const personaSuffix = personaDisplayName ? ` [${personaDisplayName}]` : "";
-          console.log(`[Processor ${this.instanceId}] processing request: ${request.next_step}${personaSuffix}`);
-          this.currentRequest = request;
-
-          if (personaId && request.next_step === LLMNextStep.HandlePersonaResponse) {
-            this.interface.onMessageProcessing?.(personaId);
+        const retryAfter = this.stateManager.queue_nextItemRetryAfter();
+        const isBackingOff = retryAfter !== null && retryAfter > new Date().toISOString();
+
+        if (!isBackingOff) {
+          const request = this.stateManager.queue_claimHighest();
+          if (request) {
+            const personaId = request.data.personaId as string | undefined;
+            const personaDisplayName = request.data.personaDisplayName as string | undefined;
+            const personaSuffix = personaDisplayName ? ` [${personaDisplayName}]` : "";
+            console.log(`[Processor ${this.instanceId}] processing request: ${request.next_step}${personaSuffix}`);
+            this.currentRequest = request;
+
+            if (personaId && request.next_step === LLMNextStep.HandlePersonaResponse) {
+              this.interface.onMessageProcessing?.(personaId);
+            }
+
+            this.queueProcessor.start(request, async (response) => {
+              this.currentRequest = null;
+              await this.handleResponse(response);
+              const nextState = this.stateManager.queue_isPaused() ? "paused" : "idle";
+              // the processor state is set in the caller, so this needs a bit of delay
+              setTimeout(() => this.interface.onQueueStateChanged?.(nextState), 0);
+            }, {
+              accounts: this.stateManager.getHuman().settings?.accounts,
+              messageFetcher: (pName) => this.fetchMessagesForLLM(pName),
+              rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
+            });
+
+            this.interface.onQueueStateChanged?.("busy");
           }
-
-          this.queueProcessor.start(request, async (response) => {
-            this.currentRequest = null;
-            await this.handleResponse(response);
-            const nextState = this.stateManager.queue_isPaused() ? "paused" : "idle";
-            // the processor state is set in the caller, so this needs a bit of delay
-            setTimeout(() => this.interface.onQueueStateChanged?.(nextState), 0);
-          }, {
-            accounts: this.stateManager.getHuman().settings?.accounts,
-            messageFetcher: (pName) => this.fetchMessagesForLLM(pName),
-            rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
-          });
-
-          this.interface.onQueueStateChanged?.("busy");
         }
       }
 
@@ -369,7 +380,7 @@ export class Processor {
 
     const human = this.stateManager.getHuman();
     
-    if (this.isTUI && human.settings?.opencode?.integration) {
+    if (this.isTUI && human.settings?.opencode?.integration && this.stateManager.queue_length() === 0) {
       await this.checkAndSyncOpenCode(human, now);
     }
     
@@ -403,6 +414,15 @@ export class Processor {
         }
       }
     }
+    // DLQ rolloff — once per day
+    const MS_PER_DAY = 86_400_000;
+    if (now - this.lastDLQTrim >= MS_PER_DAY) {
+      this.lastDLQTrim = now;
+      const trimmed = this.stateManager.queue_trimDLQ();
+      if (trimmed > 0) {
+        console.log(`[Processor] DLQ trim: removed ${trimmed} expired items`);
+      }
+    }
   }
 
   private async checkAndSyncOpenCode(human: HumanEntity, now: number): Promise<void> {
@@ -434,12 +454,10 @@ export class Processor {
       },
     });
 
-    const since = lastSync > 0 ? new Date(lastSync) : new Date(0);
-
     this.openCodeImportInProgress = true;
     import("../integrations/opencode/importer.js")
-      .then(({ importOpenCodeSessions }) => 
-        importOpenCodeSessions(since, {
+      .then(({ importOpenCodeSessions }) =>
+        importOpenCodeSessions({
           stateManager: this.stateManager,
           interface: this.interface,
         })
@@ -449,7 +467,7 @@ export class Processor {
           console.log(
             `[Processor] OpenCode sync complete: ${result.sessionsProcessed} sessions, ` +
             `${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
-            `${result.topicUpdatesQueued} topic updates queued`
+            `${result.extractionScansQueued} extraction scans queued`
           );
         }
       })
@@ -482,29 +500,36 @@ export class Processor {
       contextWindowHours
     );
     
-    return filteredHistory.map((m) => ({
-      role: m.role === "human" ? "user" : "assistant",
-      content: m.content,
-    })) as import("./types.js").ChatMessage[];
+    return filteredHistory
+      .reduce<import("./types.js").ChatMessage[]>((acc, m) => {
+        const content = buildChatMessageContent(m);
+        if (content.length > 0) {
+          acc.push({
+            role: m.role === "human" ? "user" : "assistant",
+            content,
+          });
+        }
+        return acc;
+      }, []);
   }
 
   private async queueHeartbeatCheck(personaId: string): Promise<void> {
     const persona = this.stateManager.persona_getById(personaId);
     if (!persona) return;
-
     this.stateManager.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
-
     const human = this.stateManager.getHuman();
     const history = this.stateManager.messages_get(personaId);
-    const filteredHuman = await this.filterHumanDataByVisibility(human, persona);
+    if (personaId === "ei") {
+      await this.queueEiHeartbeat(human, history);
+      return;
+    }
 
+    const filteredHuman = await this.filterHumanDataByVisibility(human, persona);
     const inactiveDays = persona.last_activity
       ? Math.floor((Date.now() - new Date(persona.last_activity).getTime()) / (1000 * 60 * 60 * 24))
       : 0;
-
     const sortByEngagementGap = <T extends { exposure_desired: number; exposure_current: number }>(items: T[]): T[] =>
       [...items].sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current));
-
     const promptData: HeartbeatCheckPromptData = {
       persona: {
         name: persona.display_name,
@@ -532,6 +557,118 @@ export class Processor {
     });
   }
 
+  private async queueEiHeartbeat(human: HumanEntity, history: import("./types.js").Message[]): Promise<void> {
+    const now = Date.now();
+    const engagementGapThreshold = 0.2;
+    const cooldownMs = 7 * 24 * 60 * 60 * 1000;
+    const personas = this.stateManager.persona_getAll();
+    const items: EiHeartbeatItem[] = [];
+
+    const unverifiedFacts = human.facts
+      .filter(f => f.validated === ValidationLevel.None && f.learned_by !== "Ei")
+      .slice(0, 5);
+    for (const fact of unverifiedFacts) {
+      const quote = human.quotes.find(q => q.data_item_ids.includes(fact.id));
+      items.push({
+        id: fact.id,
+        type: "Fact Check",
+        name: fact.name,
+        description: fact.description,
+        quote: quote?.text,
+      });
+    }
+
+    const underEngagedPeople = human.people
+      .filter(p =>
+        (p.exposure_desired - p.exposure_current) > engagementGapThreshold &&
+        (!p.last_ei_asked || now - new Date(p.last_ei_asked).getTime() > cooldownMs)
+      )
+      .sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current))
+      .slice(0, 5);
+    for (const person of underEngagedPeople) {
+      const gap = Math.round((person.exposure_desired - person.exposure_current) * 100);
+      const quote = human.quotes.find(q => q.data_item_ids.includes(person.id));
+      items.push({
+        id: person.id,
+        type: "Low-Engagement Person",
+        engagement_delta: `${gap}%`,
+        relationship: person.relationship,
+        name: person.name,
+        description: person.description,
+        quote: quote?.text,
+      });
+    }
+
+    const underEngagedTopics = human.topics
+      .filter(t =>
+        (t.exposure_desired - t.exposure_current) > engagementGapThreshold &&
+        (!t.last_ei_asked || now - new Date(t.last_ei_asked).getTime() > cooldownMs)
+      )
+      .sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current))
+      .slice(0, 5);
+    for (const topic of underEngagedTopics) {
+      const gap = Math.round((topic.exposure_desired - topic.exposure_current) * 100);
+      const quote = human.quotes.find(q => q.data_item_ids.includes(topic.id));
+      items.push({
+        id: topic.id,
+        type: "Low-Engagement Topic",
+        engagement_delta: `${gap}%`,
+        name: topic.name,
+        description: topic.description,
+        quote: quote?.text,
+      });
+    }
+
+    const activePersonas = personas
+      .filter(p => !p.is_archived && !p.is_paused && p.id !== "ei")
+      .map(p => {
+        const msgs = this.stateManager.messages_get(p.id);
+        const lastHuman = [...msgs].reverse().find(m => m.role === "human");
+        const lastTs = lastHuman?.timestamp ? new Date(lastHuman.timestamp).getTime() : 0;
+        return { persona: p, lastHumanTs: lastTs };
+      })
+      .filter(({ lastHumanTs }) => {
+        const daysSince = (now - lastHumanTs) / (1000 * 60 * 60 * 24);
+        return daysSince >= 3;
+      })
+      .sort((a, b) => a.lastHumanTs - b.lastHumanTs)
+      .slice(0, 3);
+    for (const { persona: p, lastHumanTs } of activePersonas) {
+      const daysSince = lastHumanTs > 0
+        ? Math.floor((now - lastHumanTs) / (1000 * 60 * 60 * 24))
+        : 999;
+      items.push({
+        id: p.id,
+        type: "Inactive Persona",
+        name: p.display_name,
+        short_description: p.short_description,
+        days_inactive: daysSince,
+      });
+    }
+
+    if (items.length === 0) {
+      console.log("[queueEiHeartbeat] No items to address, skipping");
+      return;
+    }
+
+    const promptData: EiHeartbeatPromptData = {
+      items,
+      recent_history: history.slice(-10),
+    };
+
+    const prompt = buildEiHeartbeatPrompt(promptData);
+
+    this.stateManager.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleEiHeartbeat,
+      model: this.getModelForPersona("ei"),
+      data: { personaId: "ei", isTUI: this.isTUI },
+    });
+  }
+
 
   private classifyLLMError(error: string): string {
     const match = error.match(/\((\d{3})\)/);
@@ -627,9 +764,7 @@ export class Processor {
         }
       }
 
-      if (response.request.next_step === LLMNextStep.HandleEiValidation) {
-        this.interface.onHumanUpdated?.();
-      }
+
 
       if (response.request.next_step === LLMNextStep.HandleHumanItemUpdate) {
         this.interface.onHumanUpdated?.();
@@ -813,23 +948,16 @@ export class Processor {
   async recallPendingMessages(personaId: string): Promise<string> {
     const persona = this.stateManager.persona_getById(personaId);
     if (!persona) return "";
-    
     this.clearPendingRequestsFor(personaId);
-    this.stateManager.queue_pause();
-    
     const messages = this.stateManager.messages_get(personaId);
     const pendingIds = messages
       .filter(m => m.role === "human" && !m.read)
       .map(m => m.id);
-    
     if (pendingIds.length === 0) return "";
-    
     const removed = this.stateManager.messages_remove(personaId, pendingIds);
-    const recalledContent = removed.map(m => m.content).join("\n\n");
-    
+    const recalledContent = removed.map(m => m.verbal_response ?? '').join("\n\n");
     this.interface.onMessageAdded?.(personaId);
     this.interface.onMessageRecalled?.(personaId, recalledContent);
-    
     return recalledContent;
   }
 
@@ -848,7 +976,7 @@ export class Processor {
     const message: Message = {
       id: crypto.randomUUID(),
       role: "human",
-      content,
+      verbal_response: content,
       timestamp: new Date().toISOString(),
       read: false,
       context_status: "default" as ContextStatus,
@@ -987,13 +1115,40 @@ export class Processor {
   ): Promise<ResponsePromptData["human"]> {
     const DEFAULT_GROUP = "General";
     const QUOTE_LIMIT = 10;
+    const DATA_ITEM_LIMIT = 15;
     const SIMILARITY_THRESHOLD = 0.3;
+    // Generic relevance selector for embedding-capable items.
+    // Falls back to returning all items when no message/embeddings are available.
+    const selectRelevantItems = async <T extends { id: string; embedding?: number[] }>(
+      items: T[],
+      limit: number
+    ): Promise<T[]> => {
+      if (items.length === 0) return [];
+
+      const withEmbeddings = items.filter(i => i.embedding?.length);
+
+      if (currentMessage && withEmbeddings.length > 0) {
+        try {
+          const embeddingService = getEmbeddingService();
+          const queryVector = await embeddingService.embed(currentMessage);
+          const results = findTopK(queryVector, withEmbeddings, limit);
+          const relevant = results
+            .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
+            .map(({ item }) => item);
+
+          if (relevant.length > 0) return relevant;
+        } catch (err) {
+          console.warn("[filterHumanDataByVisibility] Embedding search failed:", err);
+        }
+      }
 
+      // Fallback: return all items (caller may apply its own limit)
+      return items;
+    };
     const selectRelevantQuotes = async (quotes: Quote[]): Promise<Quote[]> => {
       if (quotes.length === 0) return [];
-      
       const withEmbeddings = quotes.filter(q => q.embedding?.length);
-      
+
       if (currentMessage && withEmbeddings.length > 0) {
         try {
           const embeddingService = getEmbeddingService();
@@ -1002,35 +1157,31 @@ export class Processor {
           const relevant = results
             .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
             .map(({ item }) => item);
-          
+
           if (relevant.length > 0) return relevant;
         } catch (err) {
           console.warn("[filterHumanDataByVisibility] Embedding search failed:", err);
         }
       }
-      
       return [...quotes]
         .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
         .slice(0, QUOTE_LIMIT);
     };
-
     if (persona.id === "ei") {
-      const relevantQuotes = await selectRelevantQuotes(human.quotes ?? []);
-      return {
-        facts: human.facts,
-        traits: human.traits,
-        topics: human.topics,
-        people: human.people,
-        quotes: relevantQuotes,
-      };
+      const [facts, traits, topics, people, quotes] = await Promise.all([
+        selectRelevantItems(human.facts, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.traits, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.topics, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.people, DATA_ITEM_LIMIT),
+        selectRelevantQuotes(human.quotes ?? []),
+      ]);
+      return { facts, traits, topics, people, quotes };
     }
-
     const visibleGroups = new Set<string>();
     if (persona.group_primary) {
       visibleGroups.add(persona.group_primary);
     }
     (persona.groups_visible ?? []).forEach((g) => visibleGroups.add(g));
-
     const filterByGroup = <T extends DataItemBase>(items: T[]): T[] => {
       return items.filter((item) => {
         const itemGroups = item.persona_groups ?? [];
@@ -1038,21 +1189,20 @@ export class Processor {
         return effectiveGroups.some((g) => visibleGroups.has(g));
       });
     };
-
     const groupFilteredQuotes = (human.quotes ?? []).filter((q) => {
       const effectiveGroups = q.persona_groups.length === 0 ? [DEFAULT_GROUP] : q.persona_groups;
       return effectiveGroups.some((g) => visibleGroups.has(g));
     });
 
-    const relevantQuotes = await selectRelevantQuotes(groupFilteredQuotes);
+    const [facts, traits, topics, people, quotes] = await Promise.all([
+      selectRelevantItems(filterByGroup(human.facts), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.traits), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.topics), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.people), DATA_ITEM_LIMIT),
+      selectRelevantQuotes(groupFilteredQuotes),
+    ]);
 
-    return {
-      facts: filterByGroup(human.facts),
-      traits: filterByGroup(human.traits),
-      topics: filterByGroup(human.topics),
-      people: filterByGroup(human.people),
-      quotes: relevantQuotes,
-    };
+    return { facts, traits, topics, people, quotes };
   }
 
   private getVisiblePersonas(
@@ -1342,13 +1492,31 @@ export class Processor {
     return {
       state: this.stateManager.queue_isPaused()
         ? "paused"
-        : this.queueProcessor.getState() === "busy"
+        : this.stateManager.queue_hasProcessingItem()
           ? "busy"
           : "idle",
       pending_count: this.stateManager.queue_length(),
+      dlq_count: this.stateManager.queue_dlqLength(),
     };
   }
 
+  pauseQueue(): void {
+    this.stateManager.queue_pause();
+    this.queueProcessor.abort();
+  }
+
+  getQueueActiveItems(): LLMRequest[] {
+    return this.stateManager.queue_getAllActiveItems();
+  }
+
+  getDLQItems(): LLMRequest[] {
+    return this.stateManager.queue_getDLQItems();
+  }
+
+  updateQueueItem(id: string, updates: Partial<LLMRequest>): boolean {
+    return this.stateManager.queue_updateItem(id, updates);
+  }
+
   async clearQueue(): Promise<number> {
     this.queueProcessor.abort();
     return this.stateManager.queue_clear();

package/src/core/queue-processor.ts

@@ -1,5 +1,5 @@
-import type { LLMRequest, LLMResponse, LLMRequestType, ProviderAccount, ChatMessage, Message } from "./types.js";
-import { callLLMRaw, parseJSONResponse, cleanResponseContent } from "./llm-client.js";
+import { LLMRequest, LLMResponse, LLMRequestType, ProviderAccount, ChatMessage, Message } from "./types.js";
+import { callLLMRaw, parseJSONResponse } from "./llm-client.js";
 import { hydratePromptPlaceholders } from "../prompts/message-utils.js";
 
 type QueueProcessorState = "idle" | "busy";
@@ -133,9 +133,8 @@ export class QueueProcessor {
   ): LLMResponse {
     switch (request.type) {
       case "json" as LLMRequestType:
-        return this.handleJSONResponse(request, content, finishReason);
       case "response" as LLMRequestType:
-        return this.handleConversationResponse(request, content, finishReason);
+        return this.handleJSONResponse(request, content, finishReason);
       case "raw" as LLMRequestType:
       default:
         return {
@@ -172,26 +171,4 @@ export class QueueProcessor {
     }
   }
 
-  private handleConversationResponse(
-    request: LLMRequest,
-    content: string,
-    finishReason: string | null
-  ): LLMResponse {
-    const cleaned = cleanResponseContent(content);
-    
-    const noMessagePatterns = [
-      /^no\s*(new\s*)?(message|response)/i,
-      /^nothing\s+to\s+(say|add)/i,
-      /^\[no\s+message\]/i,
-    ];
-    
-    const isNoMessage = noMessagePatterns.some((p) => p.test(cleaned));
-    
-    return {
-      request,
-      success: true,
-      content: isNoMessage ? null : cleaned,
-      finish_reason: finishReason ?? undefined,
-    };
   }
-}

package/src/core/state/checkpoints.ts

@@ -45,6 +45,10 @@ export class PersistenceState {
     return this.loadedExistingData;
   }
 
+  markExistingData(): void {
+    this.loadedExistingData = true;
+  }
+
   async flush(): Promise<void> {
     if (this.saveTimeout) {
       clearTimeout(this.saveTimeout);

package/src/core/state/personas.ts

@@ -1,5 +1,17 @@
 import type { PersonaEntity, Message, ContextStatus } from "../types.js";
 
+/**
+ * Migration: If a persisted message has the old `content` field but no `verbal_response`,
+ * move content → verbal_response. Runs on every load (no-op for already-migrated data).
+ */
+function migrateMessage(msg: Message & { content?: string }): Message {
+  if (msg.content !== undefined && msg.verbal_response === undefined) {
+    const { content, ...rest } = msg;
+    return { ...rest, verbal_response: content };
+  }
+  return msg;
+}
+
 export interface PersonaData {
   entity: PersonaEntity;
   messages: Message[];
@@ -13,7 +25,7 @@ export class PersonaState {
     this.personas = new Map(
       Object.entries(personas).map(([id, data]) => [
         id,
-        { entity: data.entity, messages: data.messages },
+        { entity: data.entity, messages: data.messages.map(migrateMessage) },
       ])
     );
   }