ei-tui 0.1.4 → 0.1.6

package/src/core/state/queue.ts

@@ -1,4 +1,5 @@
 import type { LLMRequest, QueueFailResult } from "../types.js";
+import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -27,35 +28,55 @@ export class QueueState {
   private paused = false;
 
   load(queue: LLMRequest[]): void {
-    this.queue = queue;
+    // Reset any items stuck in 'processing' from a previous session
+    this.queue = queue.map(r =>
+      r.state === "processing" ? { ...r, state: "pending" } : r
+    );
   }
 
   export(): LLMRequest[] {
     return this.queue;
   }
 
-  enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts">): string {
+  enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts" | "state">): string {
     const id = crypto.randomUUID();
     const fullRequest: LLMRequest = {
       ...request,
       id,
       created_at: new Date().toISOString(),
       attempts: 0,
+      state: "pending",
     };
     this.queue.push(fullRequest);
     return id;
   }
 
-  peekHighest(): LLMRequest | null {
+  claimHighest(): LLMRequest | null {
     if (this.paused || this.queue.length === 0) return null;
-    const now = new Date().toISOString();
-    const available = this.queue.filter(r => !r.retry_after || r.retry_after <= now);
+    const available = this.queue.filter(r => r.state === "pending");
     if (available.length === 0) return null;
     const priorityOrder = { high: 0, normal: 1, low: 2 };
     const sorted = [...available].sort(
       (a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]
     );
-    return sorted[0] ?? null;
+    const item = sorted[0] ?? null;
+    if (item) {
+      item.state = "processing";
+    }
+    return item;
+  }
+
+  // Returns the retry_after of the highest-priority pending item, or null if ready now.
+  // Used by the processor loop to decide whether to sleep instead of claiming.
+  nextItemRetryAfter(): string | null {
+    if (this.paused || this.queue.length === 0) return null;
+    const available = this.queue.filter(r => r.state === "pending");
+    if (available.length === 0) return null;
+    const priorityOrder = { high: 0, normal: 1, low: 2 };
+    const sorted = [...available].sort(
+      (a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]
+    );
+    return sorted[0]?.retry_after ?? null;
   }
 
   complete(id: string): void {
@@ -76,18 +97,21 @@ export class QueueState {
     }
 
     // No error string and not flagged permanent = just increment, no classification
+    // Still reset to pending in case it was claimed (processing state)
     if (!error && !permanent) {
+      request.state = "pending";
       return { dropped: false };
     }
 
     const shouldDrop = permanent || (error ? isPermanentError(error) : false);
 
     if (shouldDrop) {
-      this.queue.splice(idx, 1);
+      request.state = "dlq";
       return { dropped: true };
     }
 
-    // Transient error — apply exponential backoff, never drop
+    // Transient error — reset to pending with backoff timer
+    request.state = "pending";
     const delay = calculateBackoff(request.attempts);
     request.retry_after = new Date(Date.now() + delay).toISOString();
     return { dropped: false, retryDelay: delay };
@@ -98,7 +122,7 @@ export class QueueState {
   clearPersonaResponses(personaId: string, nextStep: string): string[] {
     const removedIds: string[] = [];
     this.queue = this.queue.filter((r) => {
-      if (r.next_step === nextStep && r.data.personaId === personaId) {
+      if (r.state !== "dlq" && r.next_step === nextStep && r.data.personaId === personaId) {
         removedIds.push(r.id);
         return false;
       }
@@ -108,7 +132,49 @@ export class QueueState {
   }
 
   length(): number {
-    return this.queue.length;
+    return this.queue.filter(r => r.state === "pending" || r.state === "processing").length;
+  }
+
+  dlqLength(): number {
+    return this.queue.filter(r => r.state === "dlq").length;
+  }
+
+  hasProcessingItem(): boolean {
+    return this.queue.some(r => r.state === "processing");
+  }
+
+  getDLQItems(): LLMRequest[] {
+    return this.queue.filter(r => r.state === "dlq");
+  }
+
+  getAllActiveItems(): LLMRequest[] {
+    return this.queue.filter(r => r.state !== "dlq");
+  }
+
+  updateItem(id: string, updates: Partial<LLMRequest>): boolean {
+    const idx = this.queue.findIndex(r => r.id === id);
+    if (idx < 0) return false;
+    this.queue[idx] = { ...this.queue[idx], ...updates };
+    return true;
+  }
+
+  trimDLQ(): number {
+    const dlqItems = this.queue.filter(r => r.state === "dlq");
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - DLQ_MAX_AGE_DAYS);
+    const cutoffISO = cutoff.toISOString();
+
+    // Remove by age first
+    const afterAgeTrim = dlqItems.filter(r => r.created_at >= cutoffISO);
+
+    // Then enforce count limit (oldest first)
+    const sorted = afterAgeTrim.sort((a, b) => a.created_at.localeCompare(b.created_at));
+    const kept = sorted.slice(-DLQ_MAX_COUNT);
+    const keptIds = new Set(kept.map(r => r.id));
+
+    const before = this.queue.length;
+    this.queue = this.queue.filter(r => r.state !== "dlq" || keptIds.has(r.id));
+    return before - this.queue.length;
   }
 
   pause(): void {
@@ -124,12 +190,12 @@ export class QueueState {
   }
 
   hasPendingCeremonies(): boolean {
-    return this.queue.some(r => r.data.ceremony_progress === true);
+    return this.queue.some(r => r.state !== "dlq" && r.data.ceremony_progress === true);
   }
 
   clear(): number {
-    const count = this.queue.length;
-    this.queue = [];
+    const count = this.queue.filter(r => r.state !== "dlq").length;
+    this.queue = this.queue.filter(r => r.state === "dlq");
     return count;
   }
 }

package/src/core/state-manager.ts

@@ -250,7 +250,7 @@ export class StateManager {
     return result;
   }
 
-  queue_enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts">): string {
+  queue_enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts" | "state">): string {
     const requestWithModel = {
       ...request,
       model: request.model ?? this.humanState.get().settings?.default_model,
@@ -260,8 +260,8 @@ export class StateManager {
     return id;
   }
 
-  queue_peekHighest(): LLMRequest | null {
-    return this.queueState.peekHighest();
+  queue_claimHighest(): LLMRequest | null {
+    return this.queueState.claimHighest();
   }
 
   queue_complete(id: string): void {
@@ -287,6 +287,14 @@ export class StateManager {
     return this.queueState.length();
   }
 
+  queue_hasProcessingItem(): boolean {
+    return this.queueState.hasProcessingItem();
+  }
+
+  queue_nextItemRetryAfter(): string | null {
+    return this.queueState.nextItemRetryAfter();
+  }
+
   queue_pause(): void {
     this.queueState.pause();
     this.scheduleSave();
@@ -311,6 +319,30 @@ export class StateManager {
     return result;
   }
 
+  queue_dlqLength(): number {
+    return this.queueState.dlqLength();
+  }
+
+  queue_getDLQItems(): LLMRequest[] {
+    return this.queueState.getDLQItems();
+  }
+
+  queue_getAllActiveItems(): LLMRequest[] {
+    return this.queueState.getAllActiveItems();
+  }
+
+  queue_updateItem(id: string, updates: Partial<LLMRequest>): boolean {
+    const result = this.queueState.updateItem(id, updates);
+    if (result) this.scheduleSave();
+    return result;
+  }
+
+  queue_trimDLQ(): number {
+    const result = this.queueState.trimDLQ();
+    if (result > 0) this.scheduleSave();
+    return result;
+  }
+
   async flush(): Promise<void> {
     await this.persistenceState.flush();
   }

package/src/core/types.ts

@@ -256,6 +256,10 @@ export interface PersonaCreationInput {
 export const MESSAGE_MIN_COUNT = 200;
 export const MESSAGE_MAX_AGE_DAYS = 14;
 
+// DLQ rolloff thresholds
+export const DLQ_MAX_COUNT = 50;
+export const DLQ_MAX_AGE_DAYS = 14;
+
 // Reserved persona names (command keywords that conflict with /persona subcommands)
 export const RESERVED_PERSONA_NAMES = ["new", "clone"] as const;
 export type ReservedPersonaName = typeof RESERVED_PERSONA_NAMES[number];
@@ -271,14 +275,19 @@ export function isReservedPersonaName(name: string): boolean {
 export interface Message {
   id: string;
   role: "human" | "system";
-  content: string;
+  verbal_response?: string;   // Human text or persona's spoken reply
+  action_response?: string;  // Stage direction / action the persona performs
+  silence_reason?: string;   // Why the persona chose not to respond (not shown to LLM)
   timestamp: string;
-  read: boolean;
+  read: boolean;               // Has human seen this system message?
   context_status: ContextStatus;
-  f?: boolean;  // fact extraction done
-  r?: boolean;  // trait extraction done
-  p?: boolean;  // person extraction done
-  o?: boolean;  // topic extraction done
+  
+  // Extraction completion flags (omit when false to save space)
+  // Single-letter names minimize storage overhead for large message histories
+  f?: boolean;                 // Fact extraction completed
+  r?: boolean;                 // tRait extraction completed
+  p?: boolean;                 // Person extraction completed
+  o?: boolean;                 // tOpic extraction completed
 }
 
 export interface ChatMessage {
@@ -290,12 +299,15 @@ export interface ChatMessage {
 // LLM TYPES
 // =============================================================================
 
+export type LLMRequestState = "pending" | "processing" | "dlq";
+
 export interface LLMRequest {
   id: string;
   created_at: string;
   attempts: number;
   last_attempt?: string;
   retry_after?: string;
+  state: LLMRequestState;
   type: LLMRequestType;
   priority: LLMPriority;
   system: string;
@@ -345,6 +357,7 @@ export interface MessageQueryOptions {
 export interface QueueStatus {
   state: "idle" | "busy" | "paused";
   pending_count: number;
+  dlq_count: number;
   current_operation?: string;
 }
 

package/src/integrations/opencode/importer.ts

@@ -47,7 +47,7 @@ function convertToEiMessage(ocMsg: OpenCodeMessage): Message {
   return {
     id: ocMsg.id,
     role: ocMsg.role === "user" ? "human" : "system",
-    content: ocMsg.content,
+    verbal_response: ocMsg.content,
     timestamp: ocMsg.timestamp,
     read: true,
     context_status: "default" as ContextStatus,

package/src/prompts/heartbeat/check.ts

@@ -6,9 +6,8 @@
  */
 
 import type { HeartbeatCheckPromptData, PromptOutput } from "./types.js";
-import type { Message, Topic, Person } from "../../core/types.js";
-import { formatMessagesAsPlaceholders } from "../message-utils.js";
-
+import { type Message, type Topic, type Person } from "../../core/types.js";
+import { formatMessagesAsPlaceholders, getMessageDisplayText } from "../message-utils.js";
 function formatTopicsWithGaps(topics: Topic[]): string {
   if (topics.length === 0) return "(No topics with engagement gaps)";
   
@@ -31,23 +30,37 @@ function formatPeopleWithGaps(people: Person[]): string {
     .join('\n');
 }
 
+/**
+ * A "real" persona message is one the persona actually said to the human.
+ * silence_reason messages (persona chose not to speak) and action-only messages
+ * (no verbal_response) don't count as conversational outreach.
+ */
+function isConversationalMessage(m: Message): boolean {
+  if (m.role !== 'system') return false;
+  if (m.silence_reason !== undefined) return false;
+  // Action-only: has action but no verbal response
+  if (!m.verbal_response) return false;
+  return true;
+}
+
 function countTrailingPersonaMessages(history: Message[]): number {
   if (history.length === 0) return 0;
 
   let count = 0;
   for (let i = history.length - 1; i >= 0; i--) {
-    // In heartbeat context, persona messages are "system" role (not from human)
-    if (history[i].role === "system") {
-      count++;
-    } else {
-      break;
-    }
+    const msg = history[i];
+    if (msg.role === 'human') break;
+    if (isConversationalMessage(msg)) count++;
+    // Skip non-conversational system messages and keep looking back
   }
   return count;
 }
 
 function getLastPersonaMessage(history: Message[]): Message | undefined {
-  return history.filter(m => m.role === "system").slice(-1)[0];
+  for (let i = history.length - 1; i >= 0; i--) {
+    if (isConversationalMessage(history[i])) return history[i];
+  }
+  return undefined;
 }
 
 /**
@@ -148,9 +161,10 @@ ${formatMessagesAsPlaceholders(data.recent_history, personaName)}`;
   
   let unansweredWarning = '';
   if (lastPersonaMsg && consecutiveMessages >= 1) {
-    const preview = lastPersonaMsg.content.length > 100 
-      ? lastPersonaMsg.content.substring(0, 100) + "..." 
-      : lastPersonaMsg.content;
+    const rawPreview = getMessageDisplayText(lastPersonaMsg) ?? "";
+    const preview = rawPreview.length > 100 
+      ? rawPreview.substring(0, 100) + "..." 
+      : rawPreview;
     
     unansweredWarning = `
 ### CRITICAL: You Already Reached Out

package/src/prompts/heartbeat/ei.ts

@@ -1,6 +1,6 @@
 import type { EiHeartbeatPromptData, EiHeartbeatItem, PromptOutput } from "./types.js";
 import type { Message } from "../../core/types.js";
-import { formatMessagesAsPlaceholders } from "../message-utils.js";
+import { formatMessagesAsPlaceholders, getMessageDisplayText } from "../message-utils.js";
 
 function formatItem(item: EiHeartbeatItem): string {
   switch (item.type) {
@@ -105,9 +105,10 @@ ${formatMessagesAsPlaceholders(data.recent_history, "Ei")}`;
 
   let unansweredWarning = "";
   if (lastEiMsg && consecutiveMessages >= 1) {
-    const preview = lastEiMsg.content.length > 100
-      ? lastEiMsg.content.substring(0, 100) + "..."
-      : lastEiMsg.content;
+    const rawPreview = getMessageDisplayText(lastEiMsg) ?? "";
+    const preview = rawPreview.length > 100
+      ? rawPreview.substring(0, 100) + "..."
+      : rawPreview;
 
     unansweredWarning = `
 ### CRITICAL: You Already Reached Out

package/src/prompts/human/item-update.ts

@@ -240,13 +240,26 @@ In addition to updating the ${typeLabel}, identify any **memorable, funny, impor
 - Phrases that reveal personality or communication style
 - Things you'd quote back to them later to make them laugh
 - Unique expressions, malaphors, or turns of phrase
-
-**De-prioritize:**
-- Dry technical facts (those belong in TOPICS)
-- Status updates or process descriptions
-- Generic statements that could come from anyone
-
-A quote like "Does the Pope shit in his hat?" is GOLD. A quote like "We're running a batch of 44k students" is just... data.
+- Quotable moments from EITHER speaker — humans AND AI personas both say memorable things
+
+**NEVER extract these — they are NOT quotes:**
+- Technical identifiers: ARNs, URLs, file paths, UUIDs, config keys, environment variable values, role/policy names
+- AI agent self-talk: "I notice I'm in Plan Mode", "I'll start by...", "Let me help you with...", status updates about the agent's own process
+- AI apologies or acknowledgments: "You're absolutely right", "I apologize for that overreach", "Good decision to revert"
+- Generic AI instructions or tips: "Remember to include X in your prompts", tool usage advice, workflow suggestions
+- Dry technical facts: infrastructure descriptions, process status, batch sizes, system architecture summaries
+- Status updates or process descriptions: "We're running a batch of...", "The pipeline is...", "I'm currently working on..."
+- Generic statements that could come from anyone or any AI session
+- Credentials, secrets, connection strings, or anything that looks like an access token
+
+**The litmus test**: Would you bring this up at a bar with a friend? Would it make someone laugh, think, or feel something?
+- "Does the Pope shit in his hat?" → YES. Hilarious malaphor.
+- "AWSReservedSSO_cmidp-nihl-sandbox-adm_db7b191e026bdd85" → NO. That's a credential.
+- "Slow is smooth. Smooth is fast." → YES (once). Pithy wisdom.
+- "The authentication flow is working correctly now" → NO. Status update.
+- "I built this, and now it's live." → YES. Pride and accomplishment.
+
+**When in doubt, leave it out.** An empty quotes array is always acceptable.
 
 Return them in the \`quotes\` array:
 
@@ -266,7 +279,6 @@ Return them in the \`quotes\` array:
 
 **CRITICAL**: Return the EXACT text as it appears in the message (spacing, punctuation, formatting, etc.). WE CAN ONLY USE IT IF WE FIND IT IN THE TEXT.
 
-
 # CRITICAL INSTRUCTIONS
 
 ONLY ANALYZE the "Most Recent Messages" in the following conversation. The "Earlier Conversation" is provided for your context and has already been processed!

package/src/prompts/message-utils.ts

@@ -2,6 +2,42 @@ import type { Message } from "../core/types.js";
 
 const MESSAGE_PLACEHOLDER_REGEX = /\[mid:([a-zA-Z0-9_-]+):([^\]]+)\]/g;
 
+/**
+ * Returns the display text for a message from its structured fields.
+ * - action_response as _italics_
+ * - verbal_response as plain text
+ * - silence_reason shown so the user understands why a persona stayed silent
+ */
+export function getMessageDisplayText(message: Message): string | null {
+  const parts: string[] = [];
+  if (message.action_response) parts.push(`_${message.action_response}_`);
+  if (message.verbal_response) parts.push(message.verbal_response);
+  if (message.silence_reason) {
+    const name = 'Persona'; // Caller doesn't pass persona name; frontends can override
+    parts.push(`[${name} chose not to respond because: ${message.silence_reason}]`);
+  }
+  if (parts.length === 0) return null;
+  return parts.join('\n\n');
+}
+
+/**
+ * Builds the content string for a ChatMessage sent to the LLM.
+ * Unlike getMessageDisplayText (which is for frontend rendering and skips silence),
+ * this includes ALL structured fields so the persona has full conversational context:
+ *   - action_response as _italics_
+ *   - verbal_response as plain text
+ *   - silence_reason as "You chose not to respond because: ..."
+ */
+export function buildChatMessageContent(message: Message): string {
+  const parts: string[] = [];
+  if (message.action_response) parts.push(`_${message.action_response}_`);
+  if (message.verbal_response) parts.push(message.verbal_response);
+  if (message.silence_reason) {
+    parts.push(`You chose not to respond because: ${message.silence_reason}`);
+  }
+  return parts.join('\n\n');
+}
+
 export function formatMessageAsPlaceholder(message: Message, personaName: string): string {
   const role = message.role === "human" ? "human" : personaName;
   return `[mid:${message.id}:${role}]`;
@@ -22,7 +58,8 @@ export function hydratePromptPlaceholders(
       return `[${role}]: [message not found]`;
     }
     const displayRole = message.role === "human" ? "[human]" : `[${role}]`;
-    return `${displayRole}: ${message.content}`;
+    const text = getMessageDisplayText(message) ?? "[no content]";
+    return `${displayRole}: ${text}`;
   });
 }
 

package/src/prompts/response/index.ts

@@ -19,9 +19,10 @@ import {
   buildQuotesSection,
   buildSystemKnowledgeSection,
   getConversationStateText,
+  buildResponseFormatSection,
 } from "./sections.js";
 
-export type { ResponsePromptData, PromptOutput } from "./types.js";
+export type { ResponsePromptData, PromptOutput, PersonaResponseResult } from "./types.js";
 
 /**
  * Special system prompt for Ei (the system guide persona)
@@ -47,6 +48,7 @@ Your role is unique among personas:
   const associatesSection = buildAssociatesSection(data.visible_personas);
   const systemKnowledge = buildSystemKnowledgeSection(data.isTUI);
   const priorities = buildPrioritiesSection(data.persona, data.human);
+  const responseFormat = buildResponseFormatSection();
   const currentTime = new Date().toISOString();
 
   return `${identity}
@@ -63,16 +65,19 @@ ${associatesSection}
 ${systemKnowledge}
 ${priorities}
 
+${responseFormat}
+
 Current time: ${currentTime}
 
 ## Final Instructions
 - NEVER repeat or echo the user's message in your response. Start directly with your own words.
 - The developers cannot see any message sent by the user, any response from personas, or any other data in the system.
 - If the user has a problem, THEY need to visit https://flare576.com. You cannot send the devs a message
-- DO NOT INCLUDE <thinking> PROCESS NOTES - adding "internal monologue" or story content is fine, but do not include analysis of the user's messages
-- If you decide not to respond, say exactly: No Message`;
+- Your entire reply must be the JSON object. No prose before or after it.`
 }
 
+const RESPONSE_FORMAT_INSTRUCTION = `Respond to the conversation above using the JSON format specified in the Response Format section.`;
+
 /**
  * Standard system prompt for non-Ei personas
  */
@@ -85,6 +90,7 @@ function buildStandardSystemPrompt(data: ResponsePromptData): string {
   const quotesSection = buildQuotesSection(data.human.quotes, data.human);
   const associatesSection = buildAssociatesSection(data.visible_personas);
   const priorities = buildPrioritiesSection(data.persona, data.human);
+  const responseFormat = buildResponseFormatSection();
   const currentTime = new Date().toISOString();
 
   return `${identity}
@@ -100,12 +106,13 @@ ${quotesSection}
 ${associatesSection}
 ${priorities}
 
+${responseFormat}
+
 Current time: ${currentTime}
 
 ## Final Instructions
 - NEVER repeat or echo the user's message in your response. Start directly with your own words.
-- DO NOT INCLUDE <thinking> PROCESS NOTES - adding "internal monologue" or story content is fine, but do not include analysis of the user's messages
-- If you decide not to respond, say exactly: No Message`;
+- Your entire reply must be the JSON object. No prose before or after it.`
 }
 
 function buildUserPrompt(data: ResponsePromptData): string {
@@ -113,7 +120,7 @@ function buildUserPrompt(data: ResponsePromptData): string {
   
   return `${conversationState}
 
-Respond to the conversation above. If silence is appropriate, say exactly: No Message`;
+${RESPONSE_FORMAT_INSTRUCTION}`;
 }
 
 /**