ei-tui 0.1.23 → 0.1.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.23",
+  "version": "0.1.25",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/AGENTS.md

@@ -49,6 +49,22 @@ Priority queue for LLM requests:
 
 **Async model**: Handlers queue work, don't await results inline.
 
+### llm-client.ts
+
+Multi-provider LLM abstraction layer:
+- Handles requests to Anthropic, OpenAI, Bedrock, local models
+- **Sets `max_tokens: 64000`** for all requests
+- Prevents unbounded generation (test showed timeout after 2min without limit)
+- Local models silently clamp to their configured maximums
+- Anthropic Opus 4 accepts 64K (200K total context - 64K output = 136K input budget)
+
+**JSON Response Parsing** (`parseJSONResponse()`):
+- **Strategy 1**: Extract from markdown code blocks (```json)
+- **Strategy 2**: Auto-repair malformed JSON (trailing commas, etc.)
+- **Strategy 3**: Extract outermost `{...}` from mixed prose/JSON (handles LLM preamble)
+
+No prompt changes needed for JSON-only output—parser handles natural language gracefully.
+
 ### handlers/index.ts (1000+ lines)
 
 All `LLMNextStep` handlers in one file. Each handler:

package/src/core/handlers/dedup.ts

@@ -0,0 +1,238 @@
+import { StateManager } from "../state-manager.js";
+import { LLMResponse } from "../types.js";
+import type { DedupResult } from "../../prompts/ceremony/types.js";
+import type { DataItemType, Fact, Trait, Topic, Person, Quote } from "../types/data-items.js";
+import { getEmbeddingService } from "../embedding-service.js";
+
+/**
+ * handleDedupCurate — Process Opus deduplication decisions
+ * 
+ * This handler receives merge decisions from Opus and applies them:
+ * 1. Updates: Entities with revised descriptions/merged data
+ * 2. Removes: Duplicate entities to delete (with foreign key updates)
+ * 3. Adds: New entities created from consolidation
+ * 
+ * CRITICAL: Quote foreign keys must be updated BEFORE deletions to maintain
+ * referential integrity.
+ */
+export async function handleDedupCurate(
+  response: LLMResponse,
+  stateManager: StateManager
+): Promise<void> {
+  const entity_type = response.request.data.entity_type as DataItemType;
+  const entity_ids = response.request.data.entity_ids as string[];
+  const state = stateManager.getHuman();
+  
+  // Validate entity_type
+  if (!entity_type || !['fact', 'trait', 'topic', 'person'].includes(entity_type)) {
+    console.error(`[Dedup] Invalid entity_type: "${entity_type}" (from request data)`, response.request.data);
+    return;
+  }
+  
+  // Parse Opus response
+  let decisions: DedupResult;
+  try {
+    decisions = response.parsed as DedupResult;
+    if (!decisions || typeof decisions !== 'object') {
+      throw new Error("Invalid response format");
+    }
+  } catch (err) {
+    console.error(`[Dedup] Failed to parse Opus response:`, err);
+    return;
+  }
+  
+  // Validate response structure
+  if (!Array.isArray(decisions.update) || !Array.isArray(decisions.remove) || !Array.isArray(decisions.add)) {
+    console.error(`[Dedup] Invalid response structure - missing update/remove/add arrays`);
+    return;
+  }
+  
+  console.log(`[Dedup] Processing cluster: ${decisions.update.length} updates, ${decisions.remove.length} removals, ${decisions.add.length} additions`);
+  
+  // Map entity_type to pluralized state property name
+  const pluralMap: Record<DataItemType, 'facts' | 'traits' | 'topics' | 'people'> = {
+    fact: 'facts',
+    trait: 'traits',
+    topic: 'topics',
+    person: 'people'
+  };
+  const entityList = state[pluralMap[entity_type]];
+  
+  // Validate entityList exists
+  if (!entityList || !Array.isArray(entityList)) {
+    console.error(`[Dedup] entityList is ${entityList === undefined ? 'undefined' : 'not an array'} for entity_type="${entity_type}" (looking for state.${entity_type}s)`, {
+      entity_type,
+      entity_ids,
+      stateKeys: Object.keys(state),
+      factsExists: !!state.facts,
+      traitsExists: !!state.traits,
+      topicsExists: !!state.topics,
+      peopleExists: !!state.people
+    });
+    return;
+  }
+  const entities = entity_ids
+    .map((id: string) => entityList.find((e: Fact | Trait | Topic | Person) => e.id === id))
+    .filter((e: Fact | Trait | Topic | Person | undefined): e is (Fact | Trait | Topic | Person) => e !== undefined);
+  
+  if (entities.length === 0) {
+    console.warn(`[Dedup] No entities found for cluster (already merged?)`);
+    return;
+  }
+  
+  // =========================================================================
+  // PHASE 1: Update Quote foreign keys FIRST (before deletions)
+  // =========================================================================
+  
+  for (const removal of decisions.remove) {
+    const quotes = state.quotes.filter((q: Quote) =>
+      q.data_item_ids.includes(removal.to_be_removed)
+    );
+    
+    for (const quote of quotes) {
+      const updatedIds = quote.data_item_ids
+        .map((id: string) => id === removal.to_be_removed ? removal.replaced_by : id)
+        .filter((id: string, idx: number, arr: string[]) => arr.indexOf(id) === idx);  // Dedupe
+      
+      stateManager.human_quote_update(quote.id, {
+        data_item_ids: updatedIds
+      });
+    }
+    
+    if (quotes.length > 0) {
+      console.log(`[Dedup] Updated ${quotes.length} quotes referencing ${removal.to_be_removed}`);
+    }
+  }
+  
+  // =========================================================================
+  // PHASE 2: Apply updates (merge decisions)
+  // =========================================================================
+  
+  for (const update of decisions.update) {
+    const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === update.id);
+    
+    if (!entity) {
+      console.warn(`[Dedup] Entity ${update.id} not found (already merged?)`);
+      continue;  // Graceful skip
+    }
+    
+    // Recalculate embedding if description changed
+    let embedding = entity.embedding;
+    if (update.description !== entity.description) {
+      try {
+        const embeddingService = getEmbeddingService();
+        embedding = await embeddingService.embed(update.description);
+      } catch (err) {
+        console.warn(`[Dedup] Failed to recalculate embedding for ${update.id}`, err);
+        // Fallback to old embedding if recalculation fails
+      }
+    }
+    
+    // Build complete entity with updates (preserve original fields if LLM omits them)
+      const updatedEntity = {
+      ...entity,
+      name: update.name ?? entity.name,
+      description: update.description ?? entity.description,
+      sentiment: update.sentiment ?? entity.sentiment,
+      last_updated: new Date().toISOString(),
+      embedding,
+      // Type-specific fields
+      ...(update.strength !== undefined && { strength: update.strength }),
+      ...(update.confidence !== undefined && { confidence: update.confidence }),
+      ...(update.exposure_current !== undefined && { exposure_current: update.exposure_current }),
+      ...(update.exposure_desired !== undefined && { exposure_desired: update.exposure_desired }),
+      ...(update.relationship !== undefined && { relationship: update.relationship }),
+      ...(update.category !== undefined && { category: update.category }),
+    };
+    
+    // Type-safe cast based on entity_type
+    if (entity_type === 'fact') {
+      stateManager.human_fact_upsert(updatedEntity as Fact);
+    } else if (entity_type === 'trait') {
+      stateManager.human_trait_upsert(updatedEntity as Trait);
+    } else if (entity_type === 'topic') {
+      stateManager.human_topic_upsert(updatedEntity as Topic);
+    } else if (entity_type === 'person') {
+      stateManager.human_person_upsert(updatedEntity as Person);
+    }
+    console.log(`[Dedup] Updated ${entity_type} "${update.name}"`);
+  }
+  
+  // =========================================================================
+  // PHASE 3: Apply removals (soft-delete with replaced_by tracking)
+  // =========================================================================
+  
+  for (const removal of decisions.remove) {
+    const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === removal.to_be_removed);
+    
+    if (!entity) {
+      console.warn(`[Dedup] Entity ${removal.to_be_removed} already deleted`);
+      continue;  // Graceful skip
+    }
+    
+    // Remove via StateManager (also cleans up quote references)
+    const removeMethod = `human_${entity_type}_remove` as 
+      'human_fact_remove' | 'human_trait_remove' | 'human_topic_remove' | 'human_person_remove';
+    
+    const removed = stateManager[removeMethod](removal.to_be_removed);
+    if (removed) {
+      console.log(`[Dedup] Removed ${entity_type} "${entity.name}" (merged into ${removal.replaced_by})`);
+    }
+  }
+  
+  // =========================================================================
+  // PHASE 4: Apply additions (new entities from consolidation)
+  // =========================================================================
+  
+  for (const addition of decisions.add) {
+    // Compute embedding for new entity
+    let embedding: number[] | undefined;
+    try {
+      const embeddingService = getEmbeddingService();
+      embedding = await embeddingService.embed(addition.description);
+    } catch (err) {
+      console.warn(`[Dedup] Failed to compute embedding for new entity "${addition.name}"`, err);
+      continue;  // Skip this addition if embedding fails
+    }
+    
+    // Generate ID for new entity
+    const id = crypto.randomUUID();
+    
+    // Build complete entity
+    const newEntity = {
+      id,
+      type: entity_type,
+      name: addition.name,
+      description: addition.description,
+      sentiment: addition.sentiment ?? 0.0,
+      last_updated: new Date().toISOString(),
+      embedding,
+      // Type-specific fields with defaults
+      ...(entity_type === 'trait' && { strength: addition.strength ?? 0.5 }),
+      ...(entity_type === 'fact' && { 
+        confidence: addition.confidence ?? 0.5,
+        validated: 'unknown' as import("../types/enums.js").ValidationLevel,
+        validated_date: ''
+      }),
+      ...((entity_type === 'topic' || entity_type === 'person') && {
+        exposure_current: addition.exposure_current ?? 0.0,
+        exposure_desired: addition.exposure_desired ?? 0.5,
+        last_ei_asked: null
+      }),
+      ...(entity_type === 'person' && { relationship: addition.relationship ?? 'Unknown' }),
+      ...(entity_type === 'topic' && { category: addition.category ?? 'Interest' }),
+    };
+    
+    // Type-safe cast based on entity_type
+    if (entity_type === 'fact') {
+      stateManager.human_fact_upsert(newEntity as Fact);
+    } else if (entity_type === 'trait') {
+      stateManager.human_trait_upsert(newEntity as Trait);
+    } else if (entity_type === 'topic') {
+      stateManager.human_topic_upsert(newEntity as Topic);
+    } else if (entity_type === 'person') {
+      stateManager.human_person_upsert(newEntity as Person);
+    }
+    console.log(`[Dedup] Added new ${entity_type} "${addition.name}"`);
+  }
+}

package/src/core/handlers/heartbeat.ts

@@ -19,16 +19,17 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
 
   const result = response.parsed as HeartbeatCheckResult | undefined;
   if (!result) {
-    console.error("[handleHeartbeatCheck] No parsed result");
+    console.error(`[HeartbeatCheck ${personaDisplayName}] No parsed result`);
     return;
   }
+  console.log(`[HeartbeatCheck ${personaDisplayName}] Parsed result - should_respond: ${result.should_respond}, topic: ${result.topic ?? '(none)'}, message: ${result.message ? '(present)' : '(none)'}`);
 
   const now = new Date().toISOString();
   state.persona_update(personaId, { last_heartbeat: now });
   state.queue_clearPersonaResponses(personaId, LLMNextStep.HandleHeartbeatCheck);
 
   if (!result.should_respond) {
-    console.log(`[handleHeartbeatCheck] ${personaDisplayName} chose not to reach out`);
+    console.log(`[HeartbeatCheck ${personaDisplayName}] Chose not to reach out (should_respond=false)`);
     return;
   }
 
@@ -42,21 +43,24 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
       context_status: ContextStatus.Default,
     };
     state.messages_append(personaId, message);
-    console.log(`[handleHeartbeatCheck] ${personaDisplayName} proactively messaged about: ${result.topic ?? "general"}`);
+    console.log(`[HeartbeatCheck ${personaDisplayName}] Added proactive message - topic: ${result.topic ?? 'general'}, message: "${result.message.substring(0, 100)}${result.message.length > 100 ? '...' : ''}"`);
+  } else {
+    console.log(`[HeartbeatCheck ${personaDisplayName}] should_respond=true but no message provided`);
   }
 }
 
 export function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
   const result = response.parsed as EiHeartbeatResult | undefined;
   if (!result) {
-    console.error("[handleEiHeartbeat] No parsed result");
+    console.error("[EiHeartbeat] No parsed result");
     return;
   }
+  console.log(`[EiHeartbeat] Parsed result - should_respond: ${result.should_respond}, id: ${result.id ?? '(none)'}, my_response: ${result.my_response ? '(present)' : '(none)'}`);
   const now = new Date().toISOString();
   state.persona_update("ei", { last_heartbeat: now });
   state.queue_clearPersonaResponses("ei", LLMNextStep.HandleEiHeartbeat);
   if (!result.should_respond || !result.id) {
-    console.log("[handleEiHeartbeat] Ei chose not to reach out");
+    console.log("[EiHeartbeat] Chose not to reach out (should_respond=false or no id)");
     return;
   }
   const isTUI = response.request.data.isTUI as boolean;
@@ -84,7 +88,10 @@ export function handleEiHeartbeat(response: LLMResponse, state: StateManager): v
     return;
   }
 
-  if (result.my_response) sendMessage(result.my_response);
+  if (result.my_response) {
+    console.log(`[EiHeartbeat] Sending message: "${result.my_response.substring(0, 100)}${result.my_response.length > 100 ? '...' : ''}"`);
+    sendMessage(result.my_response);
+  }
 
   switch (found.type) {
     case "person":

package/src/core/handlers/index.ts

@@ -17,6 +17,7 @@ import {
 import { handleHumanFactScan, handleHumanTraitScan, handleHumanTopicScan, handleHumanPersonScan } from "./human-extraction.js";
 import { handleHumanItemMatch, handleHumanItemUpdate } from "./human-matching.js";
 import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
+import { handleDedupCurate } from "./dedup.js";
 
 export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handlePersonaResponse,
@@ -41,4 +42,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handleToolContinuation,
   handleRewriteScan,
   handleRewriteRewrite,
+  handleDedupCurate,
 };

package/src/core/handlers/persona-response.ts

@@ -1,10 +1,12 @@
 import {
   ContextStatus,
+  LLMNextStep,
   type LLMResponse,
   type Message,
 } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import type { PersonaResponseResult } from "../../prompts/response/index.js";
+import { handlers } from "./index.js";
 
 export type ResponseHandler = (response: LLMResponse, state: StateManager) => void | Promise<void>;
 
@@ -87,11 +89,35 @@ export function handlePersonaResponse(response: LLMResponse, state: StateManager
 /**
  * handleToolContinuation — second LLM call in the tool flow (may loop if LLM calls more tools).
  * The QueueProcessor already injected tool history into messages and got the
- * final persona response. Parse and store it exactly like handlePersonaResponse.
+ * final persona response. Route to the original handler based on originalNextStep in data.
  */
 export function handleToolContinuation(response: LLMResponse, state: StateManager): void {
-  console.log(`[handleToolContinuation] Routing to handlePersonaResponse`);
-  handlePersonaResponse(response, state);
+  const originalStep = response.request.data.originalNextStep as LLMNextStep | undefined;
+  
+  if (!originalStep) {
+    console.error(`[handleToolContinuation] No originalNextStep in data, falling back to handlePersonaResponse`);
+    handlePersonaResponse(response, state);
+    return;
+  }
+  
+  console.log(`[handleToolContinuation] Original request was ${originalStep}, routing accordingly`);
+  
+  const handler = handlers[originalStep];
+  
+  if (!handler) {
+    console.error(`[handleToolContinuation] No handler found for ${originalStep}, falling back to handlePersonaResponse`);
+    handlePersonaResponse(response, state);
+    return;
+  }
+  
+  // Avoid infinite loop - if original was already HandleToolContinuation, go to PersonaResponse
+  if (originalStep === "handleToolContinuation") {
+    console.log(`[handleToolContinuation] Original was tool continuation, routing to handlePersonaResponse`);
+    handlePersonaResponse(response, state);
+    return;
+  }
+  
+  handler(response, state);
 }
 
 export function handleOneShot(_response: LLMResponse, _state: StateManager): void {

package/src/core/heartbeat-manager.ts

@@ -188,6 +188,8 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
   const persona = sm.persona_getById(personaId);
   if (!persona) return;
   sm.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
+  const model = getModelForPersona(sm, personaId);
+  console.log(`[HeartbeatCheck ${persona.display_name}] Queueing heartbeat check (model: ${model})`);
   const human = sm.getHuman();
   const history = sm.messages_get(personaId);
   const contextWindowHours = persona.context_window_hours ?? DEFAULT_CONTEXT_WINDOW_HOURS;
@@ -228,6 +230,7 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
   };
 
   const prompt = buildHeartbeatCheckPrompt(promptData);
+  console.log(`[HeartbeatCheck ${persona.display_name}] Prompt data - topics: ${promptData.human.topics.length}, people: ${promptData.human.people.length}, inactive_days: ${inactiveDays}`);
 
   sm.queue_enqueue({
     type: LLMRequestType.JSON,
@@ -238,4 +241,5 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
     model: getModelForPersona(sm, personaId),
     data: { personaId, personaDisplayName: persona.display_name },
   });
+  console.log(`[HeartbeatCheck ${persona.display_name}] Request queued`);
 }

package/src/core/llm-client.ts

@@ -189,6 +189,7 @@ export async function callLLMRaw(
     model,
     messages: finalMessages,
     temperature,
+    max_tokens: 64000,  // Opus 4: 128K max output, 200K total context. Local models clamp to their config. Prevents runaway generation.
   };
 
   if (options.tools && options.tools.length > 0) {

package/src/core/orchestrators/ceremony.ts

@@ -10,6 +10,7 @@ import {
   type ExtractionOptions,
 } from "./human-extraction.js";
 import { queuePersonaTopicScan, type PersonaTopicContext } from "./persona-topics.js";
+import { queueDedupPhase } from "./dedup-phase.js";
 import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt, buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
 
 export function isNewDay(lastCeremony: string | undefined, now: Date): boolean {
@@ -69,40 +70,19 @@ export function startCeremony(state: StateManager): void {
     },
   });
   
-  const personas = state.persona_getAll();
-  const activePersonas = personas.filter(p => 
-    !p.is_paused && 
-    !p.is_archived && 
-    !p.is_static
-  );
-  
-  const lastCeremony = human.settings?.ceremony?.last_ceremony 
-    ? new Date(human.settings.ceremony.last_ceremony).getTime() 
-    : 0;
+  // PHASE 1: Deduplication (runs BEFORE Expose)
+  console.log("[ceremony] Starting Phase 1: Deduplication");
+  queueDedupPhase(state);
   
-  const personasWithActivity = activePersonas.filter(p => {
-    const lastActivity = p.last_activity ? new Date(p.last_activity).getTime() : 0;
-    return lastActivity > lastCeremony;
-  });
-  
-  console.log(`[ceremony] Processing ${personasWithActivity.length} personas with activity (of ${activePersonas.length} active)`);
-  
-  const options: ExtractionOptions = { ceremony_progress: true };
-  
-  for (let i = 0; i < personasWithActivity.length; i++) {
-    const persona = personasWithActivity[i];
-    const isLast = i === personasWithActivity.length - 1;
-    
-    console.log(`[ceremony] Queuing exposure for ${persona.display_name} (${i + 1}/${personasWithActivity.length})${isLast ? " (last)" : ""}`);
-    queueExposurePhase(persona.id, state, options);
+  // Check if dedup work was queued
+  if (!state.queue_hasPendingCeremonies()) {
+    // No dedup work found → immediately advance to Expose phase
+    console.log("[ceremony] No dedup work, advancing to Expose phase");
+    handleCeremonyProgress(state, 1);
   }
   
   const duration = Date.now() - startTime;
-  console.log(`[ceremony] Exposure phase queued in ${duration}ms`);
-  
-  // Check immediately — if zero messages were queued (no unextracted messages for any persona),
-  // this will see an empty queue and proceed directly to Decay → Expire.
-  handleCeremonyProgress(state);
+  console.log(`[ceremony] Dedup phase queued in ${duration}ms`);
 }
 
 /**
@@ -193,11 +173,43 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
  * If any ceremony_progress items remain in the queue, does nothing — more work pending.
  * If the queue is clear of ceremony items, advances to Decay → Prune → Expire.
  */
-export function handleCeremonyProgress(state: StateManager): void {
+export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
   if (state.queue_hasPendingCeremonies()) {
-    return; // Still processing exposure scans
+    return; // Still processing ceremony items
   }
   
+  if (lastPhase === 1) {
+    // Dedup phase complete → start Expose phase
+    console.log("[ceremony:progress] Dedup complete, starting Expose phase");
+    
+    const personas = state.persona_getAll();
+    const activePersonas = personas.filter(p => 
+      !p.is_paused && 
+      !p.is_archived && 
+      !p.is_static
+    );
+    
+    // Find personas with unprocessed messages (any message with p/r/o/f = false)
+    const personasWithUnprocessed = activePersonas.filter(p => {
+      const messages = state.messages_get(p.id);
+      return messages.some(msg => 
+        !msg.p || 
+        !msg.r || 
+        !msg.o || 
+        !msg.f
+      );
+    });
+    
+    console.log(`[ceremony:expose] Found ${activePersonas.length} active personas, ${personasWithUnprocessed.length} with unprocessed messages`);
+    
+    const options: ExtractionOptions = { ceremony_progress: 2 };
+    for (const persona of personasWithUnprocessed) {
+      queueExposurePhase(persona.id, state, options);
+    }
+    return;
+  }
+  
+  // Phase 2 (Expose) complete → advance to Decay/Prune/Expire/Explore
   console.log("[ceremony:progress] All exposure scans complete, advancing to Decay");
   
   const personas = state.persona_getAll();
@@ -215,7 +227,6 @@ export function handleCeremonyProgress(state: StateManager): void {
   if (eiIndex > -1) {
     activePersonas.splice(eiIndex, 1);
   }
-  
   // Decay phase: apply decay + prune for ALL active personas
   for (const persona of activePersonas) {
     applyDecayPhase(persona.id, state);