ei-tui 0.1.3 → 0.1.5

package/src/core/handlers/index.ts

@@ -1,16 +1,17 @@
-import { 
-  ContextStatus, 
-  LLMNextStep, 
+import {
+  ContextStatus,
+  LLMNextStep,
   ValidationLevel,
-  type LLMResponse, 
-  type Message, 
-  type Trait, 
+  type LLMResponse,
+  type Message,
+  type Trait,
   type Topic,
   type PersonaTopic,
   type Fact,
   type Person,
   type Quote,
   type DataItemType,
+  type DataItemBase,
 } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import type { HeartbeatCheckResult, EiHeartbeatResult } from "../../prompts/heartbeat/types.js";
@@ -22,7 +23,8 @@ import type {
   PersonaTopicMatchResult,
   PersonaTopicUpdateResult,
 } from "../../prompts/persona/types.js";
-import type { EiValidationResult } from "../../prompts/validation/types.js";
+import type { PersonaResponseResult } from "../../prompts/response/index.js";
+
 import type { 
   PersonaExpireResult, 
   PersonaExploreResult,
@@ -50,9 +52,10 @@ import type {
   ItemUpdateResult,
   ExposureImpact,
 } from "../../prompts/human/types.js";
-import { buildEiValidationPrompt } from "../../prompts/validation/index.js";
-import { LLMRequestType, LLMPriority, LLMNextStep as NextStep } from "../types.js";
+
+import { LLMRequestType, LLMPriority } from "../types.js";
 import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
+import { crossFind } from "../utils/index.js";
 
 export type ResponseHandler = (response: LLMResponse, state: StateManager) => void | Promise<void>;
 
@@ -104,20 +107,66 @@ function handlePersonaResponse(response: LLMResponse, state: StateManager): void
   // the messages were "seen" and processed
   state.messages_markPendingAsRead(personaId);
 
+  // Structured JSON path: queue-processor parsed valid JSON into `parsed`
+  if (response.parsed !== undefined) {
+    const result = response.parsed as PersonaResponseResult;
+
+    if (!result.should_respond) {
+      const reason = result.reason;
+      if (reason) {
+        console.log(`[handlePersonaResponse] ${personaDisplayName} chose silence: ${reason}`);
+        const silentMessage: Message = {
+          id: crypto.randomUUID(),
+          role: "system",
+          silence_reason: reason,
+          timestamp: new Date().toISOString(),
+          read: false,
+          context_status: ContextStatus.Never,
+        };
+        state.messages_append(personaId, silentMessage);
+      } else {
+        console.log(`[handlePersonaResponse] ${personaDisplayName} chose not to respond (no reason given)`);
+      }
+      return;
+    }
+
+    // Build message with structured fields
+    const verbal = result.verbal_response || undefined;
+    const action = result.action_response || undefined;
+
+    if (!verbal && !action) {
+      console.log(`[handlePersonaResponse] ${personaDisplayName} JSON had should_respond=true but no content fields`);
+      return;
+    }
+
+    const message: Message = {
+      id: crypto.randomUUID(),
+      role: "system",
+      verbal_response: verbal,
+      action_response: action,
+      timestamp: new Date().toISOString(),
+      read: false,
+      context_status: ContextStatus.Default,
+    };
+    state.messages_append(personaId, message);
+    console.log(`[handlePersonaResponse] Appended structured response to ${personaDisplayName}`);
+    return;
+  }
+
+  // Legacy plain-text fallback
   if (!response.content) {
-    console.log(`[handlePersonaResponse] No content in response (${personaDisplayName} chose not to respond)`);
+    console.log(`[handlePersonaResponse] ${personaDisplayName} chose not to respond (no reason given)`);
     return;
   }
 
   const message: Message = {
     id: crypto.randomUUID(),
     role: "system",
-    content: response.content,
+    verbal_response: response.content ?? undefined,
     timestamp: new Date().toISOString(),
     read: false,
     context_status: ContextStatus.Default,
   };
-
   state.messages_append(personaId, message);
   console.log(`[handlePersonaResponse] Appended response to ${personaDisplayName}`);
 }
@@ -148,7 +197,7 @@ function handleHeartbeatCheck(response: LLMResponse, state: StateManager): void
     const message: Message = {
       id: crypto.randomUUID(),
       role: "system",
-      content: result.message,
+      verbal_response: result.message,
       timestamp: now,
       read: false,
       context_status: ContextStatus.Default,
@@ -164,29 +213,52 @@ function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
     console.error("[handleEiHeartbeat] No parsed result");
     return;
   }
-
   const now = new Date().toISOString();
   state.persona_update("ei", { last_heartbeat: now });
-
-  if (!result.should_respond) {
+  if (!result.should_respond || !result.id) {
     console.log("[handleEiHeartbeat] Ei chose not to reach out");
     return;
   }
+  const isTUI = response.request.data.isTUI as boolean;
+  const found = crossFind(result.id, state.getHuman(), state.persona_getAll());
+  if (!found) {
+    console.warn(`[handleEiHeartbeat] Could not find item with id "${result.id}"`);
+    return;
+  }
 
-  if (result.message) {
-    const message: Message = {
-      id: crypto.randomUUID(),
-      role: "system",
-      content: result.message,
-      timestamp: now,
-      read: false,
-      context_status: ContextStatus.Default,
-    };
-    state.messages_append("ei", message);
-    console.log("[handleEiHeartbeat] Ei proactively messaged");
-    if (result.priorities) {
-      console.log("[handleEiHeartbeat] Priorities:", result.priorities.map(p => p.name).join(", "));
-    }
+  const sendMessage = (verbal_response: string) => state.messages_append("ei", {
+    id: crypto.randomUUID(),
+    role: "system",
+    verbal_response,
+    timestamp: now,
+    read: false,
+    context_status: ContextStatus.Default,
+  });
+
+  if (found.type === "fact") {
+    const factsNav = isTUI ? "using /me facts" : "using \u2630 \u2192 My Data";
+    sendMessage(`Another persona updated a fact called "${found.name}" to "${found.description}". If that's right, you can lock it from further changes by ${factsNav}.`);
+    state.human_fact_upsert({ ...found, validated: ValidationLevel.Ei, validated_date: now });
+    console.log(`[handleEiHeartbeat] Notified about fact "${found.name}"`);
+    return;
+  }
+
+  if (result.my_response) sendMessage(result.my_response);
+
+  switch (found.type) {
+    case "person":
+      state.human_person_upsert({ ...found, last_ei_asked: now });
+      console.log(`[handleEiHeartbeat] Reached out about person "${found.name}"`);
+      break;
+    case "topic":
+      state.human_topic_upsert({ ...found, last_ei_asked: now });
+      console.log(`[handleEiHeartbeat] Reached out about topic "${found.name}"`);
+      break;
+    case "persona":
+      console.log(`[handleEiHeartbeat] Reached out about persona "${found.display_name}"`);
+      break;
+    default:
+      console.warn(`[handleEiHeartbeat] Unexpected item type "${found.type}" for id "${result.id}"`);
   }
 }
 
@@ -291,59 +363,7 @@ function handlePersonaTraitExtraction(response: LLMResponse, state: StateManager
   console.log(`[handlePersonaTraitExtraction] Updated ${traits.length} traits for ${personaDisplayName}`);
 }
 
-function handleEiValidation(response: LLMResponse, state: StateManager): void {
-  const validationId = response.request.data.validationId as string;
-  const dataType = response.request.data.dataType as string;
-  const itemName = response.request.data.itemName as string;
-
-  const result = response.parsed as EiValidationResult | undefined;
-  if (!result) {
-    console.error("[handleEiValidation] No parsed result");
-    return;
-  }
-
-  console.log(`[handleEiValidation] Decision for ${dataType} "${itemName}": ${result.decision} - ${result.reason}`);
-
-  if (result.decision === "reject") {
-    if (validationId) {
-      state.queue_clearValidations([validationId]);
-    }
-    return;
-  }
 
-  const itemToApply = result.decision === "modify" && result.modified_item
-    ? result.modified_item
-    : response.request.data.proposedItem;
-
-  if (itemToApply && dataType) {
-    const now = new Date().toISOString();
-    const item = { ...itemToApply, last_updated: now };
-
-    switch (dataType) {
-      case "fact":
-        state.human_fact_upsert({
-          ...item,
-          validated: ValidationLevel.Ei,
-          validated_date: now,
-        } as Fact);
-        break;
-      case "trait":
-        state.human_trait_upsert(item as Trait);
-        break;
-      case "topic":
-        state.human_topic_upsert(item as Topic);
-        break;
-      case "person":
-        state.human_person_upsert(item as Person);
-        break;
-    }
-    console.log(`[handleEiValidation] Applied ${result.decision} for ${dataType} "${itemName}"`);
-  }
-
-  if (validationId) {
-    state.queue_clearValidations([validationId]);
-  }
-}
 
 function handleOneShot(_response: LLMResponse, _state: StateManager): void {
   // One-shot is handled specially in Processor to fire onOneShotReturned
@@ -573,43 +593,48 @@ function handleHumanItemMatch(response: LLMResponse, state: StateManager): void
     console.error("[handleHumanItemMatch] No parsed result");
     return;
   }
-  // "new" isn't a valid guid and is used as a marker for LLMs to signify "no match" - update for processing
-  if (result?.matched_guid === "new") {
-    result.matched_guid = null;
-  }
 
   const candidateType = response.request.data.candidateType as DataItemType;
-  const itemName = response.request.data.itemName as string;
-  const itemValue = response.request.data.itemValue as string;
   const personaId = response.request.data.personaId as string;
   const personaDisplayName = response.request.data.personaDisplayName as string;
   const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
-  
   const allMessages = state.messages_get(personaId);
   const { messages_context, messages_analyze } = splitMessagesByTimestamp(allMessages, analyzeFrom);
-
-  if (result.matched_guid) {
-    const human = state.getHuman();
-    const matchedFact = human.facts.find(f => f.id === result.matched_guid);
-    if (matchedFact?.validated === ValidationLevel.Human) {
-      console.log(`[handleHumanItemMatch] Skipping locked fact "${matchedFact.name}" (human-validated)`);
-      return;
-    }
-  }
-
   const context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string } = {
     personaId,
     personaDisplayName,
     messages_context,
     messages_analyze,
-    itemName,
-    itemValue,
-    itemCategory: candidateType === "topic" ? itemValue : undefined,
+    itemName: response.request.data.itemName as string,
+    itemValue: response.request.data.itemValue as string,
+    itemCategory: response.request.data.itemCategory as string | undefined,
   };
 
-  queueItemUpdate(candidateType, result, context, state);
-  const matched = result.matched_guid ? `matched GUID "${result.matched_guid}"` : "no match (new item)";
-  console.log(`[handleHumanItemMatch] ${candidateType} "${itemName}": ${matched}`);
+  let resolvedType: DataItemType = candidateType;
+  let matched_guid = result.matched_guid;
+  if (matched_guid === "new") {
+    matched_guid = null;
+  } else if (matched_guid) {
+    const found = crossFind(matched_guid, state.getHuman());
+    if (!found) {
+      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" not found in human data — treating as new item`);
+      matched_guid = null;
+    } else if (found.type === "fact" && found.validated === ValidationLevel.Human) {
+      console.log(`[handleHumanItemMatch] Skipping locked fact "${found.name}" (human-validated)`);
+      return;
+    } else if (!(found.type === "fact" || found.type === "trait" || found.type === "topic" || found.type === "person")) {
+      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" resolved to non-human type "${found.type}" - Ignoring`);
+      return;
+    } else {
+      resolvedType = found.type;
+      context.itemName = found.name || context.itemName;
+      context.itemValue = found.description || context.itemValue;
+    }
+  }
+  result.matched_guid = matched_guid;
+  queueItemUpdate(resolvedType, result, context, state);
+  const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new item)";
+  console.log(`[handleHumanItemMatch] ${resolvedType} "${context.itemName}": ${matched}`);
 }
 
 async function handleHumanItemUpdate(response: LLMResponse, state: StateManager): Promise<void> {
@@ -632,7 +657,15 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
   }
 
   const now = new Date().toISOString();
-  const itemId = isNewItem ? crypto.randomUUID() : (existingItemId ?? crypto.randomUUID());
+  const resolveItemId = (): string => {
+    if (isNewItem || !existingItemId) return crypto.randomUUID();
+    const h = state.getHuman();
+    const arr = candidateType === "fact" ? h.facts : candidateType === "trait" ? h.traits : candidateType === "topic" ? h.topics : h.people;
+    // Guard: if existingItemId isn't in the correct type array, treat as new
+    // (prevents cross-type ID reuse when LLM matches against a different type's UUID)
+    return arr.find((x: DataItemBase) => x.id === existingItemId) ? existingItemId : crypto.randomUUID();
+  };
+  const itemId = resolveItemId();
 
   const persona = state.persona_getById(personaId);
   const personaGroup = persona?.group_primary ?? null;
@@ -745,6 +778,18 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
   console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
 }
 
+/**
+ * Returns the combined display text of a message for quote indexing.
+ * Mirrors the rendering logic used in the frontends.
+ */
+function getMessageText(message: Message): string {
+  const parts: string[] = [];
+  if (message.action_response) parts.push(`_${message.action_response}_`);
+  if (message.verbal_response) parts.push(message.verbal_response);
+  return parts.join('\n\n');
+}
+
+
 async function validateAndStoreQuotes(
   candidates: Array<{ text: string; reason: string }> | undefined,
   messages: Message[],
@@ -758,22 +803,53 @@ async function validateAndStoreQuotes(
   for (const candidate of candidates) {
     let found = false;
     for (const message of messages) {
-      const start = message.content.indexOf(candidate.text);
+      const msgText = getMessageText(message);
+      const start = msgText.indexOf(candidate.text);
       if (start !== -1) {
         const end = start + candidate.text.length;
         
+        // Check for ANY overlapping quote in this message (not just exact match)
         const existing = state.human_quote_getForMessage(message.id);
-        const existingQuote = existing.find(q => q.start === start && q.end === end);
+        const overlapping = existing.find(q =>
+          q.start !== null && q.end !== null &&
+          start < q.end && end > q.start  // ranges overlap
+        );
         
-        if (existingQuote) {
-          if (!existingQuote.data_item_ids.includes(dataItemId)) {
-            state.human_quote_update(existingQuote.id, {
-              data_item_ids: [...existingQuote.data_item_ids, dataItemId],
-            });
-            console.log(`[extraction] Linked existing quote to "${dataItemId}": "${candidate.text.slice(0, 30)}..."`);
-          } else {
-            console.log(`[extraction] Quote already linked to "${dataItemId}": "${candidate.text.slice(0, 30)}..."`);
+        if (overlapping) {
+          // Merge: expand to the union of both ranges
+          const mergedStart = Math.min(start, overlapping.start!);
+          const mergedEnd = Math.max(end, overlapping.end!);
+          const mergedText = msgText.slice(mergedStart, mergedEnd);
+          
+          // Merge data_item_ids and persona_groups (deduplicated)
+          const mergedDataItemIds = overlapping.data_item_ids.includes(dataItemId)
+            ? overlapping.data_item_ids
+            : [...overlapping.data_item_ids, dataItemId];
+          const group = personaGroup || "General";
+          const mergedGroups = overlapping.persona_groups.includes(group)
+            ? overlapping.persona_groups
+            : [...overlapping.persona_groups, group];
+          
+          // Only recompute embedding if the text actually changed
+          let embedding = overlapping.embedding;
+          if (mergedText !== overlapping.text) {
+            try {
+              const embeddingService = getEmbeddingService();
+              embedding = await embeddingService.embed(mergedText);
+            } catch (err) {
+              console.warn(`[extraction] Failed to recompute embedding for merged quote: "${mergedText.slice(0, 30)}..."`, err);
+            }
           }
+          
+          state.human_quote_update(overlapping.id, {
+            start: mergedStart,
+            end: mergedEnd,
+            text: mergedText,
+            data_item_ids: mergedDataItemIds,
+            persona_groups: mergedGroups,
+            embedding,
+          });
+          console.log(`[extraction] Merged overlapping quote: "${mergedText.slice(0, 50)}..." (${mergedStart}-${mergedEnd})`);
           found = true;
           break;
         }
@@ -826,23 +902,16 @@ function applyOrValidate(
   state: StateManager,
   dataType: DataItemType,
   item: Fact | Trait | Topic | Person,
-  personaName: string,
-  isEi: boolean,
-  personaGroup: string | null
+  _personaName: string,
+  _isEi: boolean,
+  _personaGroup: string | null
 ): void {
-  const isGeneralGroup = !personaGroup || personaGroup.toLowerCase() === "general";
-  const needsValidation = !isEi && isGeneralGroup;
-
   switch (dataType) {
     case "fact": state.human_fact_upsert(item as Fact); break;
     case "trait": state.human_trait_upsert(item as Trait); break;
     case "topic": state.human_topic_upsert(item as Topic); break;
     case "person": state.human_person_upsert(item as Person); break;
   }
-
-  if (needsValidation) {
-    queueEiValidation(state, dataType, item, personaName);
-  }
 }
 
 const MIN_MESSAGE_COUNT_FOR_CREATE = 2;
@@ -986,52 +1055,7 @@ function handlePersonaTopicUpdate(response: LLMResponse, state: StateManager): v
   }
 }
 
-function queueEiValidation(
-  state: StateManager,
-  dataType: DataItemType,
-  item: Fact | Trait | Topic | Person,
-  sourcePersona: string
-): void {
-  const human = state.getHuman();
-  let existingItem: Fact | Trait | Topic | Person | undefined;
 
-  switch (dataType) {
-    case "fact": existingItem = human.facts.find(f => f.id === item.id); break;
-    case "trait": existingItem = human.traits.find(t => t.id === item.id); break;
-    case "topic": existingItem = human.topics.find(t => t.id === item.id); break;
-    case "person": existingItem = human.people.find(p => p.id === item.id); break;
-  }
-
-  const prompt = buildEiValidationPrompt({
-    validation_type: "cross_persona",
-    item_name: item.name,
-    data_type: dataType,
-    context: `Learned from conversation with ${sourcePersona}`,
-    source_persona: sourcePersona,
-    current_item: existingItem,
-    proposed_item: item,
-  });
-
-  // Cross-persona validation is always normal priority
-  const priority = LLMPriority.Normal;
-
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: NextStep.HandleEiValidation,
-    data: {
-      validationId: crypto.randomUUID(),
-      dataType,
-      itemName: item.name,
-      proposedItem: item,
-      sourcePersona,
-    },
-  });
-
-  console.log(`[queueEiValidation] Queued ${dataType} "${item.name}" from ${sourcePersona} for Ei validation`);
-}
 
 export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handlePersonaResponse,
@@ -1049,7 +1073,6 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handlePersonaTopicUpdate,
   handleHeartbeatCheck,
   handleEiHeartbeat,
-  handleEiValidation,
   handleOneShot,
   handlePersonaExpire,
   handlePersonaExplore,

package/src/core/orchestrators/ceremony.ts

@@ -1,6 +1,6 @@
-import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic } from "../types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message } from "../types.js";
 import type { StateManager } from "../state-manager.js";
-import { applyDecayToValue } from "../utils/decay.js";
+import { applyDecayToValue } from "../utils/index.js";
 import {
   queueFactScan,
   queueTraitScan,
@@ -384,12 +384,12 @@ export function queueExplorePhase(personaId: string, state: StateManager): void
   });
 }
 
-function extractConversationThemes(messages: { content: string; role: string }[]): string[] {
+function extractConversationThemes(messages: Message[]): string[] {
   const humanMessages = messages.filter(m => m.role === "human");
   if (humanMessages.length === 0) return [];
   
   const words = humanMessages
-    .map(m => m.content.toLowerCase())
+    .map(m => (m.verbal_response ?? '').toLowerCase())
     .join(" ")
     .split(/\s+/)
     .filter(w => w.length > 4);

package/src/core/orchestrators/extraction-chunker.ts

@@ -12,7 +12,7 @@ function estimateTokens(text: string): number {
 }
 
 function estimateMessageTokens(messages: Message[]): number {
-  return messages.reduce((sum, msg) => sum + estimateTokens(msg.content) + 4, 0);
+  return messages.reduce((sum, msg) => sum + estimateTokens(msg.verbal_response ?? '') + 4, 0);
 }
 
 function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
@@ -20,7 +20,7 @@ function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
   let tokens = 0;
 
   for (let i = messages.length - 1; i >= 0; i--) {
-    const msgTokens = estimateTokens(messages[i].content) + 4;
+    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
     if (tokens + msgTokens > maxTokens) break;
     result.unshift(messages[i]);
     tokens += msgTokens;
@@ -39,7 +39,7 @@ function pullMessagesFromStart(
   let i = startIndex;
 
   while (i < messages.length) {
-    const msgTokens = estimateTokens(messages[i].content) + 4;
+    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
     if (tokens + msgTokens > maxTokens && pulled.length > 0) break;
     pulled.push(messages[i]);
     tokens += msgTokens;