ei-tui 0.1.19 → 0.1.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/context-utils.ts

@@ -0,0 +1,57 @@
+import type { Message, HumanEntity, Quote, DataItemBase } from "./types.js";
+import { ContextStatus as ContextStatusEnum } from "./types.js";
+
+// =============================================================================
+// CONTEXT FILTERING
+// =============================================================================
+
+export function filterMessagesForContext(
+  messages: Message[],
+  contextBoundary: string | undefined,
+  contextWindowHours: number
+): Message[] {
+  if (messages.length === 0) return [];
+
+  const now = Date.now();
+  const windowStartMs = now - contextWindowHours * 60 * 60 * 1000;
+  const boundaryMs = contextBoundary ? new Date(contextBoundary).getTime() : 0;
+
+  return messages.filter((msg) => {
+    if (msg.context_status === ContextStatusEnum.Always) return true;
+    if (msg.context_status === ContextStatusEnum.Never) return false;
+
+    const msgMs = new Date(msg.timestamp).getTime();
+
+    if (contextBoundary) {
+      return msgMs >= boundaryMs;
+    }
+
+    return msgMs >= windowStartMs;
+  });
+}
+
+// =============================================================================
+// EMBEDDING STRIPPING - Remove embeddings from data items before returning to FE
+// Embeddings are internal implementation details for similarity search.
+// =============================================================================
+
+export function stripDataItemEmbedding<T extends DataItemBase>(item: T): T {
+  const { embedding, ...rest } = item;
+  return rest as T;
+}
+
+export function stripQuoteEmbedding(quote: Quote): Quote {
+  const { embedding, ...rest } = quote;
+  return rest;
+}
+
+export function stripHumanEmbeddings(human: HumanEntity): HumanEntity {
+  return {
+    ...human,
+    facts: (human.facts ?? []).map(stripDataItemEmbedding),
+    traits: (human.traits ?? []).map(stripDataItemEmbedding),
+    topics: (human.topics ?? []).map(stripDataItemEmbedding),
+    people: (human.people ?? []).map(stripDataItemEmbedding),
+    quotes: (human.quotes ?? []).map(stripQuoteEmbedding),
+  };
+}

package/src/core/handlers/heartbeat.ts

@@ -0,0 +1,101 @@
+import {
+  ContextStatus,
+  ValidationLevel,
+  type LLMResponse,
+  type Message,
+} from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import type { HeartbeatCheckResult, EiHeartbeatResult } from "../../prompts/heartbeat/types.js";
+import { crossFind } from "../utils/index.js";
+
+export function handleHeartbeatCheck(response: LLMResponse, state: StateManager): void {
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+  if (!personaId) {
+    console.error("[handleHeartbeatCheck] No personaId in request data");
+    return;
+  }
+
+  const result = response.parsed as HeartbeatCheckResult | undefined;
+  if (!result) {
+    console.error("[handleHeartbeatCheck] No parsed result");
+    return;
+  }
+
+  const now = new Date().toISOString();
+  state.persona_update(personaId, { last_heartbeat: now });
+
+  if (!result.should_respond) {
+    console.log(`[handleHeartbeatCheck] ${personaDisplayName} chose not to reach out`);
+    return;
+  }
+
+  if (result.message) {
+    const message: Message = {
+      id: crypto.randomUUID(),
+      role: "system",
+      verbal_response: result.message,
+      timestamp: now,
+      read: false,
+      context_status: ContextStatus.Default,
+    };
+    state.messages_append(personaId, message);
+    console.log(`[handleHeartbeatCheck] ${personaDisplayName} proactively messaged about: ${result.topic ?? "general"}`);
+  }
+}
+
+export function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
+  const result = response.parsed as EiHeartbeatResult | undefined;
+  if (!result) {
+    console.error("[handleEiHeartbeat] No parsed result");
+    return;
+  }
+  const now = new Date().toISOString();
+  state.persona_update("ei", { last_heartbeat: now });
+  if (!result.should_respond || !result.id) {
+    console.log("[handleEiHeartbeat] Ei chose not to reach out");
+    return;
+  }
+  const isTUI = response.request.data.isTUI as boolean;
+  const found = crossFind(result.id, state.getHuman(), state.persona_getAll());
+  if (!found) {
+    console.warn(`[handleEiHeartbeat] Could not find item with id "${result.id}"`);
+    return;
+  }
+
+  const sendMessage = (verbal_response: string) => state.messages_append("ei", {
+    id: crypto.randomUUID(),
+    role: "system",
+    verbal_response,
+    timestamp: now,
+    read: false,
+    context_status: ContextStatus.Default,
+    f: true, r: true, p: true, o: true,
+  });
+
+  if (found.type === "fact") {
+    const factsNav = isTUI ? "using /me facts" : "using \u2630 \u2192 My Data";
+    sendMessage(`Another persona updated a fact called "${found.name}" to "${found.description}". If that's right, you can lock it from further changes by ${factsNav}.`);
+    state.human_fact_upsert({ ...found, validated: ValidationLevel.Ei, validated_date: now });
+    console.log(`[handleEiHeartbeat] Notified about fact "${found.name}"`);
+    return;
+  }
+
+  if (result.my_response) sendMessage(result.my_response);
+
+  switch (found.type) {
+    case "person":
+      state.human_person_upsert({ ...found, last_ei_asked: now });
+      console.log(`[handleEiHeartbeat] Reached out about person "${found.name}"`);
+      break;
+    case "topic":
+      state.human_topic_upsert({ ...found, last_ei_asked: now });
+      console.log(`[handleEiHeartbeat] Reached out about topic "${found.name}"`);
+      break;
+    case "persona":
+      console.log(`[handleEiHeartbeat] Reached out about persona "${found.display_name}"`);
+      break;
+    default:
+      console.warn(`[handleEiHeartbeat] Unexpected item type "${found.type}" for id "${result.id}"`);
+  }
+}

package/src/core/handlers/human-extraction.ts

@@ -0,0 +1,87 @@
+import type { LLMResponse } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import type {
+  FactScanResult,
+  TraitScanResult,
+  TopicScanResult,
+  PersonScanResult,
+} from "../../prompts/human/types.js";
+import { queueItemMatch, type ExtractionContext } from "../orchestrators/index.js";
+import { markMessagesExtracted } from "./utils.js";
+
+export async function handleHumanFactScan(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as FactScanResult | undefined;
+  
+  // Mark messages as scanned regardless of whether facts were found
+  markMessagesExtracted(response, state, "f");
+  
+  if (!result?.facts || !Array.isArray(result.facts)) {
+    console.log("[handleHumanFactScan] No facts detected or invalid result");
+    return;
+  }
+
+  const context = response.request.data as unknown as ExtractionContext;
+  if (!context?.personaId) return;
+
+  for (const candidate of result.facts) {
+    await queueItemMatch("fact", candidate, context, state);
+  }
+  console.log(`[handleHumanFactScan] Queued ${result.facts.length} fact(s) for matching`);
+}
+
+export async function handleHumanTraitScan(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as TraitScanResult | undefined;
+  
+  markMessagesExtracted(response, state, "r");
+  
+  if (!result?.traits || !Array.isArray(result.traits)) {
+    console.log("[handleHumanTraitScan] No traits detected or invalid result");
+    return;
+  }
+
+  const context = response.request.data as unknown as ExtractionContext;
+  if (!context?.personaId) return;
+
+  for (const candidate of result.traits) {
+    await queueItemMatch("trait", candidate, context, state);
+  }
+  console.log(`[handleHumanTraitScan] Queued ${result.traits.length} trait(s) for matching`);
+}
+
+export async function handleHumanTopicScan(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as TopicScanResult | undefined;
+  
+  markMessagesExtracted(response, state, "p");
+  
+  if (!result?.topics || !Array.isArray(result.topics)) {
+    console.log("[handleHumanTopicScan] No topics detected or invalid result");
+    return;
+  }
+
+  const context = response.request.data as unknown as ExtractionContext;
+  if (!context?.personaId) return;
+
+  for (const candidate of result.topics) {
+    await queueItemMatch("topic", candidate, context, state);
+  }
+  console.log(`[handleHumanTopicScan] Queued ${result.topics.length} topic(s) for matching`);
+}
+
+export async function handleHumanPersonScan(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as PersonScanResult | undefined;
+  
+  markMessagesExtracted(response, state, "o");
+  
+  if (!result?.people || !Array.isArray(result.people)) {
+    console.log("[handleHumanPersonScan] No people detected or invalid result");
+    return;
+  }
+
+  const context = response.request.data as unknown as ExtractionContext;
+  if (!context?.personaId) return;
+
+  for (const candidate of result.people) {
+    await queueItemMatch("person", candidate, context, state);
+  }
+  console.log(`[handleHumanPersonScan] Queued ${result.people.length} person(s) for matching`);
+}

package/src/core/handlers/human-matching.ts

@@ -0,0 +1,337 @@
+import {
+  ValidationLevel,
+  type LLMResponse,
+  type Message,
+  type Trait,
+  type Topic,
+  type Fact,
+  type Person,
+  type Quote,
+  type DataItemType,
+  type DataItemBase,
+} from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import type { ItemMatchResult, ItemUpdateResult, ExposureImpact } from "../../prompts/human/types.js";
+import { queueItemUpdate, type ExtractionContext } from "../orchestrators/index.js";
+import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
+import { crossFind } from "../utils/index.js";
+import { splitMessagesByTimestamp, getMessageText } from "./utils.js";
+
+export function handleHumanItemMatch(response: LLMResponse, state: StateManager): void {
+  const result = response.parsed as ItemMatchResult | undefined;
+  if (!result) {
+    console.error("[handleHumanItemMatch] No parsed result");
+    return;
+  }
+
+  const candidateType = response.request.data.candidateType as DataItemType;
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+  const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
+  const allMessages = state.messages_get(personaId);
+  const { messages_context, messages_analyze } = splitMessagesByTimestamp(allMessages, analyzeFrom);
+  const context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string } = {
+    personaId,
+    personaDisplayName,
+    messages_context,
+    messages_analyze,
+    itemName: response.request.data.itemName as string,
+    itemValue: response.request.data.itemValue as string,
+    itemCategory: response.request.data.itemCategory as string | undefined,
+  };
+
+  let resolvedType: DataItemType = candidateType;
+  let matched_guid = result.matched_guid;
+  if (matched_guid === "new") {
+    matched_guid = null;
+  } else if (matched_guid) {
+    const found = crossFind(matched_guid, state.getHuman());
+    if (!found) {
+      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" not found in human data — treating as new item`);
+      matched_guid = null;
+    } else if (found.type === "fact" && found.validated === ValidationLevel.Human) {
+      console.log(`[handleHumanItemMatch] Skipping locked fact "${found.name}" (human-validated)`);
+      return;
+    } else if (!(found.type === "fact" || found.type === "trait" || found.type === "topic" || found.type === "person")) {
+      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" resolved to non-human type "${found.type}" - Ignoring`);
+      return;
+    } else {
+      resolvedType = found.type;
+      context.itemName = found.name || context.itemName;
+      context.itemValue = found.description || context.itemValue;
+    }
+  }
+  result.matched_guid = matched_guid;
+  queueItemUpdate(resolvedType, result, context, state);
+  const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new item)";
+  console.log(`[handleHumanItemMatch] ${resolvedType} "${context.itemName}": ${matched}`);
+}
+
+export async function handleHumanItemUpdate(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as ItemUpdateResult | undefined;
+  
+  if (!result || Object.keys(result).length === 0) {
+    console.log("[handleHumanItemUpdate] No changes needed (empty result)");
+    return;
+  }
+
+  const candidateType = response.request.data.candidateType as DataItemType;
+  const isNewItem = response.request.data.isNewItem as boolean;
+  const existingItemId = response.request.data.existingItemId as string | undefined;
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+
+  if (!result.name || !result.description || result.sentiment === undefined) {
+    console.error("[handleHumanItemUpdate] Missing required fields in result");
+    return;
+  }
+
+  const now = new Date().toISOString();
+  const resolveItemId = (): string => {
+    if (isNewItem || !existingItemId) return crypto.randomUUID();
+    const h = state.getHuman();
+    const arr = candidateType === "fact" ? h.facts : candidateType === "trait" ? h.traits : candidateType === "topic" ? h.topics : h.people;
+    // Guard: if existingItemId isn't in the correct type array, treat as new
+    // (prevents cross-type ID reuse when LLM matches against a different type's UUID)
+    return arr.find((x: DataItemBase) => x.id === existingItemId) ? existingItemId : crypto.randomUUID();
+  };
+  const itemId = resolveItemId();
+
+  const persona = state.persona_getById(personaId);
+  const personaGroup = persona?.group_primary ?? null;
+  const isEi = personaDisplayName.toLowerCase() === "ei";
+
+  const human = state.getHuman();
+  const getExistingItem = (): { learned_by?: string; last_changed_by?: string; persona_groups?: string[] } | undefined => {
+    if (isNewItem) return undefined;
+    switch (candidateType) {
+      case "fact": return human.facts.find(f => f.id === existingItemId);
+      case "trait": return human.traits.find(t => t.id === existingItemId);
+      case "topic": return human.topics.find(t => t.id === existingItemId);
+      case "person": return human.people.find(p => p.id === existingItemId);
+    }
+  };
+  const existingItem = getExistingItem();
+
+  const mergeGroups = (existing: string[] | undefined): string[] | undefined => {
+    if (!personaGroup) return existing;
+    if (isNewItem) return [personaGroup];
+    const groups = new Set(existing ?? []);
+    groups.add(personaGroup);
+    return Array.from(groups);
+  };
+
+  let embedding: number[] | undefined;
+  try {
+    const embeddingService = getEmbeddingService();
+    const text = getItemEmbeddingText({ name: result.name, description: result.description });
+    embedding = await embeddingService.embed(text);
+  } catch (err) {
+    console.warn(`[handleHumanItemUpdate] Failed to compute embedding for ${candidateType} "${result.name}":`, err);
+  }
+
+  switch (candidateType) {
+    case "fact": {
+      const fact: Fact = {
+        id: itemId,
+        name: result.name,
+        description: result.description,
+        sentiment: result.sentiment,
+        validated: ValidationLevel.None,
+        validated_date: now,
+        last_updated: now,
+        learned_by: isNewItem ? personaId : existingItem?.learned_by,
+        last_changed_by: personaId,
+        persona_groups: mergeGroups(existingItem?.persona_groups),
+        embedding,
+      };
+      applyOrValidate(state, "fact", fact, personaDisplayName, isEi, personaGroup);
+      break;
+    }
+    case "trait": {
+      const trait: Trait = {
+        id: itemId,
+        name: result.name,
+        description: result.description,
+        sentiment: result.sentiment,
+        strength: (result as any).strength ?? 0.5,
+        last_updated: now,
+        learned_by: isNewItem ? personaId : existingItem?.learned_by,
+        last_changed_by: personaId,
+        persona_groups: mergeGroups(existingItem?.persona_groups),
+        embedding,
+      };
+      applyOrValidate(state, "trait", trait, personaDisplayName, isEi, personaGroup);
+      break;
+    }
+    case "topic": {
+      const exposureImpact = (result as any).exposure_impact as ExposureImpact | undefined;
+      const itemCategory = response.request.data.itemCategory as string | undefined;
+      const existingTopic = human.topics.find(t => t.id === existingItemId);
+      const topic: Topic = {
+        id: itemId,
+        name: result.name,
+        description: result.description,
+        sentiment: result.sentiment,
+        category: (result as any).category ?? itemCategory ?? existingTopic?.category,
+        exposure_current: calculateExposureCurrent(exposureImpact),
+        exposure_desired: (result as any).exposure_desired ?? 0.5,
+        last_updated: now,
+        learned_by: isNewItem ? personaId : existingItem?.learned_by,
+        last_changed_by: personaId,
+        persona_groups: mergeGroups(existingItem?.persona_groups),
+        embedding,
+      };
+      applyOrValidate(state, "topic", topic, personaDisplayName, isEi, personaGroup);
+      break;
+    }
+    case "person": {
+      const exposureImpact = (result as any).exposure_impact as ExposureImpact | undefined;
+      const person: Person = {
+        id: itemId,
+        name: result.name,
+        description: result.description,
+        sentiment: result.sentiment,
+        relationship: (result as any).relationship ?? "Unknown",
+        exposure_current: calculateExposureCurrent(exposureImpact),
+        exposure_desired: (result as any).exposure_desired ?? 0.5,
+        last_updated: now,
+        learned_by: isNewItem ? personaId : existingItem?.learned_by,
+        last_changed_by: personaId,
+        persona_groups: mergeGroups(existingItem?.persona_groups),
+        embedding,
+      };
+      applyOrValidate(state, "person", person, personaDisplayName, isEi, personaGroup);
+      break;
+    }
+  }
+
+  const allMessages = state.messages_get(personaId);
+  await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
+
+  console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
+}
+
+async function validateAndStoreQuotes(
+  candidates: Array<{ text: string; reason: string }> | undefined,
+  messages: Message[],
+  dataItemId: string,
+  personaName: string,
+  personaGroup: string | null,
+  state: StateManager
+): Promise<void> {
+  if (!candidates || candidates.length === 0) return;
+  
+  for (const candidate of candidates) {
+    let found = false;
+    for (const message of messages) {
+      const msgText = getMessageText(message);
+      const start = msgText.indexOf(candidate.text);
+      if (start !== -1) {
+        const end = start + candidate.text.length;
+        
+        // Check for ANY overlapping quote in this message (not just exact match)
+        const existing = state.human_quote_getForMessage(message.id);
+        const overlapping = existing.find(q =>
+          q.start !== null && q.end !== null &&
+          start < q.end && end > q.start  // ranges overlap
+        );
+        
+        if (overlapping) {
+          // Merge: expand to the union of both ranges
+          const mergedStart = Math.min(start, overlapping.start!);
+          const mergedEnd = Math.max(end, overlapping.end!);
+          const mergedText = msgText.slice(mergedStart, mergedEnd);
+          
+          // Merge data_item_ids and persona_groups (deduplicated)
+          const mergedDataItemIds = overlapping.data_item_ids.includes(dataItemId)
+            ? overlapping.data_item_ids
+            : [...overlapping.data_item_ids, dataItemId];
+          const group = personaGroup || "General";
+          const mergedGroups = overlapping.persona_groups.includes(group)
+            ? overlapping.persona_groups
+            : [...overlapping.persona_groups, group];
+          
+          // Only recompute embedding if the text actually changed
+          let embedding = overlapping.embedding;
+          if (mergedText !== overlapping.text) {
+            try {
+              const embeddingService = getEmbeddingService();
+              embedding = await embeddingService.embed(mergedText);
+            } catch (err) {
+              console.warn(`[extraction] Failed to recompute embedding for merged quote: "${mergedText.slice(0, 30)}..."`, err);
+            }
+          }
+          
+          state.human_quote_update(overlapping.id, {
+            start: mergedStart,
+            end: mergedEnd,
+            text: mergedText,
+            data_item_ids: mergedDataItemIds,
+            persona_groups: mergedGroups,
+            embedding,
+          });
+          console.log(`[extraction] Merged overlapping quote: "${mergedText.slice(0, 50)}..." (${mergedStart}-${mergedEnd})`);
+          found = true;
+          break;
+        }
+        
+        let embedding: number[] | undefined;
+        try {
+          const embeddingService = getEmbeddingService();
+          embedding = await embeddingService.embed(candidate.text);
+        } catch (err) {
+          console.warn(`[extraction] Failed to compute embedding for quote: "${candidate.text.slice(0, 30)}..."`, err);
+        }
+        
+        const quote: Quote = {
+          id: crypto.randomUUID(),
+          message_id: message.id,
+          data_item_ids: [dataItemId],
+          persona_groups: [personaGroup || "General"],
+          text: candidate.text,
+          speaker: message.role === "human" ? "human" : personaName,
+          timestamp: message.timestamp,
+          start: start,
+          end: end,
+          created_at: new Date().toISOString(),
+          created_by: "extraction",
+          embedding,
+        };
+        state.human_quote_add(quote);
+        console.log(`[extraction] Captured quote: "${candidate.text.slice(0, 50)}..."`);
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      console.log(`[extraction] Quote not found in messages, skipping: "${candidate.text?.slice(0, 50)}..."`);
+    }
+  }
+}
+
+function calculateExposureCurrent(impact: ExposureImpact | undefined): number {
+  switch (impact) {
+    case "high": return 0.9;
+    case "medium": return 0.6;
+    case "low": return 0.3;
+    case "none": return 0.1;
+    default: return 0.5;
+  }
+}
+
+function applyOrValidate(
+  state: StateManager,
+  dataType: DataItemType,
+  item: Fact | Trait | Topic | Person,
+  _personaName: string,
+  _isEi: boolean,
+  _personaGroup: string | null
+): void {
+  switch (dataType) {
+    case "fact": state.human_fact_upsert(item as Fact); break;
+    case "trait": state.human_trait_upsert(item as Trait); break;
+    case "topic": state.human_topic_upsert(item as Topic); break;
+    case "person": state.human_person_upsert(item as Person); break;
+  }
+}