ei-tui 0.1.3 → 0.1.4

package/src/core/processor.ts

@@ -2,6 +2,7 @@ import {
   LLMRequestType,
   LLMPriority,
   LLMNextStep,
+  ValidationLevel,
   RESERVED_PERSONA_NAMES,
   isReservedPersonaName,
   type LLMRequest,
@@ -35,9 +36,12 @@ import {
   buildResponsePrompt,
   buildPersonaTraitExtractionPrompt,
   buildHeartbeatCheckPrompt,
+  buildEiHeartbeatPrompt,
   type ResponsePromptData,
   type PersonaTraitExtractionPromptData,
   type HeartbeatCheckPromptData,
+  type EiHeartbeatPromptData,
+  type EiHeartbeatItem,
 } from "../prompts/index.js";
 import { 
   orchestratePersonaGeneration,
@@ -369,7 +373,7 @@ export class Processor {
 
     const human = this.stateManager.getHuman();
     
-    if (this.isTUI && human.settings?.opencode?.integration) {
+    if (this.isTUI && human.settings?.opencode?.integration && this.stateManager.queue_length() === 0) {
       await this.checkAndSyncOpenCode(human, now);
     }
     
@@ -434,12 +438,10 @@ export class Processor {
       },
     });
 
-    const since = lastSync > 0 ? new Date(lastSync) : new Date(0);
-
     this.openCodeImportInProgress = true;
     import("../integrations/opencode/importer.js")
-      .then(({ importOpenCodeSessions }) => 
-        importOpenCodeSessions(since, {
+      .then(({ importOpenCodeSessions }) =>
+        importOpenCodeSessions({
           stateManager: this.stateManager,
           interface: this.interface,
         })
@@ -449,7 +451,7 @@ export class Processor {
           console.log(
             `[Processor] OpenCode sync complete: ${result.sessionsProcessed} sessions, ` +
             `${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
-            `${result.topicUpdatesQueued} topic updates queued`
+            `${result.extractionScansQueued} extraction scans queued`
           );
         }
       })
@@ -491,20 +493,20 @@ export class Processor {
   private async queueHeartbeatCheck(personaId: string): Promise<void> {
     const persona = this.stateManager.persona_getById(personaId);
     if (!persona) return;
-
     this.stateManager.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
-
     const human = this.stateManager.getHuman();
     const history = this.stateManager.messages_get(personaId);
-    const filteredHuman = await this.filterHumanDataByVisibility(human, persona);
+    if (personaId === "ei") {
+      await this.queueEiHeartbeat(human, history);
+      return;
+    }
 
+    const filteredHuman = await this.filterHumanDataByVisibility(human, persona);
     const inactiveDays = persona.last_activity
       ? Math.floor((Date.now() - new Date(persona.last_activity).getTime()) / (1000 * 60 * 60 * 24))
       : 0;
-
     const sortByEngagementGap = <T extends { exposure_desired: number; exposure_current: number }>(items: T[]): T[] =>
       [...items].sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current));
-
     const promptData: HeartbeatCheckPromptData = {
       persona: {
         name: persona.display_name,
@@ -532,6 +534,118 @@ export class Processor {
     });
   }
 
+  private async queueEiHeartbeat(human: HumanEntity, history: import("./types.js").Message[]): Promise<void> {
+    const now = Date.now();
+    const engagementGapThreshold = 0.2;
+    const cooldownMs = 7 * 24 * 60 * 60 * 1000;
+    const personas = this.stateManager.persona_getAll();
+    const items: EiHeartbeatItem[] = [];
+
+    const unverifiedFacts = human.facts
+      .filter(f => f.validated === ValidationLevel.None && f.learned_by !== "ei")
+      .slice(0, 5);
+    for (const fact of unverifiedFacts) {
+      const quote = human.quotes.find(q => q.data_item_ids.includes(fact.id));
+      items.push({
+        id: fact.id,
+        type: "Fact Check",
+        name: fact.name,
+        description: fact.description,
+        quote: quote?.text,
+      });
+    }
+
+    const underEngagedPeople = human.people
+      .filter(p =>
+        (p.exposure_desired - p.exposure_current) > engagementGapThreshold &&
+        (!p.last_ei_asked || now - new Date(p.last_ei_asked).getTime() > cooldownMs)
+      )
+      .sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current))
+      .slice(0, 5);
+    for (const person of underEngagedPeople) {
+      const gap = Math.round((person.exposure_desired - person.exposure_current) * 100);
+      const quote = human.quotes.find(q => q.data_item_ids.includes(person.id));
+      items.push({
+        id: person.id,
+        type: "Low-Engagement Person",
+        engagement_delta: `${gap}%`,
+        relationship: person.relationship,
+        name: person.name,
+        description: person.description,
+        quote: quote?.text,
+      });
+    }
+
+    const underEngagedTopics = human.topics
+      .filter(t =>
+        (t.exposure_desired - t.exposure_current) > engagementGapThreshold &&
+        (!t.last_ei_asked || now - new Date(t.last_ei_asked).getTime() > cooldownMs)
+      )
+      .sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current))
+      .slice(0, 5);
+    for (const topic of underEngagedTopics) {
+      const gap = Math.round((topic.exposure_desired - topic.exposure_current) * 100);
+      const quote = human.quotes.find(q => q.data_item_ids.includes(topic.id));
+      items.push({
+        id: topic.id,
+        type: "Low-Engagement Topic",
+        engagement_delta: `${gap}%`,
+        name: topic.name,
+        description: topic.description,
+        quote: quote?.text,
+      });
+    }
+
+    const activePersonas = personas
+      .filter(p => !p.is_archived && !p.is_paused && p.id !== "ei")
+      .map(p => {
+        const msgs = this.stateManager.messages_get(p.id);
+        const lastHuman = [...msgs].reverse().find(m => m.role === "human");
+        const lastTs = lastHuman?.timestamp ? new Date(lastHuman.timestamp).getTime() : 0;
+        return { persona: p, lastHumanTs: lastTs };
+      })
+      .filter(({ lastHumanTs }) => {
+        const daysSince = (now - lastHumanTs) / (1000 * 60 * 60 * 24);
+        return daysSince >= 3;
+      })
+      .sort((a, b) => a.lastHumanTs - b.lastHumanTs)
+      .slice(0, 3);
+    for (const { persona: p, lastHumanTs } of activePersonas) {
+      const daysSince = lastHumanTs > 0
+        ? Math.floor((now - lastHumanTs) / (1000 * 60 * 60 * 24))
+        : 999;
+      items.push({
+        id: p.id,
+        type: "Inactive Persona",
+        name: p.display_name,
+        short_description: p.short_description,
+        days_inactive: daysSince,
+      });
+    }
+
+    if (items.length === 0) {
+      console.log("[queueEiHeartbeat] No items to address, skipping");
+      return;
+    }
+
+    const promptData: EiHeartbeatPromptData = {
+      items,
+      recent_history: history.slice(-10),
+    };
+
+    const prompt = buildEiHeartbeatPrompt(promptData);
+
+    this.stateManager.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleEiHeartbeat,
+      model: this.getModelForPersona("ei"),
+      data: { personaId: "ei", isTUI: this.isTUI },
+    });
+  }
+
 
   private classifyLLMError(error: string): string {
     const match = error.match(/\((\d{3})\)/);
@@ -627,9 +741,7 @@ export class Processor {
         }
       }
 
-      if (response.request.next_step === LLMNextStep.HandleEiValidation) {
-        this.interface.onHumanUpdated?.();
-      }
+
 
       if (response.request.next_step === LLMNextStep.HandleHumanItemUpdate) {
         this.interface.onHumanUpdated?.();
@@ -813,23 +925,16 @@ export class Processor {
   async recallPendingMessages(personaId: string): Promise<string> {
     const persona = this.stateManager.persona_getById(personaId);
     if (!persona) return "";
-    
     this.clearPendingRequestsFor(personaId);
-    this.stateManager.queue_pause();
-    
     const messages = this.stateManager.messages_get(personaId);
     const pendingIds = messages
       .filter(m => m.role === "human" && !m.read)
       .map(m => m.id);
-    
     if (pendingIds.length === 0) return "";
-    
     const removed = this.stateManager.messages_remove(personaId, pendingIds);
     const recalledContent = removed.map(m => m.content).join("\n\n");
-    
     this.interface.onMessageAdded?.(personaId);
     this.interface.onMessageRecalled?.(personaId, recalledContent);
-    
     return recalledContent;
   }
 
@@ -987,13 +1092,40 @@ export class Processor {
   ): Promise<ResponsePromptData["human"]> {
     const DEFAULT_GROUP = "General";
     const QUOTE_LIMIT = 10;
+    const DATA_ITEM_LIMIT = 15;
     const SIMILARITY_THRESHOLD = 0.3;
+    // Generic relevance selector for embedding-capable items.
+    // Falls back to returning all items when no message/embeddings are available.
+    const selectRelevantItems = async <T extends { id: string; embedding?: number[] }>(
+      items: T[],
+      limit: number
+    ): Promise<T[]> => {
+      if (items.length === 0) return [];
+
+      const withEmbeddings = items.filter(i => i.embedding?.length);
+
+      if (currentMessage && withEmbeddings.length > 0) {
+        try {
+          const embeddingService = getEmbeddingService();
+          const queryVector = await embeddingService.embed(currentMessage);
+          const results = findTopK(queryVector, withEmbeddings, limit);
+          const relevant = results
+            .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
+            .map(({ item }) => item);
 
+          if (relevant.length > 0) return relevant;
+        } catch (err) {
+          console.warn("[filterHumanDataByVisibility] Embedding search failed:", err);
+        }
+      }
+
+      // Fallback: return all items (caller may apply its own limit)
+      return items;
+    };
     const selectRelevantQuotes = async (quotes: Quote[]): Promise<Quote[]> => {
       if (quotes.length === 0) return [];
-      
       const withEmbeddings = quotes.filter(q => q.embedding?.length);
-      
+
       if (currentMessage && withEmbeddings.length > 0) {
         try {
           const embeddingService = getEmbeddingService();
@@ -1002,35 +1134,31 @@ export class Processor {
           const relevant = results
             .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
             .map(({ item }) => item);
-          
+
           if (relevant.length > 0) return relevant;
         } catch (err) {
           console.warn("[filterHumanDataByVisibility] Embedding search failed:", err);
         }
       }
-      
       return [...quotes]
         .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
         .slice(0, QUOTE_LIMIT);
     };
-
     if (persona.id === "ei") {
-      const relevantQuotes = await selectRelevantQuotes(human.quotes ?? []);
-      return {
-        facts: human.facts,
-        traits: human.traits,
-        topics: human.topics,
-        people: human.people,
-        quotes: relevantQuotes,
-      };
+      const [facts, traits, topics, people, quotes] = await Promise.all([
+        selectRelevantItems(human.facts, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.traits, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.topics, DATA_ITEM_LIMIT),
+        selectRelevantItems(human.people, DATA_ITEM_LIMIT),
+        selectRelevantQuotes(human.quotes ?? []),
+      ]);
+      return { facts, traits, topics, people, quotes };
     }
-
     const visibleGroups = new Set<string>();
     if (persona.group_primary) {
       visibleGroups.add(persona.group_primary);
     }
     (persona.groups_visible ?? []).forEach((g) => visibleGroups.add(g));
-
     const filterByGroup = <T extends DataItemBase>(items: T[]): T[] => {
       return items.filter((item) => {
         const itemGroups = item.persona_groups ?? [];
@@ -1038,21 +1166,20 @@ export class Processor {
         return effectiveGroups.some((g) => visibleGroups.has(g));
       });
     };
-
     const groupFilteredQuotes = (human.quotes ?? []).filter((q) => {
       const effectiveGroups = q.persona_groups.length === 0 ? [DEFAULT_GROUP] : q.persona_groups;
       return effectiveGroups.some((g) => visibleGroups.has(g));
     });
 
-    const relevantQuotes = await selectRelevantQuotes(groupFilteredQuotes);
+    const [facts, traits, topics, people, quotes] = await Promise.all([
+      selectRelevantItems(filterByGroup(human.facts), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.traits), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.topics), DATA_ITEM_LIMIT),
+      selectRelevantItems(filterByGroup(human.people), DATA_ITEM_LIMIT),
+      selectRelevantQuotes(groupFilteredQuotes),
+    ]);
 
-    return {
-      facts: filterByGroup(human.facts),
-      traits: filterByGroup(human.traits),
-      topics: filterByGroup(human.topics),
-      people: filterByGroup(human.people),
-      quotes: relevantQuotes,
-    };
+    return { facts, traits, topics, people, quotes };
   }
 
   private getVisiblePersonas(

package/src/core/state/checkpoints.ts

@@ -45,6 +45,10 @@ export class PersistenceState {
     return this.loadedExistingData;
   }
 
+  markExistingData(): void {
+    this.loadedExistingData = true;
+  }
+
   async flush(): Promise<void> {
     if (this.saveTimeout) {
       clearTimeout(this.saveTimeout);

package/src/core/state/queue.ts

@@ -1,4 +1,4 @@
-import type { LLMRequest, LLMNextStep, QueueFailResult } from "../types.js";
+import type { LLMRequest, QueueFailResult } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -93,16 +93,7 @@ export class QueueState {
     return { dropped: false, retryDelay: delay };
   }
 
-  getValidations(): LLMRequest[] {
-    return this.queue.filter(
-      (r) => r.next_step === ("handleEiValidation" as LLMNextStep)
-    );
-  }
 
-  clearValidations(ids: string[]): void {
-    const idSet = new Set(ids);
-    this.queue = this.queue.filter((r) => !idSet.has(r.id));
-  }
 
   clearPersonaResponses(personaId: string, nextStep: string): string[] {
     const removedIds: string[] = [];

package/src/core/state-manager.ts

@@ -275,14 +275,7 @@ export class StateManager {
     return result;
   }
 
-  queue_getValidations(): LLMRequest[] {
-    return this.queueState.getValidations();
-  }
 
-  queue_clearValidations(ids: string[]): void {
-    this.queueState.clearValidations(ids);
-    this.scheduleSave();
-  }
 
   queue_clearPersonaResponses(personaId: string, nextStep: string): string[] {
     const result = this.queueState.clearPersonaResponses(personaId, nextStep);
@@ -338,6 +331,7 @@ export class StateManager {
     this.humanState.load(state.human);
     this.personaState.load(state.personas);
     this.queueState.load(state.queue);
+    this.persistenceState.markExistingData();
     this.scheduleSave();
   }
 

package/src/core/types.ts

@@ -18,7 +18,6 @@ export enum ValidationLevel {
   Ei = "ei",         // Ei mentioned it to user (don't mention again)
   Human = "human",   // User explicitly confirmed (locked)
 }
-
 export enum LLMRequestType {
   Response = "response",
   JSON = "json",
@@ -47,9 +46,7 @@ export enum LLMNextStep {
   HandlePersonaTopicUpdate = "handlePersonaTopicUpdate",
   HandleHeartbeatCheck = "handleHeartbeatCheck",
   HandleEiHeartbeat = "handleEiHeartbeat",
-  HandleEiValidation = "handleEiValidation",
   HandleOneShot = "handleOneShot",
-  // Ceremony handlers
   HandlePersonaExpire = "handlePersonaExpire",
   HandlePersonaExplore = "handlePersonaExplore",
   HandleDescriptionCheck = "handleDescriptionCheck",
@@ -83,6 +80,7 @@ export interface Topic extends DataItemBase {
   category?: string; // Interest, Goal, Dream, Conflict, Concern, Fear, Hope, Plan, Project
   exposure_current: number;
   exposure_desired: number;
+  last_ei_asked?: string | null;  // ISO timestamp of last time Ei proactively asked about this
 }
 
 /**
@@ -109,6 +107,7 @@ export interface Person extends DataItemBase {
   relationship: string;
   exposure_current: number;
   exposure_desired: number;
+  last_ei_asked?: string | null;  // ISO timestamp of last time Ei proactively asked about this
 }
 
 export interface Quote {
@@ -180,7 +179,8 @@ export interface OpenCodeSettings {
   integration?: boolean;
   polling_interval_ms?: number;  // Default: 1800000 (30 min)
   last_sync?: string;  // ISO timestamp
-  extraction_point?: string;  // ISO timestamp - earliest unprocessed message, gradual extraction advances this
+  extraction_point?: string;  // ISO timestamp - cursor for single-session archive scan
+  processed_sessions?: Record<string, string>;  // sessionId → ISO timestamp of last import
 }
 
 export interface CeremonyConfig {
@@ -238,7 +238,6 @@ export interface PersonaEntity {
   last_activity: string;
   last_heartbeat?: string;
   last_extraction?: string;
-  last_inactivity_ping?: string;
 }
 
 export interface PersonaCreationInput {

package/src/core/utils/crossFind.ts

@@ -0,0 +1,44 @@
+import type { HumanEntity, PersonaEntity, Fact, Trait, Topic, Person, Quote, PersonaTopic } from "../types.ts";
+export type CrossFindResult =
+  | { type: "fact"        } & Fact
+  | { type: "trait"       } & Trait
+  | { type: "topic"       } & Topic
+  | { type: "person"      } & Person
+  | { type: "quote"       } & Quote
+  | { type: "persona"     } & PersonaEntity
+  | { type: "personaTopic"; personaId: string } & PersonaTopic
+  | { type: "personaTrait"; personaId: string } & Trait;
+
+export function crossFind(
+  id: string,
+  human: HumanEntity,
+  personas?: PersonaEntity[],
+): CrossFindResult | null {
+
+  const fact  = human.facts.find(f => f.id === id);
+  if (fact)  return { type: "fact",   ...fact };
+
+  const trait = human.traits.find(t => t.id === id);
+  if (trait) return { type: "trait",  ...trait };
+
+  const person = human.people.find(p => p.id === id);
+  if (person) return { type: "person", ...person };
+
+  const topic = human.topics.find(t => t.id === id);
+  if (topic) return { type: "topic",  ...topic };
+
+  const quote = human.quotes.find(q => q.id === id);
+  if (quote) return { type: "quote",  ...quote };
+
+  for (const persona of personas ?? []) {
+    if (persona.id === id) return { type: "persona", ...persona };
+
+    const pTopic = persona.topics.find(t => t.id === id);
+    if (pTopic) return { type: "personaTopic", personaId: persona.id, ...pTopic };
+
+    const pTrait = persona.traits.find(t => t.id === id);
+    if (pTrait) return { type: "personaTrait", personaId: persona.id, ...pTrait };
+  }
+
+  return null;
+}

package/src/core/utils/index.ts

@@ -0,0 +1,4 @@
+import { crossFind } from "./crossFind.js";
+import { applyDecayToValue } from "./decay.js";
+
+export { crossFind, applyDecayToValue };