ei-tui 0.1.3

package/src/core/orchestrators/human-extraction.ts

@@ -0,0 +1,457 @@
+import { LLMRequestType, LLMPriority, LLMNextStep, type Message, type DataItemType, type Fact, type Trait, type Topic, type Person } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import {
+  buildHumanFactScanPrompt,
+  buildHumanTraitScanPrompt,
+  buildHumanTopicScanPrompt,
+  buildHumanPersonScanPrompt,
+  buildHumanItemMatchPrompt,
+  buildHumanItemUpdatePrompt,
+  type FactScanCandidate,
+  type TraitScanCandidate,
+  type TopicScanCandidate,
+  type PersonScanCandidate,
+  type ItemMatchResult,
+} from "../../prompts/human/index.js";
+import { chunkExtractionContext } from "./extraction-chunker.js";
+import { getEmbeddingService, findTopK } from "../embedding-service.js";
+import { resolveTokenLimit } from "../llm-client.js";
+
+type ScanCandidate = FactScanCandidate | TraitScanCandidate | TopicScanCandidate | PersonScanCandidate;
+
+export interface ExtractionContext {
+  personaId: string;
+  personaDisplayName: string;
+  messages_context: Message[];
+  messages_analyze: Message[];
+  extraction_flag?: "f" | "r" | "p" | "o";
+}
+
+export interface ExtractionOptions {
+  ceremony_progress?: boolean;
+}
+
+function getAnalyzeFromTimestamp(context: ExtractionContext): string | null {
+  if (context.messages_analyze.length === 0) return null;
+  return context.messages_analyze[0].timestamp;
+}
+
+const EXTRACTION_BUDGET_RATIO = 0.75;
+const MIN_EXTRACTION_TOKENS = 10000;
+
+function getExtractionMaxTokens(state: StateManager): number {
+  const human = state.getHuman();
+  const tokenLimit = resolveTokenLimit(human.settings?.default_model, human.settings?.accounts);
+  return Math.max(MIN_EXTRACTION_TOKENS, Math.floor(tokenLimit * EXTRACTION_BUDGET_RATIO));
+}
+
+export function queueFactScan(context: ExtractionContext, state: StateManager, options?: ExtractionOptions): number {
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  
+  if (chunks.length === 0) return 0;
+  
+  for (const chunk of chunks) {
+    const prompt = buildHumanFactScanPrompt({
+      persona_name: chunk.personaDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Normal,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanFactScan,
+      data: {
+        ...options,
+        personaId: chunk.personaId,
+        personaDisplayName: chunk.personaDisplayName,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+        extraction_flag: context.extraction_flag,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+  
+  return chunks.length;
+}
+
+export function queueTraitScan(context: ExtractionContext, state: StateManager, options?: ExtractionOptions): number {
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  
+  if (chunks.length === 0) return 0;
+  
+  for (const chunk of chunks) {
+    const prompt = buildHumanTraitScanPrompt({
+      persona_name: chunk.personaDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Normal,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanTraitScan,
+      data: {
+        ...options,
+        personaId: chunk.personaId,
+        personaDisplayName: chunk.personaDisplayName,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+        extraction_flag: context.extraction_flag,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+  
+  return chunks.length;
+}
+
+export function queueTopicScan(context: ExtractionContext, state: StateManager, options?: ExtractionOptions): number {
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  
+  if (chunks.length === 0) return 0;
+  
+  for (const chunk of chunks) {
+    const prompt = buildHumanTopicScanPrompt({
+      persona_name: chunk.personaDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanTopicScan,
+      data: {
+        ...options,
+        personaId: chunk.personaId,
+        personaDisplayName: chunk.personaDisplayName,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+        extraction_flag: context.extraction_flag,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+  
+  return chunks.length;
+}
+
+export function queuePersonScan(context: ExtractionContext, state: StateManager, options?: ExtractionOptions): number {
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  
+  if (chunks.length === 0) return 0;
+  
+  const personas = state.persona_getAll();
+  const knownPersonaNames = personas.flatMap(p => p.aliases ?? []);
+
+  for (const chunk of chunks) {
+    const prompt = buildHumanPersonScanPrompt({
+      persona_name: chunk.personaDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+      known_persona_names: knownPersonaNames,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Normal,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanPersonScan,
+      data: {
+        ...options,
+        personaId: chunk.personaId,
+        personaDisplayName: chunk.personaDisplayName,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+        extraction_flag: context.extraction_flag,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+  
+  return chunks.length;
+}
+
+export function queueAllScans(context: ExtractionContext, state: StateManager, options?: ExtractionOptions): void {
+  queueFactScan(context, state, options);
+  queueTraitScan(context, state, options);
+  queuePersonScan(context, state, options);
+  queueTopicScan(context, state, options);
+}
+
+/**
+ * Queue a direct Topic Update, bypassing scan/match.
+ * 
+ * Use this when we KNOW the topic already exists (e.g., OpenCode sessions
+ * where each session IS a topic). This avoids the queue explosion from
+ * scan → match → update pipeline.
+ * 
+ * @param topic - The known Topic to update
+ * @param context - Messages to analyze for this topic
+ * @param state - StateManager for queue operations
+ * @returns Number of chunks queued
+ */
+export function queueDirectTopicUpdate(
+  topic: import("../types.js").Topic,
+  context: ExtractionContext,
+  state: StateManager
+): number {
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+
+  if (chunks.length === 0) return 0;
+
+  for (const chunk of chunks) {
+    const prompt = buildHumanItemUpdatePrompt({
+      data_type: "topic",
+      existing_item: topic,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+      persona_name: chunk.personaDisplayName,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanItemUpdate,
+      data: {
+        personaId: context.personaId,
+        personaDisplayName: context.personaDisplayName,
+        candidateType: "topic",
+        matchedType: "topic",
+        isNewItem: false,
+        existingItemId: topic.id,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+      },
+    });
+  }
+
+  return chunks.length;
+}
+
+function truncateDescription(description: string, maxLength: number = 255): string {
+  if (description.length <= maxLength) return description;
+  return description.slice(0, maxLength) + "...";
+}
+
+const EMBEDDING_TOP_K = 20;
+const EMBEDDING_MIN_SIMILARITY = 0.3;
+
+/**
+ * Queue an item match request using embedding-based similarity.
+ * 
+ * Instead of sending ALL items to the LLM, we:
+ * 1. Compute embedding for the candidate (name + value)
+ * 2. Find top-K most similar existing items via cosine similarity
+ * 3. Send only those candidates to the LLM for final matching decision
+ * 
+ * This reduces prompt size from O(all_items) to O(K) where K=20.
+ */
+export async function queueItemMatch(
+  dataType: DataItemType,
+  candidate: ScanCandidate,
+  context: ExtractionContext,
+  state: StateManager
+): Promise<void> {
+  const human = state.getHuman();
+  
+  let itemName: string;
+  let itemValue: string;
+
+  switch (dataType) {
+    case "fact":
+      itemName = (candidate as FactScanCandidate).type_of_fact;
+      itemValue = (candidate as FactScanCandidate).value_of_fact;
+      break;
+    case "trait":
+      itemName = (candidate as TraitScanCandidate).type_of_trait;
+      itemValue = (candidate as TraitScanCandidate).value_of_trait;
+      break;
+    case "topic":
+      itemName = (candidate as TopicScanCandidate).value_of_topic;
+      itemValue = (candidate as TopicScanCandidate).type_of_topic;
+      break;
+    case "person":
+      itemName = (candidate as PersonScanCandidate).name_of_person;
+      itemValue = (candidate as PersonScanCandidate).type_of_person;
+      break;
+  }
+
+  const allItemsWithEmbeddings = [
+    ...human.facts.map(f => ({ ...f, data_type: "fact" as DataItemType })),
+    ...human.traits.map(t => ({ ...t, data_type: "trait" as DataItemType })),
+    ...human.topics.map(t => ({ ...t, data_type: "topic" as DataItemType })),
+    ...human.people.map(p => ({ ...p, data_type: "person" as DataItemType })),
+  ].filter(item => item.embedding && item.embedding.length > 0);
+
+  let topKItems: Array<{
+    data_type: DataItemType;
+    data_id: string;
+    data_name: string;
+    data_description: string;
+  }> = [];
+
+  if (allItemsWithEmbeddings.length > 0) {
+    try {
+      const embeddingService = getEmbeddingService();
+      const candidateText = `${itemName}: ${itemValue}`;
+      const candidateVector = await embeddingService.embed(candidateText);
+
+      const topK = findTopK(candidateVector, allItemsWithEmbeddings, EMBEDDING_TOP_K);
+      
+      topKItems = topK
+        .filter(({ similarity }) => similarity >= EMBEDDING_MIN_SIMILARITY)
+        .map(({ item }) => ({
+          data_type: item.data_type,
+          data_id: item.id,
+          data_name: item.name,
+          data_description: item.data_type === dataType 
+            ? item.description 
+            : truncateDescription(item.description),
+        }));
+
+      console.log(`[queueItemMatch] Embedding search: ${allItemsWithEmbeddings.length} items → ${topKItems.length} candidates (top-K=${EMBEDDING_TOP_K}, min_sim=${EMBEDDING_MIN_SIMILARITY})`);
+    } catch (err) {
+      console.error(`[queueItemMatch] Embedding search failed, falling back to all items:`, err);
+    }
+  }
+
+  if (topKItems.length === 0) {
+    console.log(`[queueItemMatch] No embeddings available, using all ${human.facts.length + human.traits.length + human.topics.length + human.people.length} items`);
+    
+    for (const fact of human.facts) {
+      topKItems.push({
+        data_type: "fact",
+        data_id: fact.id,
+        data_name: fact.name,
+        data_description: dataType === "fact" ? fact.description : truncateDescription(fact.description),
+      });
+    }
+
+    for (const trait of human.traits) {
+      topKItems.push({
+        data_type: "trait",
+        data_id: trait.id,
+        data_name: trait.name,
+        data_description: dataType === "trait" ? trait.description : truncateDescription(trait.description),
+      });
+    }
+
+    for (const topic of human.topics) {
+      topKItems.push({
+        data_type: "topic",
+        data_id: topic.id,
+        data_name: topic.name,
+        data_description: dataType === "topic" ? topic.description : truncateDescription(topic.description),
+      });
+    }
+
+    for (const person of human.people) {
+      topKItems.push({
+        data_type: "person",
+        data_id: person.id,
+        data_name: person.name,
+        data_description: dataType === "person" ? person.description : truncateDescription(person.description),
+      });
+    }
+  }
+
+  const prompt = buildHumanItemMatchPrompt({
+    candidate_type: dataType,
+    candidate_name: itemName,
+    candidate_value: itemValue,
+    all_items: topKItems,
+  });
+
+
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Low,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandleHumanItemMatch,
+    data: {
+      ...context,
+      candidateType: dataType,
+      itemName,
+      itemValue,
+    },
+  });
+}
+
+export function queueItemUpdate(
+  candidateType: DataItemType,
+  matchResult: ItemMatchResult,
+  context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string },
+  state: StateManager
+): number {
+  const human = state.getHuman();
+  const matchedGuid = matchResult.matched_guid;
+  const isNewItem = matchedGuid === null;
+
+  let existingItem: Fact | Trait | Topic | Person | null = null;
+  let matchedType: DataItemType | null = null;
+
+  if (!isNewItem) {
+    existingItem = human.facts.find(f => f.id === matchedGuid) ?? null;
+    if (existingItem) matchedType = "fact";
+
+    if (!existingItem) {
+      existingItem = human.traits.find(t => t.id === matchedGuid) ?? null;
+      if (existingItem) matchedType = "trait";
+    }
+
+    if (!existingItem) {
+      existingItem = human.topics.find(t => t.id === matchedGuid) ?? null;
+      if (existingItem) matchedType = "topic";
+    }
+
+    if (!existingItem) {
+      existingItem = human.people.find(p => p.id === matchedGuid) ?? null;
+      if (existingItem) matchedType = "person";
+    }
+  }
+
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+
+  if (chunks.length === 0) return 0;
+
+  for (const chunk of chunks) {
+    const prompt = buildHumanItemUpdatePrompt({
+      data_type: candidateType,
+      existing_item: existingItem,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+      persona_name: chunk.personaDisplayName,
+      new_item_name: isNewItem ? context.itemName : undefined,
+      new_item_value: isNewItem ? context.itemValue : undefined,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanItemUpdate,
+      data: {
+        personaId: context.personaId,
+        personaDisplayName: context.personaDisplayName,
+        candidateType,
+        matchedType,
+        isNewItem,
+        existingItemId: existingItem?.id,
+        itemCategory: context.itemCategory,
+        analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
+      },
+    });
+  }
+
+  return chunks.length;
+}

package/src/core/orchestrators/index.ts

@@ -0,0 +1,28 @@
+export { orchestratePersonaGeneration, type PartialPersona } from "./persona-generation.js";
+export {
+  queueFactScan,
+  queueTraitScan,
+  queueTopicScan,
+  queuePersonScan,
+  queueAllScans,
+  queueItemMatch,
+  queueItemUpdate,
+  type ExtractionContext,
+  type ExtractionOptions,
+} from "./human-extraction.js";
+export { 
+  shouldStartCeremony, 
+  startCeremony,
+  handleCeremonyProgress,
+  prunePersonaMessages,
+  queueExpirePhase,
+  queueExplorePhase,
+  queueDescriptionCheck,
+  runHumanCeremony,
+} from "./ceremony.js";
+export {
+  queuePersonaTopicScan,
+  queuePersonaTopicMatch,
+  queuePersonaTopicUpdate,
+  type PersonaTopicContext,
+} from "./persona-topics.js";

package/src/core/orchestrators/persona-generation.ts

@@ -0,0 +1,76 @@
+import { LLMRequestType, LLMPriority, LLMNextStep, type Trait, type PersonaTopic } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import { buildPersonaGenerationPrompt } from "../../prompts/index.js";
+
+const MAX_ORCHESTRATOR_LOOPS = 4;
+
+export interface PartialPersona {
+  id: string;
+  name: string;
+  aliases?: string[];
+  description?: string;
+  short_description?: string;
+  long_description?: string;
+  traits?: Partial<Trait>[];
+  topics?: Partial<PersonaTopic>[];
+  model?: string;
+  group_primary?: string;
+  groups_visible?: string[];
+  loop_counter?: number;
+  step?: "description" | "traits" | "topics";
+}
+
+export function orchestratePersonaGeneration(
+  partial: PartialPersona,
+  stateManager: StateManager,
+  onComplete?: () => void
+): void {
+  const loopCounter = (partial.loop_counter ?? 0) + 1;
+
+  if (loopCounter > MAX_ORCHESTRATOR_LOOPS) {
+    console.error(`[orchestratePersonaGeneration] Max loops (${MAX_ORCHESTRATOR_LOOPS}) exceeded for ${partial.name}`);
+    return;
+  }
+
+  const needsShortDescription = !partial.short_description;
+  const traitCount = partial.traits?.filter(t => t.name?.trim()).length ?? 0;
+  const topicCount = partial.topics?.filter(t => t.name?.trim()).length ?? 0;
+  const needsMoreTraits = traitCount < 3;
+  const needsMoreTopics = topicCount < 3;
+
+  if (needsShortDescription || needsMoreTraits || needsMoreTopics) {
+    const prompt = buildPersonaGenerationPrompt({
+      name: partial.name,
+      long_description: partial.long_description,
+      short_description: partial.short_description,
+      existing_traits: partial.traits,
+      existing_topics: partial.topics,
+    });
+
+    stateManager.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.High,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandlePersonaGeneration,
+      data: {
+        personaId: partial.id,
+        personaDisplayName: partial.name,
+        partial: { ...partial, loop_counter: loopCounter },
+      },
+    });
+    return;
+  }
+
+  const now = new Date().toISOString();
+  stateManager.persona_update(partial.id, {
+    short_description: partial.short_description,
+    long_description: partial.long_description,
+    traits: partial.traits as Trait[],
+    topics: partial.topics as PersonaTopic[],
+    last_updated: now,
+  });
+
+  console.log(`[orchestratePersonaGeneration] Completed: ${partial.name}`);
+  onComplete?.();
+}

package/src/core/orchestrators/persona-topics.ts

@@ -0,0 +1,117 @@
+import { LLMRequestType, LLMPriority, LLMNextStep, type Message, type PersonaTopic } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import {
+  buildPersonaTopicScanPrompt,
+  buildPersonaTopicMatchPrompt,
+  buildPersonaTopicUpdatePrompt,
+  type PersonaTopicScanCandidate,
+  type PersonaTopicMatchResult,
+} from "../../prompts/persona/index.js";
+
+export interface PersonaTopicContext {
+  personaId: string;
+  personaDisplayName: string;
+  messages_context: Message[];
+  messages_analyze: Message[];
+}
+
+function getAnalyzeFromTimestamp(context: PersonaTopicContext): string | null {
+  if (context.messages_analyze.length === 0) return null;
+  return context.messages_analyze[0].timestamp;
+}
+
+export function queuePersonaTopicScan(context: PersonaTopicContext, state: StateManager): void {
+  const prompt = buildPersonaTopicScanPrompt({
+    persona_name: context.personaDisplayName,
+    messages_context: context.messages_context,
+    messages_analyze: context.messages_analyze,
+  });
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Low,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandlePersonaTopicScan,
+    data: {
+      personaId: context.personaId,
+      personaDisplayName: context.personaDisplayName,
+      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
+    },
+  });
+}
+
+export function queuePersonaTopicMatch(
+  candidate: PersonaTopicScanCandidate,
+  context: PersonaTopicContext,
+  state: StateManager
+): void {
+  const persona = state.persona_getById(context.personaId);
+  if (!persona) {
+    console.error(`[queuePersonaTopicMatch] Persona not found: ${context.personaId}`);
+    return;
+  }
+
+  const prompt = buildPersonaTopicMatchPrompt({
+    persona_name: context.personaDisplayName,
+    candidate,
+    existing_topics: persona.topics,
+  });
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Low,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandlePersonaTopicMatch,
+    data: {
+      personaId: context.personaId,
+      personaDisplayName: context.personaDisplayName,
+      candidate,
+      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
+    },
+  });
+}
+
+export function queuePersonaTopicUpdate(
+  candidate: PersonaTopicScanCandidate,
+  matchResult: PersonaTopicMatchResult,
+  context: PersonaTopicContext,
+  state: StateManager
+): void {
+  const persona = state.persona_getById(context.personaId);
+  if (!persona) {
+    console.error(`[queuePersonaTopicUpdate] Persona not found: ${context.personaId}`);
+    return;
+  }
+
+  const existingTopic = matchResult.matched_id
+    ? persona.topics.find((t: PersonaTopic) => t.id === matchResult.matched_id)
+    : undefined;
+
+  const prompt = buildPersonaTopicUpdatePrompt({
+    persona_name: context.personaDisplayName,
+    short_description: persona.short_description,
+    long_description: persona.long_description,
+    traits: persona.traits,
+    existing_topic: existingTopic,
+    candidate,
+    messages_context: context.messages_context,
+    messages_analyze: context.messages_analyze,
+  });
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Low,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandlePersonaTopicUpdate,
+    data: {
+      personaId: context.personaId,
+      personaDisplayName: context.personaDisplayName,
+      candidate,
+      matched_id: matchResult.matched_id,
+      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
+    },
+  });
+}

package/src/core/personas/index.ts

@@ -0,0 +1,5 @@
+export {
+  ensureAgentPersona,
+  ensureAllAgentPersonas,
+  type EnsureAgentPersonaOptions,
+} from "./opencode-agent.js";