ei-tui 0.5.4 → 0.6.1

package/src/core/orchestrators/human-extraction.ts

@@ -1,4 +1,5 @@
 import { LLMRequestType, LLMPriority, LLMNextStep, type Message, type Topic, type Person } from "../types.js";
+import type { PersonIdentifier } from "../types/data-items.js";
 import type { StateManager } from "../state-manager.js";
 import {
   buildFactFindPrompt,
@@ -6,16 +7,14 @@ import {
   buildHumanPersonScanPrompt,
   buildTopicMatchPrompt,
   buildTopicUpdatePrompt,
-  buildPersonMatchPrompt,
   buildPersonUpdatePrompt,
   buildEventScanPrompt,
   type TopicScanCandidate,
-  type PersonScanCandidate,
   type ItemMatchResult,
   type ParticipantContext,
 } from "../../prompts/human/index.js";
 import { chunkExtractionContext } from "./extraction-chunker.js";
-import { getEmbeddingService, findTopK, getTopicEmbeddingText, getPersonEmbeddingText } from "../embedding-service.js";
+import { getEmbeddingService, findTopK, getTopicEmbeddingText } from "../embedding-service.js";
 import { resolveTokenLimit } from "../llm-client.js";
 import { BUILT_IN_FACT_NAMES } from "../constants/built-in-facts.js";
 import { buildEventWindows } from "../utils/event-windows.js";
@@ -192,11 +191,20 @@ export function queuePersonScan(context: ExtractionContext, state: StateManager,
     state.messages_markExtracted(chunk.personaId, chunk.messages_analyze.map(m => m.id), "p");
   }
 
+  const humanForScan = state.getHuman();
+  const userIdentifierTypesForScan = [...new Set(
+    humanForScan.people
+      .flatMap(p => (p.identifiers ?? []).map(i => i.type))
+      .filter(Boolean)
+  )];
+
   for (const chunk of chunks) {
     const prompt = buildHumanPersonScanPrompt({
       persona_name: chunk.personaDisplayName,
       messages_context: chunk.messages_context,
       messages_analyze: chunk.messages_analyze,
+      participant_context: buildParticipantContext(context.personaId, state),
+      known_identifier_types: userIdentifierTypesForScan,
     });
 
     state.queue_enqueue({
@@ -364,88 +372,6 @@ export async function queueTopicMatch(
   });
 }
 
-/**
- * Queue a person match request using embedding-based similarity (people only).
- */
-export async function queuePersonMatch(
-  candidate: PersonScanCandidate,
-  context: ExtractionContext,
-  state: StateManager,
-  extractionModel?: string
-): Promise<void> {
-  const human = state.getHuman();
-
-  const peopleWithEmbeddings = human.people.filter(p => p.embedding && p.embedding.length > 0);
-
-  let topKItems: Array<{ id: string; name: string; description: string; relationship?: string }> = [];
-
-  if (peopleWithEmbeddings.length > 0) {
-    try {
-      const embeddingService = getEmbeddingService();
-      const candidateText = getPersonEmbeddingText({
-        name: candidate.name,
-        relationship: candidate.relationship,
-        description: candidate.description,
-      });
-      const candidateVector = await embeddingService.embed(candidateText);
-
-      const topK = findTopK(candidateVector, peopleWithEmbeddings, EMBEDDING_TOP_K);
-      topKItems = topK
-        .filter(({ similarity }) => similarity >= EMBEDDING_MIN_SIMILARITY)
-        .map(({ item }) => ({
-          id: item.id,
-          name: item.name,
-          description: item.description,
-          relationship: item.relationship,
-        }));
-
-      console.log(`[queuePersonMatch] Embedding search: ${peopleWithEmbeddings.length} people → ${topKItems.length} candidates`);
-      if (topKItems.length > 0) state.embedding_setWarning(false);
-    } catch (err) {
-      console.error(`[queuePersonMatch] Embedding search failed, falling back to recent people:`, err);
-      state.embedding_setWarning(true);
-    }
-  }
-
-  if (topKItems.length === 0) {
-    const sorted = [...human.people].sort((a, b) => {
-      const aDate = a.last_mentioned ?? a.last_updated;
-      const bDate = b.last_mentioned ?? b.last_updated;
-      return bDate.localeCompare(aDate);
-    });
-    topKItems = sorted.slice(0, EMBEDDING_TOP_K).map(p => ({
-      id: p.id,
-      name: p.name,
-      description: p.description,
-      relationship: p.relationship,
-    }));
-    console.log(`[queuePersonMatch] No embedding matches, using ${topKItems.length} most-recent people`);
-  }
-
-  const prompt = buildPersonMatchPrompt({
-    candidate_name: candidate.name,
-    candidate_description: candidate.description,
-    candidate_relationship: candidate.relationship,
-    existing_people: topKItems,
-  });
-
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Normal,
-    model: extractionModel,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonMatch,
-    data: {
-      ...context,
-      candidateName: candidate.name,
-      candidateDescription: candidate.description,
-      candidateRelationship: candidate.relationship,
-      extraction_model: extractionModel,
-    },
-  });
-}
-
 export function queueTopicUpdate(
   matchResult: ItemMatchResult,
   context: ExtractionContext & {
@@ -592,6 +518,7 @@ export function queuePersonUpdate(
     candidateName: string;
     candidateDescription: string;
     candidateRelationship: string;
+    candidateIdentifiers?: PersonIdentifier[];
     extraction_model?: string;
   },
   state: StateManager
@@ -605,11 +532,32 @@ export function queuePersonUpdate(
     existingItem = human.people.find(p => p.id === matchedGuid) ?? null;
   }
 
+  const candidateIdentifiers = context.candidateIdentifiers ?? [];
+
+  if (!isNewItem && existingItem && candidateIdentifiers.length > 0) {
+    const merged = [...(existingItem.identifiers ?? [])];
+    for (const ci of candidateIdentifiers) {
+      if (!merged.some(ei => ei.value === ci.value)) {
+        merged.push(ci);
+      }
+    }
+    existingItem = { ...existingItem, identifiers: merged };
+    state.human_person_upsert(existingItem);
+  }
+
+  const userIdentifierTypes = [...new Set(
+    human.people
+      .flatMap(p => (p.identifiers ?? []).map(i => i.type))
+      .filter(Boolean)
+  )];
+
   const extractionOptions: ExtractionOptions = { extraction_model: context.extraction_model };
   const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state, extractionOptions));
 
   if (chunks.length === 0) return 0;
 
+  const primaryPersonaIdForUpdate = context.personaId.split("|")[0];
+
   for (const chunk of chunks) {
     const prompt = buildPersonUpdatePrompt({
       existing_item: existingItem,
@@ -619,6 +567,8 @@ export function queuePersonUpdate(
       messages_context: chunk.messages_context,
       messages_analyze: chunk.messages_analyze,
       persona_name: chunk.personaDisplayName,
+      participant_context: buildParticipantContext(primaryPersonaIdForUpdate, state),
+      known_identifier_types: userIdentifierTypes,
     });
 
     state.queue_enqueue({
@@ -634,7 +584,9 @@ export function queuePersonUpdate(
         roomId: context.roomId,
         isNewItem,
         existingItemId: existingItem?.id,
+        candidateName: context.candidateName,
         candidateRelationship: context.candidateRelationship,
+        candidateIdentifiers: isNewItem ? candidateIdentifiers : undefined,
         analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
       },
     });

package/src/core/orchestrators/index.ts

@@ -6,7 +6,6 @@ export {
   queueAllScans,
   queueTopicMatch,
   queueTopicUpdate,
-  queuePersonMatch,
   queuePersonUpdate,
   queueEventSummary,
   type ExtractionContext,
@@ -17,16 +16,13 @@ export {
   startCeremony,
   handleCeremonyProgress,
   prunePersonaMessages,
-  queueExpirePhase,
-  queueExplorePhase,
-  queueDescriptionCheck,
   runHumanCeremony,
   } from "./ceremony.js";
-export { queueDedupPhase, queueUserDedupRequest } from "./dedup-phase.js";
+export { queueUserDedupRequest } from "./dedup-phase.js";
+export { queuePersonMigration } from "./person-migration.js";
 export {
-  queuePersonaTopicScan,
-  queuePersonaTopicMatch,
-  queuePersonaTopicUpdate,
+  queuePersonaTopicRating,
   type PersonaTopicContext,
+  type PersonaTopicOptions,
 } from "./persona-topics.js";
 export { queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction } from "./room-extraction.js";

package/src/core/orchestrators/person-migration.ts

@@ -0,0 +1,55 @@
+import { LLMRequestType, LLMPriority, LLMNextStep } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import { buildPersonMigrationPrompt } from "../../prompts/ceremony/index.js";
+
+export function queuePersonMigration(state: StateManager): void {
+  const human = state.getHuman();
+
+  if (human.settings?.people_migration_complete) {
+    console.log("[PersonMigration] Migration complete flag set — skipping");
+    return;
+  }
+
+  const unmigrated = human.people.filter(p => !p.identifiers || p.identifiers.length === 0);
+
+  if (unmigrated.length === 0) {
+    console.log("[PersonMigration] All Person records have identifiers — marking migration complete");
+    state.setHuman({
+      ...human,
+      settings: {
+        ...human.settings,
+        people_migration_complete: true,
+      },
+    });
+    return;
+  }
+
+  console.log(`[PersonMigration] Queuing migration for ${unmigrated.length} Person record(s)`);
+
+  const rewriteModel = human.settings?.rewrite_model;
+
+  for (const person of unmigrated) {
+    const prompt = buildPersonMigrationPrompt({
+      person: {
+        name: person.name,
+        description: person.description,
+        relationship: person.relationship,
+      },
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Normal,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandlePersonIdentifierMigration,
+      ...(rewriteModel ? { model: rewriteModel } : {}),
+      data: {
+        person_id: person.id,
+        ceremony_progress: 1,
+      },
+    });
+  }
+
+  console.log(`[PersonMigration] Queued ${unmigrated.length} migration request(s)`);
+}

package/src/core/orchestrators/persona-topics.ts

@@ -1,118 +1,93 @@
 import { LLMRequestType, LLMPriority, LLMNextStep, type Message, type PersonaTopic } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import {
-  buildPersonaTopicScanPrompt,
-  buildPersonaTopicMatchPrompt,
-  buildPersonaTopicUpdatePrompt,
-  type PersonaTopicScanCandidate,
-  type PersonaTopicMatchResult,
+  buildPersonaTopicRatingPrompt,
 } from "../../prompts/persona/index.js";
+import { chunkExtractionContext } from "./extraction-chunker.js";
+import { resolveTokenLimit } from "../llm-client.js";
 
 export interface PersonaTopicContext {
   personaId: string;
   personaDisplayName: string;
   messages_context: Message[];
   messages_analyze: Message[];
+  topics: PersonaTopic[];
 }
 
-function getAnalyzeFromTimestamp(context: PersonaTopicContext): string | null {
-  if (context.messages_analyze.length === 0) return null;
-  return context.messages_analyze[0].timestamp;
+export interface PersonaTopicOptions {
+  ceremony_progress?: number;
+  roomId?: string;
 }
 
-export function queuePersonaTopicScan(context: PersonaTopicContext, state: StateManager): void {
-  const prompt = buildPersonaTopicScanPrompt({
-    persona_name: context.personaDisplayName,
-    messages_context: context.messages_context,
-    messages_analyze: context.messages_analyze,
-  });
+const EXTRACTION_BUDGET_RATIO = 0.75;
+const MIN_EXTRACTION_TOKENS = 10000;
 
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonaTopicScan,
-    data: {
-      personaId: context.personaId,
-      personaDisplayName: context.personaDisplayName,
-      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
-    },
-  });
+function getExtractionMaxTokens(state: StateManager): number {
+  const human = state.getHuman();
+  const modelForTokenLimit = human.settings?.default_model;
+  const tokenLimit = resolveTokenLimit(modelForTokenLimit, human.settings?.accounts);
+  return Math.max(MIN_EXTRACTION_TOKENS, Math.floor(tokenLimit * EXTRACTION_BUDGET_RATIO));
 }
 
-export function queuePersonaTopicMatch(
-  candidate: PersonaTopicScanCandidate,
+export function queuePersonaTopicRating(
   context: PersonaTopicContext,
-  state: StateManager
+  state: StateManager,
+  options?: PersonaTopicOptions
 ): void {
-  const persona = state.persona_getById(context.personaId);
-  if (!persona) {
-    console.error(`[queuePersonaTopicMatch] Persona not found: ${context.personaId}`);
-    return;
-  }
-
-  const prompt = buildPersonaTopicMatchPrompt({
-    persona_name: context.personaDisplayName,
-    candidate,
-    existing_topics: persona.topics,
-  });
-
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonaTopicMatch,
-    data: {
+  const maxTokens = getExtractionMaxTokens(state);
+  const { chunks } = chunkExtractionContext(
+    {
       personaId: context.personaId,
       personaDisplayName: context.personaDisplayName,
-      candidate,
-      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
+      messages_context: context.messages_context,
+      messages_analyze: context.messages_analyze,
     },
-  });
-}
+    maxTokens
+  );
 
-export function queuePersonaTopicUpdate(
-  candidate: PersonaTopicScanCandidate,
-  matchResult: PersonaTopicMatchResult,
-  context: PersonaTopicContext,
-  state: StateManager
-): void {
-  const persona = state.persona_getById(context.personaId);
-  if (!persona) {
-    console.error(`[queuePersonaTopicUpdate] Persona not found: ${context.personaId}`);
+  if (chunks.length === 0) {
+    console.log(`[queuePersonaTopicRating] No chunks to process for ${context.personaDisplayName}`);
     return;
   }
 
-  const existingTopic = matchResult.matched_id
-    ? persona.topics.find((t: PersonaTopic) => t.id === matchResult.matched_id)
-    : undefined;
+  // Mark messages BEFORE queueing to prevent duplicate queueing
+  const shortId = context.personaId.slice(0, 8);
+  const allAnalyzeIds = context.messages_analyze.map(m => m.id);
+  if (options?.roomId) {
+    state.markRoomMessagesPersonaExtracted(options.roomId, allAnalyzeIds, shortId);
+  } else {
+    state.messages_markPersonaExtracted(context.personaId, allAnalyzeIds, shortId);
+  }
 
-  const prompt = buildPersonaTopicUpdatePrompt({
-    persona_name: context.personaDisplayName,
-    short_description: persona.short_description,
-    long_description: persona.long_description,
-    traits: persona.traits,
-    existing_topic: existingTopic,
-    candidate,
-    messages_context: context.messages_context,
-    messages_analyze: context.messages_analyze,
-  });
+  for (const chunk of chunks) {
+    const topicsForPrompt = context.topics.map(t => ({
+      id: t.id,
+      name: t.name,
+      description_hint: t.perspective?.slice(0, 80) || t.name,
+    }));
 
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonaTopicUpdate,
-    data: {
-      personaId: context.personaId,
-      personaDisplayName: context.personaDisplayName,
-      candidate,
-      existingTopicId: existingTopic?.id ?? null,
-      isNewTopic: !existingTopic,
-      analyze_from_timestamp: getAnalyzeFromTimestamp(context),
-    },
-  });
+    const prompt = buildPersonaTopicRatingPrompt({
+      persona_name: context.personaDisplayName,
+      topics: topicsForPrompt,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+    });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandlePersonaTopicRating,
+      data: {
+        personaId: context.personaId,
+        personaDisplayName: context.personaDisplayName,
+        message_ids: chunk.messages_analyze.map(m => m.id),
+        ceremony_progress: options?.ceremony_progress,
+        roomId: options?.roomId,
+      },
+    });
+  }
+
+  console.log(`[queuePersonaTopicRating] Queued ${chunks.length} rating chunk(s) for ${context.personaDisplayName}`);
 }

package/src/core/orchestrators/room-extraction.ts

@@ -17,6 +17,7 @@ import {
   queueEventSummary,
   type ExtractionContext as HumanExtractionContext,
 } from "./human-extraction.js";
+import { queuePersonaTopicRating, type PersonaTopicContext } from "./persona-topics.js";
 
 const EXTRACTION_BUDGET_RATIO = 0.75;
 const MIN_EXTRACTION_TOKENS = 10000;
@@ -269,6 +270,24 @@ export function queueRoomCapture(state: StateManager, roomId: string): void {
   }
   queueRoomEventScan(roomId, roomDisplayName, allVisible, state, participantContext);
 
+  for (const personaId of room.persona_ids) {
+    const shortId = personaId.slice(0, 8);
+    const unprocessedRaw = state.getRoomUnextractedMessagesForPersona(roomId, shortId);
+    if (unprocessedRaw.length === 0) continue;
+    const personaForRoom = state.persona_getById(personaId);
+    if (!personaForRoom) continue;
+    const processedIds = new Set(allVisible.filter(m => !!m.persona_extracted?.[shortId]).map(m => m.id));
+    const personaTopicContext: PersonaTopicContext = {
+      personaId,
+      personaDisplayName: personaForRoom.display_name,
+      messages_context: allVisible.filter(m => processedIds.has(m.id)),
+      messages_analyze: normalizeRoomMessages(unprocessedRaw, state),
+      topics: personaForRoom.topics,
+    };
+    queuePersonaTopicRating(personaTopicContext, state, { roomId: roomId });
+    console.log(`[queueRoomCapture] Queued persona topic scan: ${personaForRoom.display_name} (${unprocessedRaw.length} messages)`);
+  }
+
   console.log(`[queueRoomCapture] Queued extraction for room ${roomDisplayName}`);
 }
 
@@ -314,5 +333,20 @@ export function queuePersonaCapture(state: StateManager, personaId: string): voi
 
   queueEventSummary(personaId, state, options);
 
+  const shortId = personaId.slice(0, 8);
+  const unprocessedForPersona = state.messages_getUnextractedForPersona(personaId, shortId);
+  if (unprocessedForPersona.length > 0) {
+    const processedIds = new Set(allMessages.filter(m => !!m.persona_extracted?.[shortId]).map(m => m.id));
+    const personaTopicContext: PersonaTopicContext = {
+      personaId,
+      personaDisplayName: persona.display_name,
+      messages_context: allMessages.filter(m => processedIds.has(m.id)),
+      messages_analyze: unprocessedForPersona,
+      topics: persona.topics,
+    };
+    queuePersonaTopicRating(personaTopicContext, state);
+    console.log(`[queuePersonaCapture] Queued persona topic scan for ${persona.display_name} (${unprocessedForPersona.length} messages)`);
+  }
+
   console.log(`[queuePersonaCapture] Queued extraction for persona ${persona.display_name}`);
 }

package/src/core/persona-manager.ts

@@ -7,6 +7,7 @@ import {
 } from "./types.js";
 import { StateManager } from "./state-manager.js";
 import { orchestratePersonaGeneration } from "./orchestrators/index.js";
+import { computePersonaDescriptionEmbedding } from "./embedding-service.js";
 
 export async function getPersonaList(sm: StateManager): Promise<PersonaSummary[]> {
   return sm.persona_getAll().map((entity) => ({
@@ -41,6 +42,9 @@ export async function createPersona(
       `Cannot create persona with reserved name "${input.name}". Reserved names: ${RESERVED_PERSONA_NAMES.join(", ")}`
     );
   }
+  if (!input.long_description?.trim()) {
+    throw new Error(`Persona "${input.name}" requires a long description.`);
+  }
   const now = new Date().toISOString();
   const DEFAULT_GROUP = "General";
   const personaId = crypto.randomUUID();
@@ -117,16 +121,31 @@ export async function updatePersona(
 ): Promise<boolean> {
   const persona = sm.persona_getById(personaId);
   if (!persona) return false;
+
+  if ('long_description' in updates) {
+    const merged = { ...persona, ...updates };
+    const embedding = await computePersonaDescriptionEmbedding(merged);
+    if (embedding) {
+      updates = { ...updates, description_embedding: embedding };
+    }
+  }
+
   sm.persona_update(personaId, updates);
   return true;
 }
 
 export async function getGroupList(sm: StateManager): Promise<string[]> {
-  const personas = sm.persona_getAll();
   const groups = new Set<string>();
-  for (const p of personas) {
+
+  for (const p of sm.persona_getAll()) {
     if (p.group_primary) groups.add(p.group_primary);
     for (const g of p.groups_visible || []) groups.add(g);
   }
+
+  const human = sm.getHuman();
+  for (const item of [...(human.topics || []), ...(human.people || []), ...(human.facts || [])]) {
+    for (const g of item.persona_groups || []) groups.add(g);
+  }
+
   return [...groups].sort();
 }

package/src/core/personas/opencode-agent.ts

@@ -56,6 +56,7 @@ export async function ensureAgentPersona(
     sentiment: t.sentiment,
     strength: t.strength,
     last_updated: now,
+    learned_on: now,
   }));
   const persona: PersonaEntity = {
     id: personaId,