ei-tui 0.4.3 → 0.5.0

package/src/core/handlers/rooms.ts

@@ -0,0 +1,176 @@
+/**
+ * Room Response Handlers
+ */
+
+import { ContextStatus, LLMNextStep, LLMPriority, LLMRequestType } from "../types.js";
+import type { LLMResponse, RoomMessage } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import type { PersonaResponseResult } from "../../prompts/response/index.js";
+import type { RoomJudgeResult } from "../../prompts/room/index.js";
+import { buildRoomResponsePromptData } from "../prompt-context-builder.js";
+
+export function handleRoomResponse(response: LLMResponse, state: StateManager): void {
+  const roomId = response.request.data.roomId as string;
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+  const parentMessageId = response.request.data.parentMessageId as string | null ?? null;
+
+  if (!roomId || !personaId) {
+    console.error("[handleRoomResponse] Missing roomId or personaId in request data");
+    return;
+  }
+
+  const now = new Date().toISOString();
+
+  if (response.parsed !== undefined) {
+    const result = response.parsed as PersonaResponseResult;
+
+    if (!result.should_respond) {
+      const reason = result.reason;
+      console.log(`[handleRoomResponse] ${personaDisplayName} chose silence in room ${roomId}: ${reason ?? "(no reason)"}`);
+      if (reason) {
+        const msg: RoomMessage = {
+          id: crypto.randomUUID(),
+          parent_id: parentMessageId,
+          role: "persona",
+          persona_id: personaId,
+          silence_reason: reason,
+          timestamp: now,
+          read: false,
+          context_status: ContextStatus.Default,
+        };
+        state.appendRoomMessage(roomId, msg);
+      }
+      return;
+    }
+
+    const verbal = result.verbal_response || undefined;
+    const action = result.action_response || undefined;
+
+    if (!verbal && !action) {
+      console.log(`[handleRoomResponse] ${personaDisplayName} returned should_respond=true but no content`);
+      return;
+    }
+
+    const msg: RoomMessage = {
+      id: crypto.randomUUID(),
+      parent_id: parentMessageId,
+      role: "persona",
+      persona_id: personaId,
+      verbal_response: verbal,
+      action_response: action,
+      timestamp: now,
+      read: false,
+      context_status: ContextStatus.Default,
+    };
+    state.appendRoomMessage(roomId, msg);
+    console.log(`[handleRoomResponse] Appended response from ${personaDisplayName} to room ${roomId}`);
+    return;
+  }
+
+  if (!response.content) {
+    console.log(`[handleRoomResponse] ${personaDisplayName} no response (empty content)`);
+    return;
+  }
+
+  const msg: RoomMessage = {
+    id: crypto.randomUUID(),
+    parent_id: parentMessageId,
+    role: "persona",
+    persona_id: personaId,
+    verbal_response: response.content,
+    timestamp: now,
+    read: false,
+    context_status: ContextStatus.Default,
+  };
+  state.appendRoomMessage(roomId, msg);
+  console.log(`[handleRoomResponse] Appended plain-text response from ${personaDisplayName} to room ${roomId}`);
+}
+
+export async function handleRoomJudge(response: LLMResponse, state: StateManager): Promise<void> {
+  const roomId = response.request.data.roomId as string;
+  const judgeDisplayName = response.request.data.judgePersonaDisplayName as string;
+
+  if (!roomId) {
+    console.error("[handleRoomJudge] Missing roomId in request data");
+    return;
+  }
+
+  if (!response.parsed) {
+    console.error(`[handleRoomJudge] No parsed result from judge ${judgeDisplayName}`);
+    return;
+  }
+
+  const result = response.parsed as RoomJudgeResult;
+  if (!result.winner_message_id) {
+    console.error(`[handleRoomJudge] Judge ${judgeDisplayName} returned no winner_message_id`);
+    return;
+  }
+
+  const judgePersonaId = response.request.data.judgePersonaId as string;
+
+  const allMessages = state.getRoomMessages(roomId);
+  const winner = allMessages.find(m => m.id === result.winner_message_id);
+  if (!winner) {
+    console.error(`[handleRoomJudge] Winner message ${result.winner_message_id} not found in room ${roomId}`);
+    return;
+  }
+
+  const verdictParentId = winner.parent_id;
+
+  const ok = state.setRoomActiveNode(roomId, result.winner_message_id);
+  if (!ok) {
+    console.error(`[handleRoomJudge] Could not set active node ${result.winner_message_id} in room ${roomId}`);
+    return;
+  }
+
+  const losers = allMessages
+    .filter(m => m.parent_id === verdictParentId && m.id !== winner.id)
+    .map(m => m.id);
+  if (losers.length > 0) {
+    state.removeRoomMessages(roomId, losers);
+  }
+
+  if (result.reason) {
+    console.log(`[handleRoomJudge] ${judgeDisplayName} verdict: ${result.reason}`);
+    const verdictMsg = {
+      id: crypto.randomUUID(),
+      parent_id: verdictParentId,
+      role: "persona" as const,
+      persona_id: judgePersonaId,
+      silence_reason: result.reason,
+      timestamp: new Date().toISOString(),
+      read: false,
+      context_status: "default" as import("../types.js").ContextStatus,
+    };
+    state.appendRoomMessage(roomId, verdictMsg);
+  }
+
+  const room = state.getRoom(roomId);
+  if (!room) return;
+
+  for (const personaId of room.persona_ids) {
+    if (room.judge_persona_id === personaId) continue;
+    const persona = state.persona_getById(personaId);
+    if (!persona || persona.is_archived || persona.is_paused) continue;
+
+    const isTUI = false;
+    const promptData = await buildRoomResponsePromptData(state, room, persona, isTUI);
+    const model = persona.model ?? state.getHuman().settings?.default_model ?? "";
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Room,
+      system: promptData.system,
+      user: promptData.user,
+      next_step: LLMNextStep.HandleRoomResponse,
+      model,
+      data: {
+        roomId,
+        personaId,
+        personaDisplayName: persona.display_name,
+        parentMessageId: result.winner_message_id,
+      },
+    });
+  }
+}

package/src/core/handlers/utils.ts

@@ -1,12 +1,53 @@
-import type { Message, LLMResponse } from "../types.js";
+import type { Message, RoomMessage, LLMResponse } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 
+export function normalizeRoomMessages(messages: RoomMessage[], state: StateManager): Message[] {
+  const human = state.getHuman();
+  const humanName = human.settings?.name_display ?? "Human";
+  return messages.map(m => {
+    const speakerName = m.role === "human"
+      ? humanName
+      : (state.persona_getById(m.persona_id ?? "")?.display_name ?? "Participant");
+    return {
+      id: m.id,
+      role: m.role === "human" ? "human" as const : "system" as const,
+      speaker_name: speakerName,
+      verbal_response: m.verbal_response,
+      action_response: m.action_response,
+      silence_reason: m.silence_reason,
+      timestamp: m.timestamp,
+      read: m.read,
+      context_status: m.context_status,
+      f: m.f,
+      t: m.t,
+      p: m.p,
+      e: m.e,
+    };
+  });
+}
+
 export function resolveMessageWindow(
   response: LLMResponse,
   state: StateManager
 ): { messages_context: Message[]; messages_analyze: Message[] } {
-  const personaId = response.request.data.personaId as string;
+  const roomId = response.request.data.roomId as string | undefined;
   const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
+
+  if (roomId) {
+    const allRoomMessages = normalizeRoomMessages(state.getRoomMessages(roomId), state);
+    if (messageIdsToMark && messageIdsToMark.length > 0) {
+      const idSet = new Set(messageIdsToMark);
+      const messages_analyze = allRoomMessages.filter(m => idSet.has(m.id));
+      const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
+      const messages_context = allRoomMessages.filter(m =>
+        !idSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
+      );
+      return { messages_context, messages_analyze };
+    }
+    return { messages_context: [], messages_analyze: allRoomMessages };
+  }
+
+  const personaId = response.request.data.personaId as string;
   const allMessages = state.messages_get(personaId);
 
   if (messageIdsToMark && messageIdsToMark.length > 0) {
@@ -48,10 +89,21 @@ export function markMessagesExtracted(
   state: StateManager, 
   flag: ExtractionFlag
 ): void {
+  const roomId = response.request.data.roomId as string | undefined;
   const personaId = response.request.data.personaId as string | undefined;
   const messageIds = response.request.data.message_ids_to_mark as string[] | undefined;
   
-  if (!personaId || !messageIds?.length) return;
+  if (!messageIds?.length) return;
+
+  if (roomId) {
+    const count = state.markRoomMessagesExtracted(roomId, messageIds, flag);
+    if (count > 0) {
+      console.log(`[markMessagesExtracted] Marked ${count} room messages with flag '${flag}' for room ${roomId}`);
+    }
+    return;
+  }
+
+  if (!personaId) return;
   
   const count = state.messages_markExtracted(personaId, messageIds, flag);
   if (count > 0) {

package/src/core/heartbeat-manager.ts

@@ -158,9 +158,11 @@ export async function queueEiHeartbeat(
     return;
   }
 
+  const recentHistory = history.slice(-10);
   const promptData: EiHeartbeatPromptData = {
     items,
-    recent_history: history.slice(-10),
+    recent_history: recentHistory,
+    system_messages: recentHistory.filter(m => m.role === "system"),
   };
 
   const prompt = buildEiHeartbeatPrompt(promptData);

package/src/core/llm-client.ts

@@ -159,7 +159,7 @@ export async function callLLMRaw(
   const chatMessages: ChatMessage[] = [
     { role: "system", content: systemPrompt },
     ...messages,
-    { role: "user", content: userPrompt },
+    ...(userPrompt ? [{ role: "user" as const, content: userPrompt }] : []),
   ];
   
   const finalMessages = ensureUserFirst(chatMessages);

package/src/core/message-manager.ts

@@ -36,7 +36,7 @@ export async function getMessages(
 ): Promise<Message[]> {
   const persona = sm.persona_getById(personaId);
   if (!persona) return [];
-  return sm.messages_get(personaId);
+  return sm.messages_get(personaId).filter(m => m.external !== true);
 }
 
 export async function markMessageRead(
@@ -128,12 +128,13 @@ export async function sendMessage(
   qp: QueueProcessor,
   currentRequest: LLMRequest | null,
   personaId: string,
-  content: string,
+  content: string | null,
   isTUI: boolean,
   getModelForPersona: (id?: string) => string | undefined,
   onError: (err: { code: string; message: string }) => void,
   onMessageAdded: (id: string) => void,
-  onMessageQueued: (id: string) => void
+  onMessageQueued: (id: string) => void,
+  silenceReason?: string
 ): Promise<void> {
   const persona = sm.persona_getById(personaId);
   if (!persona) {
@@ -149,7 +150,8 @@ export async function sendMessage(
   const message: Message = {
     id: crypto.randomUUID(),
     role: "human",
-    verbal_response: content,
+    verbal_response: content ?? undefined,
+    silence_reason: content ? undefined : (silenceReason ?? "passed"),
     timestamp: new Date().toISOString(),
     read: false,
     context_status: "default" as ContextStatus,
@@ -158,7 +160,7 @@ export async function sendMessage(
   onMessageAdded(persona.id);
 
   const tools = sm.tools_getForPersona(persona.id, isTUI);
-  const promptData = await buildResponsePromptData(sm, persona, isTUI, content, tools);
+  const promptData = await buildResponsePromptData(sm, persona, isTUI, content ?? "", tools);
   const prompt = buildResponsePrompt(promptData);
 
   sm.queue_enqueue({
@@ -215,7 +217,7 @@ export function checkAndQueueHumanExtraction(
 ): void {
   const human = sm.getHuman();
 
-  const unextractedFacts = sm.messages_getUnextracted(personaId, "f");
+  const unextractedFacts = sm.messages_getUnextracted(personaId, "f", undefined, "exclude");
   const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.filter(f => f.description && f.description !== "").length);
   if (unextractedFacts.length > 0 && unextractedFacts.length >= factsThreshold) {
     const context: ExtractionContext = {
@@ -231,7 +233,7 @@ export function checkAndQueueHumanExtraction(
     );
   }
 
-  const unextractedTopics = sm.messages_getUnextracted(personaId, "t");
+  const unextractedTopics = sm.messages_getUnextracted(personaId, "t", undefined, "exclude");
   const topicsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.topics.length);
   if (unextractedTopics.length > 0 && unextractedTopics.length >= topicsThreshold) {
     const context: ExtractionContext = {
@@ -247,7 +249,7 @@ export function checkAndQueueHumanExtraction(
     );
   }
 
-  const unextractedPeople = sm.messages_getUnextracted(personaId, "p");
+  const unextractedPeople = sm.messages_getUnextracted(personaId, "p", undefined, "exclude");
   const peopleThreshold = Math.min(EXTRACTION_TAPER_CAP, human.people.length);
   if (unextractedPeople.length > 0 && unextractedPeople.length >= peopleThreshold) {
     const context: ExtractionContext = {

package/src/core/orchestrators/human-extraction.ts

@@ -58,6 +58,7 @@ export interface ExtractionContext {
   messages_context: Message[];
   messages_analyze: Message[];
   extraction_flag?: "f" | "t" | "p" | "e";
+  roomId?: string;
 }
 
 export interface ExtractionOptions {
@@ -67,6 +68,15 @@ export interface ExtractionOptions {
   extraction_model?: string;
   /** Override token budget for chunking */
   extraction_token_limit?: number;
+  /**
+   * Controls whether external (integration-imported) messages are included.
+   * - "exclude": skip messages where external === true
+   * - "only": include ONLY messages where external === true
+   * - "include": include all messages (backward-compat default; omit means same)
+   *
+   * NOTE: "include" is the backward-compat default only. All new callers must explicitly pass "exclude" or "only". Will be removed in a future release.
+   */
+  external_filter?: "include" | "exclude" | "only";
 }
 
 function getAnalyzeFromTimestamp(context: ExtractionContext): string | null {
@@ -466,6 +476,7 @@ export function queueTopicUpdate(
   if (chunks.length === 0) return 0;
 
   for (const chunk of chunks) {
+    const primaryPersonaId = context.personaId.split("|")[0];
     const prompt = buildTopicUpdatePrompt({
       existing_item: existingItem,
       new_topic_name: isNewItem ? context.candidateName : undefined,
@@ -474,7 +485,7 @@ export function queueTopicUpdate(
       messages_context: chunk.messages_context,
       messages_analyze: chunk.messages_analyze,
       persona_name: chunk.personaDisplayName,
-      participant_context: buildParticipantContext(context.personaId, state),
+      participant_context: buildParticipantContext(primaryPersonaId, state),
     });
 
     state.queue_enqueue({
@@ -487,6 +498,7 @@ export function queueTopicUpdate(
       data: {
         personaId: context.personaId,
         personaDisplayName: context.personaDisplayName,
+        roomId: context.roomId,
         isNewItem,
         existingItemId: existingItem?.id,
         candidateCategory: context.candidateCategory,
@@ -509,7 +521,7 @@ export function queueEventSummary(
     return 0;
   }
 
-  const unextractedMessages = state.messages_getUnextracted(personaId, "e");
+  const unextractedMessages = state.messages_getUnextracted(personaId, "e", undefined, options?.external_filter);
   if (unextractedMessages.length === 0) {
     console.log(`[queueEventSummary] No unprocessed messages for ${persona.display_name}`);
     return 0;
@@ -624,6 +636,7 @@ export function queuePersonUpdate(
       data: {
         personaId: context.personaId,
         personaDisplayName: context.personaDisplayName,
+        roomId: context.roomId,
         isNewItem,
         existingItemId: existingItem?.id,
         candidateRelationship: context.candidateRelationship,

package/src/core/orchestrators/index.ts

@@ -29,3 +29,4 @@ export {
   queuePersonaTopicUpdate,
   type PersonaTopicContext,
 } from "./persona-topics.js";
+export { queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction } from "./room-extraction.js";

package/src/core/orchestrators/persona-generation.ts

@@ -39,12 +39,16 @@ export function orchestratePersonaGeneration(
   const needsMoreTopics = topicCount < 3;
 
   if (needsShortDescription || needsMoreTraits || needsMoreTopics) {
+    const filteredTraits = (partial.traits ?? []).filter(t => t.name?.trim());
+    const filteredTopics = (partial.topics ?? []).filter(t => t.name?.trim());
     const prompt = buildPersonaGenerationPrompt({
       name: partial.name,
       long_description: partial.long_description,
       short_description: partial.short_description,
       existing_traits: partial.traits,
       existing_topics: partial.topics,
+      filtered_traits: filteredTraits,
+      filtered_topics: filteredTopics,
     });
 
     stateManager.queue_enqueue({

package/src/core/orchestrators/persona-topics.ts

@@ -110,7 +110,8 @@ export function queuePersonaTopicUpdate(
       personaId: context.personaId,
       personaDisplayName: context.personaDisplayName,
       candidate,
-      matched_id: matchResult.matched_id,
+      existingTopicId: existingTopic?.id ?? null,
+      isNewTopic: !existingTopic,
       analyze_from_timestamp: getAnalyzeFromTimestamp(context),
     },
   });