ei-tui 0.4.3 → 0.5.1

package/src/core/message-manager.ts

@@ -36,7 +36,7 @@ export async function getMessages(
 ): Promise<Message[]> {
   const persona = sm.persona_getById(personaId);
   if (!persona) return [];
-  return sm.messages_get(personaId);
+  return sm.messages_get(personaId).filter(m => m.external !== true);
 }
 
 export async function markMessageRead(
@@ -128,12 +128,13 @@ export async function sendMessage(
   qp: QueueProcessor,
   currentRequest: LLMRequest | null,
   personaId: string,
-  content: string,
+  content: string | null,
   isTUI: boolean,
   getModelForPersona: (id?: string) => string | undefined,
   onError: (err: { code: string; message: string }) => void,
   onMessageAdded: (id: string) => void,
-  onMessageQueued: (id: string) => void
+  onMessageQueued: (id: string) => void,
+  silenceReason?: string
 ): Promise<void> {
   const persona = sm.persona_getById(personaId);
   if (!persona) {
@@ -149,7 +150,8 @@ export async function sendMessage(
   const message: Message = {
     id: crypto.randomUUID(),
     role: "human",
-    verbal_response: content,
+    verbal_response: content ?? undefined,
+    silence_reason: content ? undefined : (silenceReason ?? "passed"),
     timestamp: new Date().toISOString(),
     read: false,
     context_status: "default" as ContextStatus,
@@ -158,7 +160,7 @@ export async function sendMessage(
   onMessageAdded(persona.id);
 
   const tools = sm.tools_getForPersona(persona.id, isTUI);
-  const promptData = await buildResponsePromptData(sm, persona, isTUI, content, tools);
+  const promptData = await buildResponsePromptData(sm, persona, isTUI, content ?? "", tools);
   const prompt = buildResponsePrompt(promptData);
 
   sm.queue_enqueue({
@@ -215,7 +217,7 @@ export function checkAndQueueHumanExtraction(
 ): void {
   const human = sm.getHuman();
 
-  const unextractedFacts = sm.messages_getUnextracted(personaId, "f");
+  const unextractedFacts = sm.messages_getUnextracted(personaId, "f", undefined, "exclude");
   const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.filter(f => f.description && f.description !== "").length);
   if (unextractedFacts.length > 0 && unextractedFacts.length >= factsThreshold) {
     const context: ExtractionContext = {
@@ -231,7 +233,7 @@ export function checkAndQueueHumanExtraction(
     );
   }
 
-  const unextractedTopics = sm.messages_getUnextracted(personaId, "t");
+  const unextractedTopics = sm.messages_getUnextracted(personaId, "t", undefined, "exclude");
   const topicsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.topics.length);
   if (unextractedTopics.length > 0 && unextractedTopics.length >= topicsThreshold) {
     const context: ExtractionContext = {
@@ -247,7 +249,7 @@ export function checkAndQueueHumanExtraction(
     );
   }
 
-  const unextractedPeople = sm.messages_getUnextracted(personaId, "p");
+  const unextractedPeople = sm.messages_getUnextracted(personaId, "p", undefined, "exclude");
   const peopleThreshold = Math.min(EXTRACTION_TAPER_CAP, human.people.length);
   if (unextractedPeople.length > 0 && unextractedPeople.length >= peopleThreshold) {
     const context: ExtractionContext = {

package/src/core/orchestrators/human-extraction.ts

@@ -58,6 +58,7 @@ export interface ExtractionContext {
   messages_context: Message[];
   messages_analyze: Message[];
   extraction_flag?: "f" | "t" | "p" | "e";
+  roomId?: string;
 }
 
 export interface ExtractionOptions {
@@ -67,6 +68,15 @@ export interface ExtractionOptions {
   extraction_model?: string;
   /** Override token budget for chunking */
   extraction_token_limit?: number;
+  /**
+   * Controls whether external (integration-imported) messages are included.
+   * - "exclude": skip messages where external === true
+   * - "only": include ONLY messages where external === true
+   * - "include": include all messages (backward-compat default; omit means same)
+   *
+   * NOTE: "include" is the backward-compat default only. All new callers must explicitly pass "exclude" or "only". Will be removed in a future release.
+   */
+  external_filter?: "include" | "exclude" | "only";
 }
 
 function getAnalyzeFromTimestamp(context: ExtractionContext): string | null {
@@ -466,6 +476,7 @@ export function queueTopicUpdate(
   if (chunks.length === 0) return 0;
 
   for (const chunk of chunks) {
+    const primaryPersonaId = context.personaId.split("|")[0];
     const prompt = buildTopicUpdatePrompt({
       existing_item: existingItem,
       new_topic_name: isNewItem ? context.candidateName : undefined,
@@ -474,7 +485,7 @@ export function queueTopicUpdate(
       messages_context: chunk.messages_context,
       messages_analyze: chunk.messages_analyze,
       persona_name: chunk.personaDisplayName,
-      participant_context: buildParticipantContext(context.personaId, state),
+      participant_context: buildParticipantContext(primaryPersonaId, state),
     });
 
     state.queue_enqueue({
@@ -487,6 +498,7 @@ export function queueTopicUpdate(
       data: {
         personaId: context.personaId,
         personaDisplayName: context.personaDisplayName,
+        roomId: context.roomId,
         isNewItem,
         existingItemId: existingItem?.id,
         candidateCategory: context.candidateCategory,
@@ -509,7 +521,7 @@ export function queueEventSummary(
     return 0;
   }
 
-  const unextractedMessages = state.messages_getUnextracted(personaId, "e");
+  const unextractedMessages = state.messages_getUnextracted(personaId, "e", undefined, options?.external_filter);
   if (unextractedMessages.length === 0) {
     console.log(`[queueEventSummary] No unprocessed messages for ${persona.display_name}`);
     return 0;
@@ -624,6 +636,7 @@ export function queuePersonUpdate(
       data: {
         personaId: context.personaId,
         personaDisplayName: context.personaDisplayName,
+        roomId: context.roomId,
         isNewItem,
         existingItemId: existingItem?.id,
         candidateRelationship: context.candidateRelationship,

package/src/core/orchestrators/index.ts

@@ -29,3 +29,4 @@ export {
   queuePersonaTopicUpdate,
   type PersonaTopicContext,
 } from "./persona-topics.js";
+export { queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction } from "./room-extraction.js";

package/src/core/orchestrators/persona-generation.ts

@@ -39,12 +39,16 @@ export function orchestratePersonaGeneration(
   const needsMoreTopics = topicCount < 3;
 
   if (needsShortDescription || needsMoreTraits || needsMoreTopics) {
+    const filteredTraits = (partial.traits ?? []).filter(t => t.name?.trim());
+    const filteredTopics = (partial.topics ?? []).filter(t => t.name?.trim());
     const prompt = buildPersonaGenerationPrompt({
       name: partial.name,
       long_description: partial.long_description,
       short_description: partial.short_description,
       existing_traits: partial.traits,
       existing_topics: partial.topics,
+      filtered_traits: filteredTraits,
+      filtered_topics: filteredTopics,
     });
 
     stateManager.queue_enqueue({

package/src/core/orchestrators/persona-topics.ts

@@ -110,7 +110,8 @@ export function queuePersonaTopicUpdate(
       personaId: context.personaId,
       personaDisplayName: context.personaDisplayName,
       candidate,
-      matched_id: matchResult.matched_id,
+      existingTopicId: existingTopic?.id ?? null,
+      isNewTopic: !existingTopic,
       analyze_from_timestamp: getAnalyzeFromTimestamp(context),
     },
   });

package/src/core/orchestrators/room-extraction.ts

@@ -0,0 +1,318 @@
+import { LLMNextStep, LLMPriority, LLMRequestType, RoomMode } from "../types.js";
+import type { Message } from "../types.js";
+import type { StateManager } from "../state-manager.js";
+import {
+  buildHumanTopicScanPrompt,
+  buildHumanPersonScanPrompt,
+  buildEventScanPrompt,
+  type ParticipantContext,
+} from "../../prompts/human/index.js";
+import { chunkExtractionContext } from "./extraction-chunker.js";
+import { buildEventWindows } from "../utils/event-windows.js";
+import { resolveTokenLimit } from "../llm-client.js";
+import { normalizeRoomMessages } from "../handlers/utils.js";
+import {
+  queueTopicScan,
+  queuePersonScan,
+  queueEventSummary,
+  type ExtractionContext as HumanExtractionContext,
+} from "./human-extraction.js";
+
+const EXTRACTION_BUDGET_RATIO = 0.75;
+const MIN_EXTRACTION_TOKENS = 10000;
+
+function getExtractionMaxTokens(state: StateManager): number {
+  const human = state.getHuman();
+  const tokenLimit = resolveTokenLimit(human.settings?.default_model, human.settings?.accounts);
+  return Math.max(MIN_EXTRACTION_TOKENS, Math.floor(tokenLimit * EXTRACTION_BUDGET_RATIO));
+}
+
+function buildRoomParticipantContext(roomId: string, state: StateManager): ParticipantContext {
+  const room = state.getRoom(roomId);
+  const human = state.getHuman();
+  const humanName = human.settings?.name_display;
+
+  const fullNameFact = human.facts.find(f => f.name === "Full Name");
+  const nicknameFact = human.facts.find(f => f.name === "Nickname/Preferred Name");
+  const fullName = fullNameFact?.description || "";
+  const nickname = nicknameFact?.description || "";
+  let human_name: string | undefined;
+  if (fullName && nickname) human_name = `${fullName} (${nickname})`;
+  else if (fullName) human_name = fullName;
+  else if (nickname) human_name = nickname;
+  else if (humanName) human_name = humanName;
+
+  const participantNames = (room?.persona_ids ?? [])
+    .map(id => state.persona_getById(id)?.display_name)
+    .filter(Boolean)
+    .join(", ");
+
+  return {
+    persona_name: room?.display_name ?? roomId,
+    persona_description: participantNames
+      ? `A conversation room with participants: ${participantNames}`
+      : "A multi-persona conversation room",
+    human_name,
+    human_age: undefined,
+  };
+}
+
+function getRoomVisibleMessages(state: StateManager, roomId: string): Message[] {
+  const room = state.getRoom(roomId);
+  if (!room) return [];
+  const rawMessages = room.mode === RoomMode.FreeForAll
+    ? [...state.getRoomMessages(roomId)].sort((a, b) => a.timestamp.localeCompare(b.timestamp))
+    : state.getRoomActivePath(roomId);
+  return normalizeRoomMessages(rawMessages, state);
+}
+
+function queueRoomTopicScan(
+  roomId: string,
+  roomDisplayName: string,
+  messages_context: Message[],
+  messages_analyze: Message[],
+  state: StateManager,
+  participantContext: ParticipantContext
+): void {
+  const context: HumanExtractionContext = {
+    personaId: roomId,
+    personaDisplayName: roomDisplayName,
+    messages_context,
+    messages_analyze,
+    extraction_flag: "t",
+  };
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  if (chunks.length === 0) return;
+
+  state.markRoomMessagesExtracted(roomId, messages_analyze.map(m => m.id), "t");
+
+  for (const chunk of chunks) {
+    const prompt = buildHumanTopicScanPrompt({
+      persona_name: roomDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+      participant_context: participantContext,
+    });
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanTopicScan,
+      data: {
+        roomId,
+        personaId: (state.getRoom(roomId)?.persona_ids ?? []).join("|"),
+        personaDisplayName: roomDisplayName,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+}
+
+function queueRoomPersonScan(
+  roomId: string,
+  roomDisplayName: string,
+  messages_context: Message[],
+  messages_analyze: Message[],
+  state: StateManager
+): void {
+  const context: HumanExtractionContext = {
+    personaId: roomId,
+    personaDisplayName: roomDisplayName,
+    messages_context,
+    messages_analyze,
+    extraction_flag: "p",
+  };
+  const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+  if (chunks.length === 0) return;
+
+  state.markRoomMessagesExtracted(roomId, messages_analyze.map(m => m.id), "p");
+
+  for (const chunk of chunks) {
+    const prompt = buildHumanPersonScanPrompt({
+      persona_name: roomDisplayName,
+      messages_context: chunk.messages_context,
+      messages_analyze: chunk.messages_analyze,
+    });
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleHumanPersonScan,
+      data: {
+        roomId,
+        personaId: (state.getRoom(roomId)?.persona_ids ?? []).join("|"),
+        personaDisplayName: roomDisplayName,
+        message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+      },
+    });
+  }
+}
+
+function queueRoomEventScan(
+  roomId: string,
+  roomDisplayName: string,
+  allMessages: Message[],
+  state: StateManager,
+  participantContext: ParticipantContext
+): void {
+  const unextracted = allMessages.filter(m => !m.e);
+  if (unextracted.length === 0) return;
+
+  const human = state.getHuman();
+  const gapHours = human.settings?.ceremony?.event_window_hours ?? 8;
+  const sorted = [...unextracted].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+  const windows = buildEventWindows(sorted, gapHours);
+
+  state.markRoomMessagesExtracted(roomId, sorted.map(m => m.id), "e");
+
+  for (const windowMessages of windows) {
+    if (windowMessages.length === 0) continue;
+    const windowStartTime = new Date(windowMessages[0].timestamp).getTime();
+    const messages_context = allMessages.filter(
+      m => m.e === true && new Date(m.timestamp).getTime() < windowStartTime
+    );
+    const context: HumanExtractionContext = {
+      personaId: roomId,
+      personaDisplayName: roomDisplayName,
+      messages_context,
+      messages_analyze: windowMessages,
+      extraction_flag: "e",
+    };
+    const { chunks } = chunkExtractionContext(context, getExtractionMaxTokens(state));
+    for (const chunk of chunks) {
+      const prompt = buildEventScanPrompt({
+        persona_name: roomDisplayName,
+        messages_context: chunk.messages_context,
+        messages_analyze: chunk.messages_analyze,
+        participant_context: participantContext,
+      });
+      state.queue_enqueue({
+        type: LLMRequestType.JSON,
+        priority: LLMPriority.Low,
+        system: prompt.system,
+        user: prompt.user,
+        next_step: LLMNextStep.HandleEventScan,
+        data: {
+          roomId,
+          personaId: (state.getRoom(roomId)?.persona_ids ?? []).join("|"),
+          personaDisplayName: roomDisplayName,
+          message_ids_to_mark: chunk.messages_analyze.map(m => m.id),
+        },
+      });
+    }
+  }
+}
+
+export function checkAndQueueRoomExtraction(state: StateManager, roomId: string): void {
+  const room = state.getRoom(roomId);
+  if (!room || room.mode === RoomMode.ChooseYourPath) return;
+
+  const N = room.persona_ids.length + 1;
+  const threshold = room.mode === RoomMode.FreeForAll ? 5 * N : 10;
+
+  const allVisible = getRoomVisibleMessages(state, roomId);
+  const unextractedT = allVisible.filter(m => !m.t);
+  const unextractedP = allVisible.filter(m => !m.p);
+
+  if (unextractedT.length < threshold && unextractedP.length < threshold) return;
+
+  const participantContext = buildRoomParticipantContext(roomId, state);
+  const roomDisplayName = room.display_name;
+
+  if (unextractedT.length >= threshold) {
+    const analyzeStart = unextractedT[0].timestamp;
+    const messages_contextT = allVisible.filter(
+      m => m.t === true && new Date(m.timestamp).getTime() < new Date(analyzeStart).getTime()
+    );
+    queueRoomTopicScan(roomId, roomDisplayName, messages_contextT, unextractedT, state, participantContext);
+  }
+
+  if (unextractedP.length >= threshold) {
+    const analyzeStart = unextractedP[0].timestamp;
+    const messages_contextP = allVisible.filter(
+      m => m.p === true && new Date(m.timestamp).getTime() < new Date(analyzeStart).getTime()
+    );
+    queueRoomPersonScan(roomId, roomDisplayName, messages_contextP, unextractedP, state);
+  }
+
+  console.log(`[checkAndQueueRoomExtraction] Auto-triggered extraction for room ${roomDisplayName} (threshold: ${threshold})`);
+}
+
+export function queueRoomCapture(state: StateManager, roomId: string): void {
+  const room = state.getRoom(roomId);
+  if (!room) return;
+
+  const allVisible = getRoomVisibleMessages(state, roomId);
+  if (allVisible.length === 0) return;
+
+  const participantContext = buildRoomParticipantContext(roomId, state);
+  const roomDisplayName = room.display_name;
+
+  const unextractedT = allVisible.filter(m => !m.t);
+  const unextractedP = allVisible.filter(m => !m.p);
+  const analyzeStartT = unextractedT[0]?.timestamp ?? "9999";
+  const messages_contextT = allVisible.filter(
+    m => m.t === true && new Date(m.timestamp).getTime() < new Date(analyzeStartT).getTime()
+  );
+  const analyzeStartP = unextractedP[0]?.timestamp ?? "9999";
+  const messages_contextP = allVisible.filter(
+    m => m.p === true && new Date(m.timestamp).getTime() < new Date(analyzeStartP).getTime()
+  );
+
+  if (unextractedT.length > 0) {
+    queueRoomTopicScan(roomId, roomDisplayName, messages_contextT, unextractedT, state, participantContext);
+  }
+  if (unextractedP.length > 0) {
+    queueRoomPersonScan(roomId, roomDisplayName, messages_contextP, unextractedP, state);
+  }
+  queueRoomEventScan(roomId, roomDisplayName, allVisible, state, participantContext);
+
+  console.log(`[queueRoomCapture] Queued extraction for room ${roomDisplayName}`);
+}
+
+export function queuePersonaCapture(state: StateManager, personaId: string): void {
+  const persona = state.persona_getById(personaId);
+  if (!persona) return;
+
+  const allMessages = state.messages_get(personaId);
+  if (allMessages.length === 0) return;
+
+  const unextractedT = state.messages_getUnextracted(personaId, "t");
+  const unextractedP = state.messages_getUnextracted(personaId, "p");
+  const model = state.getHuman().settings?.default_model;
+  const options = { extraction_model: model };
+
+  if (unextractedT.length > 0) {
+    const analyzeStart = unextractedT[0].timestamp;
+    const messages_context = allMessages.filter(
+      m => m.t === true && new Date(m.timestamp).getTime() < new Date(analyzeStart).getTime()
+    );
+    const context: HumanExtractionContext = {
+      personaId,
+      personaDisplayName: persona.display_name,
+      messages_context,
+      messages_analyze: unextractedT,
+    };
+    queueTopicScan(context, state, options);
+  }
+
+  if (unextractedP.length > 0) {
+    const analyzeStart = unextractedP[0].timestamp;
+    const messages_context = allMessages.filter(
+      m => m.p === true && new Date(m.timestamp).getTime() < new Date(analyzeStart).getTime()
+    );
+    const context: HumanExtractionContext = {
+      personaId,
+      personaDisplayName: persona.display_name,
+      messages_context,
+      messages_analyze: unextractedP,
+    };
+    queuePersonScan(context, state, options);
+  }
+
+  queueEventSummary(personaId, state, options);
+
+  console.log(`[queuePersonaCapture] Queued extraction for persona ${persona.display_name}`);
+}

package/src/core/persona-manager.ts

@@ -65,11 +65,22 @@ export async function createPersona(
   };
   sm.persona_add(placeholder);
 
-  orchestratePersonaGeneration(
-    { ...input, id: personaId },
-    sm,
-    () => onPersonaUpdated(placeholder.display_name)
-  );
+  const hasTraits = input.traits && input.traits.length >= 3;
+  const hasTopics = input.topics && input.topics.length >= 3;
+  if (!hasTraits || !hasTopics) {
+    orchestratePersonaGeneration(
+      { ...input, id: personaId },
+      sm,
+      () => onPersonaUpdated(placeholder.display_name)
+    );
+  } else {
+    sm.persona_update(personaId, {
+      traits: input.traits as PersonaEntity["traits"],
+      topics: input.topics as PersonaEntity["topics"],
+      last_updated: now,
+    });
+    onPersonaUpdated(placeholder.display_name);
+  }
 
   return personaId;
 }

package/src/core/personas/opencode-agent.ts

@@ -1,8 +1,10 @@
 import type { PersonaEntity, Ei_Interface } from "../types.js";
+import type { PersonaTrait } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import type { IOpenCodeReader } from "../../integrations/opencode/types.js";
 import { AGENT_ALIASES } from "../../integrations/opencode/types.js";
 import { createOpenCodeReader } from "../../integrations/opencode/reader-factory.js";
+import { DEFAULT_SEED_TRAITS } from "../constants/seed-traits.js";
 
 const OPENCODE_GROUP = "OpenCode";
 const TWELVE_HOURS_MS = 43200000;
@@ -13,7 +15,7 @@ export interface EnsureAgentPersonaOptions {
   reader?: IOpenCodeReader;
 }
 
-function resolveCanonicalAgent(agentName: string): { canonical: string; aliases: string[] } {
+export function resolveCanonicalAgent(agentName: string): { canonical: string; aliases: string[] } {
   for (const [canonical, variants] of Object.entries(AGENT_ALIASES)) {
     if (variants.includes(agentName)) {
       return { canonical, aliases: variants };
@@ -47,6 +49,14 @@ export async function ensureAgentPersona(
 
   const now = new Date().toISOString();
   const personaId = crypto.randomUUID();
+  const seedTraits: PersonaTrait[] = DEFAULT_SEED_TRAITS.map((t) => ({
+    id: crypto.randomUUID(),
+    name: t.name,
+    description: t.description,
+    sentiment: t.sentiment,
+    strength: t.strength,
+    last_updated: now,
+  }));
   const persona: PersonaEntity = {
     id: personaId,
     display_name: canonical,
@@ -56,7 +66,7 @@ export async function ensureAgentPersona(
     long_description: "An OpenCode agent that assists with coding tasks.",
     group_primary: OPENCODE_GROUP,
     groups_visible: [OPENCODE_GROUP],
-    traits: [],
+    traits: seedTraits,
     topics: [],
     is_paused: false,
     is_archived: false,