ei-tui 0.4.3 → 0.5.0

package/src/core/processor.ts

@@ -1,5 +1,7 @@
 import {
   LLMNextStep,
+  LLMRequestType,
+  LLMPriority,
   type LLMRequest,
   type Ei_Interface,
   type PersonaSummary,
@@ -21,18 +23,23 @@ import {
   type ToolDefinition,
   type ToolProvider,
 } from "./types.js";
+import { buildPersonaFromPersonPrompt } from "../prompts/index.js";
+import type { PersonaGenerationResult } from "../prompts/generation/types.js";
+
 import type { Storage } from "../storage/interface.js";
 import { remoteSync } from "../storage/remote.js";
 import { yoloMerge } from "../storage/merge.js";
 import { StateManager } from "./state-manager.js";
 import { QueueProcessor } from "./queue-processor.js";
 import { handlers } from "./handlers/index.js";
+import { normalizeRoomMessages } from "./handlers/utils.js";
 import { ContextStatus as ContextStatusEnum } from "./types.js";
 import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
 import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
-import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueUserDedupRequest } from "./orchestrators/index.js";
+import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction } from "./orchestrators/index.js";
 import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
+import { DEFAULT_SEED_TRAITS } from "./constants/seed-traits.js";
 
 // Static module imports
 import {
@@ -103,6 +110,24 @@ import {
   clearQueue,
   submitOneShot,
 } from "./queue-manager.js";
+import {
+  getRoomList,
+  getRoom,
+  getRoomMessages,
+  getRoomActivePath,
+  resolveRoomName,
+  createRoom,
+  submitHumanRoomMessage,
+  recallHumanRoomMessage,
+  sendFfaMessage,
+  activateRoom,
+  selectCYPBranch,
+  archiveRoom,
+  unarchiveRoom,
+  deleteRoom,
+  markAllRoomMessagesRead,
+} from "./room-manager.js";
+import type { RoomCreationInput, RoomEntity, RoomMessage, RoomSummary } from "./types.js";
 
 const DEFAULT_LOOP_INTERVAL_MS = 100;
 const DEFAULT_OPENCODE_POLLING_MS = 60000;
@@ -133,6 +158,7 @@ export class Processor {
   private pendingConflict: StateConflictData | null = null;
   private storage: Storage | null = null;
   private importAbortController = new AbortController();
+  private personaPreviewResolvers = new Map<string, { resolve: (r: PersonaGenerationResult) => void; reject: (e: Error) => void }>();
 
   constructor(ei: Ei_Interface) {
     this.interface = ei;
@@ -601,6 +627,180 @@ export class Processor {
         max_calls_per_interaction: 1,
       });
     }
+
+    // submit_response tool — auto-injected for HandlePersonaResponse and HandleRoomResponse.
+    // Not user-configurable; invisible in the tools UI. Terminates the tool loop immediately
+    // when called; its arguments become response.parsed.
+    if (!this.stateManager.tools_getByName("submit_response")) {
+      this.stateManager.tools_add({
+        id: crypto.randomUUID(),
+        provider_id: "ei",
+        name: "submit_response",
+        display_name: "Submit Response",
+        description: "Submit your response to the conversation. Call this when you are ready to respond — after any research or tool use is complete.",
+        input_schema: {
+          type: "object",
+          properties: {
+            should_respond: {
+              type: "boolean",
+              description: "Whether you are responding (true) or staying silent (false)",
+            },
+            verbal_response: {
+              type: "string",
+              description: "What you say out loud. Required when should_respond is true (unless action_response is provided).",
+            },
+            action_response: {
+              type: "string",
+              description: "What you do — rendered as italics stage directions. Optional alongside verbal_response.",
+            },
+            reason: {
+              type: "string",
+              description: "Why you are staying silent. Only used when should_respond is false.",
+            },
+          },
+          required: ["should_respond"],
+          additionalProperties: false,
+        },
+        runtime: "any",
+        builtin: true,
+        enabled: true,
+        is_submit: true,
+        max_calls_per_interaction: 1,
+        created_at: now,
+      });
+    }
+
+    if (!this.stateManager.tools_getByName("submit_heartbeat_check")) {
+      this.stateManager.tools_add({
+        id: crypto.randomUUID(),
+        provider_id: "ei",
+        name: "submit_heartbeat_check",
+        display_name: "Submit Heartbeat Decision",
+        description: "Submit your decision on whether to reach out with a message. Call this when you have decided.",
+        input_schema: {
+          type: "object",
+          properties: {
+            should_respond: { type: "boolean", description: "Whether you want to initiate a message" },
+            topic: { type: "string", description: "The specific topic you want to discuss (when should_respond is true)" },
+            message: { type: "string", description: "Your actual message to them (when should_respond is true)" },
+          },
+          required: ["should_respond"],
+          additionalProperties: false,
+        },
+        runtime: "any",
+        builtin: true,
+        enabled: true,
+        is_submit: true,
+        max_calls_per_interaction: 1,
+        created_at: now,
+      });
+    }
+
+    if (!this.stateManager.tools_getByName("submit_ei_heartbeat")) {
+      this.stateManager.tools_add({
+        id: crypto.randomUUID(),
+        provider_id: "ei",
+        name: "submit_ei_heartbeat",
+        display_name: "Submit Ei Heartbeat Decision",
+        description: "Submit your choice of item to follow up on, or indicate nothing warrants reaching out.",
+        input_schema: {
+          type: "object",
+          properties: {
+            should_respond: { type: "boolean", description: "Whether Ei wants to check in about an item" },
+            id: { type: "string", description: "ID of the item you chose (when should_respond is true)" },
+            my_response: { type: "string", description: "The check-in message (for Person/Topic/Persona items)" },
+          },
+          required: ["should_respond"],
+          additionalProperties: false,
+        },
+        runtime: "any",
+        builtin: true,
+        enabled: true,
+        is_submit: true,
+        max_calls_per_interaction: 1,
+        created_at: now,
+      });
+    }
+
+    if (!this.stateManager.tools_getByName("submit_dedup_decisions")) {
+      this.stateManager.tools_add({
+        id: crypto.randomUUID(),
+        provider_id: "ei",
+        name: "submit_dedup_decisions",
+        display_name: "Submit Dedup Decisions",
+        description: "Submit your merge, remove, and add decisions for this cluster of records.",
+        input_schema: {
+          type: "object",
+          properties: {
+            update: {
+              type: "array",
+              description: "Records to update with merged data. Must include at least one (the canonical record).",
+              items: {
+                type: "object",
+                properties: {
+                  id: { type: "string" },
+                  type: { type: "string", enum: ["topic", "person", "trait"] },
+                  name: { type: "string" },
+                  description: { type: "string" },
+                  sentiment: { type: "number" },
+                  strength: { type: "number" },
+                  confidence: { type: "number" },
+                  exposure_current: { type: "number" },
+                  exposure_desired: { type: "number" },
+                  relationship: { type: "string" },
+                  category: { type: "string" },
+                  last_updated: { type: "string" },
+                },
+                required: ["id", "type", "name", "description"],
+                additionalProperties: false,
+              },
+            },
+            remove: {
+              type: "array",
+              description: "Duplicates to remove. Each must reference its canonical record via replaced_by.",
+              items: {
+                type: "object",
+                properties: {
+                  to_be_removed: { type: "string" },
+                  replaced_by: { type: "string" },
+                },
+                required: ["to_be_removed", "replaced_by"],
+                additionalProperties: false,
+              },
+            },
+            add: {
+              type: "array",
+              description: "New records to create. Only when merging reveals a missing concept.",
+              items: {
+                type: "object",
+                properties: {
+                  type: { type: "string", enum: ["topic", "person", "trait"] },
+                  name: { type: "string" },
+                  description: { type: "string" },
+                  sentiment: { type: "number" },
+                  strength: { type: "number" },
+                  confidence: { type: "number" },
+                  exposure_current: { type: "number" },
+                  exposure_desired: { type: "number" },
+                  relationship: { type: "string" },
+                  category: { type: "string" },
+                },
+                required: ["type", "name", "description"],
+                additionalProperties: false,
+              },
+            },
+          },
+          required: ["update", "remove", "add"],
+          additionalProperties: false,
+        },
+        runtime: "any",
+        builtin: true,
+        enabled: true,
+        is_submit: true,
+        max_calls_per_interaction: 1,
+        created_at: now,
+      });
+    }
   }
 
   /**
@@ -837,8 +1037,14 @@ export class Processor {
               this.interface.onMessageProcessing?.(personaId);
             }
 
+            const roomId = request.data.roomId as string | undefined;
+            if (roomId && (request.next_step === LLMNextStep.HandleRoomResponse || request.next_step === LLMNextStep.HandleRoomJudge)) {
+              this.interface.onRoomMessageProcessing?.(roomId);
+            }
+
 const toolNextSteps = new Set([
   LLMNextStep.HandlePersonaResponse,
+  LLMNextStep.HandleRoomResponse,
   LLMNextStep.HandleHeartbeatCheck,
   LLMNextStep.HandleEiHeartbeat,
   LLMNextStep.HandleToolContinuation,
@@ -866,6 +1072,25 @@ const toolNextSteps = new Set([
             } else if (toolNextSteps.has(request.next_step) && toolPersonaId) {
               tools = this.stateManager.tools_getForPersona(toolPersonaId, this.isTUI);
             }
+
+            // Auto-inject each handler's dedicated submit tool — infrastructure, not user-visible.
+            const submitToolByStep: Partial<Record<string, string>> = {
+              [LLMNextStep.HandlePersonaResponse]:  "submit_response",
+              [LLMNextStep.HandleRoomResponse]:     "submit_response",
+              [LLMNextStep.HandleHeartbeatCheck]:   "submit_heartbeat_check",
+              [LLMNextStep.HandleEiHeartbeat]:      "submit_ei_heartbeat",
+              [LLMNextStep.HandleDedupCurate]:      "submit_dedup_decisions",
+            };
+            const effectiveStep = request.next_step === LLMNextStep.HandleToolContinuation
+              ? (request.data.originalNextStep as string | undefined)
+              : request.next_step;
+            const submitToolName = effectiveStep ? submitToolByStep[effectiveStep] : undefined;
+            if (submitToolName) {
+              const submitTool = this.stateManager.tools_getByName(submitToolName);
+              if (submitTool?.enabled && !tools.find(t => t.name === submitToolName)) {
+                tools = [...tools, submitTool];
+              }
+            }
             
             console.log(
               `[Tools] Dispatch for ${request.next_step} persona=${toolPersonaId ?? "none"}: ${tools.length} tool(s) attached`
@@ -882,7 +1107,13 @@ const toolNextSteps = new Set([
               {
                 accounts: this.stateManager.getHuman().settings?.accounts,
                 messageFetcher: (pName) => fetchMessagesForLLM(this.stateManager, pName),
-                rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
+                rawMessageFetcher: (id) => {
+                  if (id.startsWith("room:")) {
+                    const roomId = id.slice(5);
+                    return normalizeRoomMessages(this.stateManager.getRoomMessages(roomId), this.stateManager);
+                  }
+                  return this.stateManager.messages_get(id);
+                },
                 tools: tools.length > 0 ? tools : undefined,
                 onEnqueue: (req) => this.stateManager.queue_enqueue(req),
                 onProviderConfigUpdate: (providerId, updates) => {
@@ -1206,6 +1437,15 @@ const toolNextSteps = new Set([
         message += ` (attempt ${response.request.attempts}, retrying in ${Math.round(result.retryDelay / 1000)}s)`;
       } else if (result.dropped) {
         message += " (permanent failure \u2014 request removed)";
+        if (response.request.next_step === LLMNextStep.HandlePersonaPreview) {
+          const guid = response.request.data.guid as string;
+          const entry = this.personaPreviewResolvers.get(guid);
+          if (entry) {
+            this.personaPreviewResolvers.delete(guid);
+            entry.reject(new Error("Persona preview generation failed after max retries"));
+            return;
+          }
+        }
         if (response.request.next_step === LLMNextStep.HandleOneShot) {
           const guid = response.request.data.guid as string;
           this.interface.onOneShotReturned?.(guid, "");
@@ -1255,6 +1495,88 @@ const toolNextSteps = new Set([
         this.interface.onOneShotReturned?.(guid, content);
       }
 
+      if (response.request.next_step === LLMNextStep.HandlePersonaPreview) {
+        const guid = response.request.data.guid as string;
+        const loopCounter = (response.request.data.loop_counter as number) ?? 0;
+        const existingPersonaId = response.request.data.personaId as string | undefined;
+        const MAX_PREVIEW_LOOPS = 3;
+        const entry = this.personaPreviewResolvers.get(guid);
+        if (!entry) return;
+
+        if (!response.success || !response.parsed) {
+          return;
+        }
+
+        let result = response.parsed as import("../prompts/generation/types.js").PersonaGenerationResult;
+        const isComplete =
+          result.traits && result.traits.length >= 3 &&
+          result.topics && result.topics.length >= 3 &&
+          result.long_description && result.short_description;
+
+        const hydrateWithExisting = (r: typeof result): typeof result => {
+          if (!existingPersonaId) return r;
+          const existing = this.stateManager.persona_getById(existingPersonaId);
+          if (!existing) return r;
+
+          const existingTraitNames = new Set(existing.traits.map(t => t.name.toLowerCase().trim()));
+          const newTraits = r.traits.filter(t => !existingTraitNames.has(t.name.toLowerCase().trim()));
+          const mergedTraits = [
+            ...existing.traits.map(t => ({
+              name: t.name,
+              description: t.description,
+              strength: t.strength ?? 0.5,
+              sentiment: t.sentiment,
+            })),
+            ...newTraits,
+          ];
+
+          const existingTopicNames = new Set(existing.topics.map(t => t.name.toLowerCase().trim()));
+          const newTopics = r.topics.filter(t => !existingTopicNames.has(t.name.toLowerCase().trim()));
+          const mergedTopics = [...existing.topics, ...newTopics];
+
+          return {
+            ...r,
+            traits: mergedTraits,
+            topics: mergedTopics,
+            previous_long_description: existing.long_description,
+            previous_short_description: existing.short_description,
+            aliases: existing.aliases ?? [],
+          };
+        };
+
+        if (isComplete) {
+          this.personaPreviewResolvers.delete(guid);
+          const hydratedComplete = hydrateWithExisting(result);
+          const seedTraitNamesComplete = new Set(hydratedComplete.traits.map((t: { name: string }) => t.name.toLowerCase().trim()));
+          DEFAULT_SEED_TRAITS
+            .filter(s => !seedTraitNamesComplete.has(s.name.toLowerCase().trim()))
+            .forEach(s => hydratedComplete.traits.push({ name: s.name, description: s.description, sentiment: s.sentiment, strength: s.strength }));
+          entry.resolve(hydratedComplete);
+          return;
+        }
+
+        if (loopCounter < MAX_PREVIEW_LOOPS) {
+          this.stateManager.queue_enqueue({
+            type: LLMRequestType.JSON,
+            priority: LLMPriority.High,
+            system: response.request.system,
+            user: response.request.user,
+            next_step: LLMNextStep.HandlePersonaPreview,
+            model: response.request.model,
+            data: { guid, loop_counter: loopCounter + 1, personaId: existingPersonaId },
+          });
+          return;
+        }
+
+        this.personaPreviewResolvers.delete(guid);
+        const hydratedFallback = hydrateWithExisting(result);
+        const seedTraitNamesFallback = new Set(hydratedFallback.traits.map((t: { name: string }) => t.name.toLowerCase().trim()));
+        DEFAULT_SEED_TRAITS
+          .filter(s => !seedTraitNamesFallback.has(s.name.toLowerCase().trim()))
+          .forEach(s => hydratedFallback.traits.push({ name: s.name, description: s.description, sentiment: s.sentiment, strength: s.strength }));
+        entry.resolve(hydratedFallback);
+      }
+
       if (response.request.next_step === LLMNextStep.HandlePersonaGeneration) {
         const personaId = response.request.data.personaId as string;
         if (personaId) {
@@ -1307,6 +1629,27 @@ const toolNextSteps = new Set([
         this.interface.onHumanUpdated?.();
       }
 
+      if (response.request.next_step === LLMNextStep.HandleDedupCurate) {
+        this.interface.onHumanUpdated?.();
+      }
+
+      const isRoomResponse =
+        response.request.next_step === LLMNextStep.HandleRoomResponse ||
+        (response.request.next_step === LLMNextStep.HandleToolContinuation &&
+          response.request.data.originalNextStep === LLMNextStep.HandleRoomResponse);
+      if (isRoomResponse) {
+        const roomId = response.request.data.roomId as string;
+        if (roomId) {
+          this.interface.onRoomMessageAdded?.(roomId);
+          checkAndQueueRoomExtraction(this.stateManager, roomId);
+        }
+      }
+
+      if (response.request.next_step === LLMNextStep.HandleRoomJudge) {
+        const roomId = response.request.data.roomId as string;
+        if (roomId) this.interface.onRoomUpdated?.(roomId);
+      }
+
       if (typeof response.request.data.ceremony_progress === "number") {
         handleCeremonyProgress(this.stateManager, response.request.data.ceremony_progress);
       }
@@ -1377,6 +1720,11 @@ const toolNextSteps = new Set([
     if (ok) this.interface.onPersonaUpdated?.(personaId);
   }
 
+  async updateRoom(roomId: string, updates: Partial<RoomEntity>): Promise<void> {
+    const ok = this.stateManager.updateRoom(roomId, updates);
+    if (ok) this.interface.onRoomUpdated?.(roomId);
+  }
+
   async getGroupList(): Promise<string[]> {
     return getGroupList(this.stateManager);
   }
@@ -1408,7 +1756,7 @@ const toolNextSteps = new Set([
     );
   }
 
-  async sendMessage(personaId: string, content: string): Promise<void> {
+  async sendMessage(personaId: string, content: string | null, silenceReason?: string): Promise<void> {
     return sendMessage(
       this.stateManager,
       this.queueProcessor,
@@ -1419,7 +1767,8 @@ const toolNextSteps = new Set([
       (id) => getModelForPersona(this.stateManager, id),
       (err) => this.interface.onError?.(err),
       (id) => this.interface.onMessageAdded?.(id),
-      (id) => this.interface.onMessageQueued?.(id)
+      (id) => this.interface.onMessageQueued?.(id),
+      silenceReason
     );
   }
 
@@ -1590,6 +1939,14 @@ const toolNextSteps = new Set([
     queueUserDedupRequest(this.stateManager, itemType, entityIds);
   }
 
+  capturePersona(personaId: string): void {
+    queuePersonaCapture(this.stateManager, personaId);
+  }
+
+  captureRoom(roomId: string): void {
+    queueRoomCapture(this.stateManager, roomId);
+  }
+
   async submitOneShot(guid: string, systemPrompt: string, userPrompt: string): Promise<void> {
     return submitOneShot(
       this.stateManager,
@@ -1600,6 +1957,45 @@ const toolNextSteps = new Set([
     );
   }
 
+  async generatePersonaPreview(
+    name: string,
+    description: string,
+    relationship?: string,
+    personaId?: string
+  ): Promise<PersonaGenerationResult> {
+    let existing_trait_names: string[] | undefined;
+    let existing_topic_names: string[] | undefined;
+
+    if (personaId) {
+      const existing = this.stateManager.persona_getById(personaId);
+      if (existing) {
+        existing_trait_names = existing.traits.map((t) => t.name);
+        existing_topic_names = existing.topics.map((t) => t.name);
+      }
+    }
+
+    const prompt = buildPersonaFromPersonPrompt({
+      name,
+      description,
+      relationship,
+      existing_trait_names,
+      existing_topic_names,
+    });
+    const guid = crypto.randomUUID();
+    return new Promise<PersonaGenerationResult>((resolve, reject) => {
+      this.personaPreviewResolvers.set(guid, { resolve, reject });
+      this.stateManager.queue_enqueue({
+        type: LLMRequestType.JSON,
+        priority: LLMPriority.High,
+        system: prompt.system,
+        user: prompt.user,
+        next_step: LLMNextStep.HandlePersonaPreview,
+        model: getOneshotModel(this.stateManager),
+        data: { guid, loop_counter: 0, personaId },
+      });
+    });
+  }
+
   // ==========================================================================
   // TOOL API
   // ==========================================================================
@@ -1662,6 +2058,126 @@ const toolNextSteps = new Set([
     return result;
   }
 
+  // ==========================================================================
+  // ROOM API
+  // ==========================================================================
+
+  getRoomList(includeArchived = false): RoomSummary[] {
+    return getRoomList(this.stateManager, includeArchived);
+  }
+
+  getRoom(roomId: string): RoomEntity | null {
+    return getRoom(this.stateManager, roomId);
+  }
+
+  getRoomMessages(roomId: string): RoomMessage[] {
+    return getRoomMessages(this.stateManager, roomId);
+  }
+
+  getRoomActivePath(roomId: string): RoomMessage[] {
+    return getRoomActivePath(this.stateManager, roomId);
+  }
+
+  resolveRoomName(nameOrAlias: string): string | null {
+    return resolveRoomName(this.stateManager, nameOrAlias);
+  }
+
+  async createRoom(input: RoomCreationInput): Promise<string> {
+    const id = await createRoom(
+      this.stateManager,
+      input,
+      this.isTUI,
+      (err) => this.interface.onError?.(err),
+      (id) => this.interface.onRoomMessageAdded?.(id),
+      (id) => this.interface.onRoomMessageQueued?.(id)
+    );
+    if (id) this.interface.onRoomAdded?.();
+    return id;
+  }
+
+  submitHumanRoomMessage(
+    roomId: string,
+    content: string | null,
+    silenceReason?: string
+  ): string | null {
+    return submitHumanRoomMessage(
+      this.stateManager,
+      roomId,
+      content,
+      silenceReason,
+      (err) => this.interface.onError?.(err),
+      (id) => this.interface.onRoomMessageAdded?.(id)
+    );
+  }
+
+  recallHumanRoomMessage(roomId: string): boolean {
+    return recallHumanRoomMessage(
+      this.stateManager,
+      roomId,
+      (id) => this.interface.onRoomUpdated?.(id)
+    );
+  }
+
+  async activateRoom(roomId: string): Promise<void> {
+    return activateRoom(
+      this.stateManager,
+      roomId,
+      this.isTUI,
+      (err) => this.interface.onError?.(err),
+      (id) => this.interface.onRoomUpdated?.(id),
+      (id) => this.interface.onRoomMessageQueued?.(id)
+    );
+  }
+
+  async sendFfaMessage(
+    roomId: string,
+    content: string | null,
+    silenceReason?: string
+  ): Promise<void> {
+    return sendFfaMessage(
+      this.stateManager,
+      roomId,
+      content,
+      silenceReason,
+      this.isTUI,
+      (err) => this.interface.onError?.(err),
+      (id) => this.interface.onRoomUpdated?.(id),
+      (id) => this.interface.onRoomMessageAdded?.(id),
+      (id) => this.interface.onRoomMessageQueued?.(id)
+    );
+  }
+
+  async selectCYPBranch(roomId: string, messageId: string): Promise<void> {
+    return selectCYPBranch(
+      this.stateManager,
+      roomId,
+      messageId,
+      this.isTUI,
+      (err) => this.interface.onError?.(err),
+      (id) => this.interface.onRoomUpdated?.(id),
+      (id) => this.interface.onRoomMessageQueued?.(id)
+    );
+  }
+
+  async archiveRoom(roomId: string): Promise<void> {
+    const ok = archiveRoom(this.stateManager, roomId);
+    if (ok) this.interface.onRoomRemoved?.();
+  }
+
+  async unarchiveRoom(roomId: string): Promise<void> {
+    const ok = unarchiveRoom(this.stateManager, roomId);
+    if (ok) this.interface.onRoomAdded?.();
+  }
+
+  async deleteRoom(roomId: string): Promise<void> {
+    const ok = deleteRoom(this.stateManager, roomId);
+    if (ok) this.interface.onRoomRemoved?.();
+  }
+
+  async markAllRoomMessagesRead(roomId: string): Promise<number> {
+    return markAllRoomMessagesRead(this.stateManager, roomId);
+  }
+
   // ==========================================================================
   // DEBUG / TESTING UTILITIES
   // ==========================================================================