ei-tui 0.1.3

package/src/core/queue-processor.ts

@@ -0,0 +1,197 @@
+import type { LLMRequest, LLMResponse, LLMRequestType, ProviderAccount, ChatMessage, Message } from "./types.js";
+import { callLLMRaw, parseJSONResponse, cleanResponseContent } from "./llm-client.js";
+import { hydratePromptPlaceholders } from "../prompts/message-utils.js";
+
+type QueueProcessorState = "idle" | "busy";
+type ResponseCallback = (response: LLMResponse) => Promise<void>;
+type MessageFetcher = (personaId: string) => ChatMessage[];
+type RawMessageFetcher = (personaId: string) => Message[];
+
+export interface QueueProcessorStartOptions {
+  accounts?: ProviderAccount[];
+  messageFetcher?: MessageFetcher;
+  rawMessageFetcher?: RawMessageFetcher;
+}
+
+export class QueueProcessor {
+  private state: QueueProcessorState = "idle";
+  private abortController: AbortController | null = null;
+  private currentCallback: ResponseCallback | null = null;
+  private currentAccounts: ProviderAccount[] | undefined;
+  private currentMessageFetcher: MessageFetcher | undefined;
+  private currentRawMessageFetcher: RawMessageFetcher | undefined;
+
+  getState(): QueueProcessorState {
+    return this.state;
+  }
+
+  start(request: LLMRequest, callback: ResponseCallback, options?: QueueProcessorStartOptions): void {
+    if (this.state !== "idle") {
+      throw new Error("QUEUE_BUSY: QueueProcessor is already processing a request");
+    }
+
+    this.state = "busy";
+    this.currentCallback = callback;
+    this.currentAccounts = options?.accounts;
+    this.currentMessageFetcher = options?.messageFetcher;
+    this.currentRawMessageFetcher = options?.rawMessageFetcher;
+    this.abortController = new AbortController();
+
+    this.processRequest(request)
+      .then((response) => {
+        this.finishWith(response);
+      })
+      .catch((error) => {
+        this.finishWith({
+          request,
+          success: false,
+          content: null,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      });
+  }
+
+  abort(): void {
+    if (this.abortController) {
+      this.abortController.abort();
+    }
+  }
+
+  private finishWith(response: LLMResponse): void {
+    const mortalKombat = this.currentCallback ? this.currentCallback : () => Promise.resolve();
+    mortalKombat(response).finally(() => {
+      this.state = "idle";
+      this.currentCallback = null;
+      this.currentAccounts = undefined;
+      this.currentMessageFetcher = undefined;
+      this.currentRawMessageFetcher = undefined;
+      this.abortController = null;
+    });
+  }
+
+  private async processRequest(request: LLMRequest): Promise<LLMResponse> {
+    let messages: ChatMessage[] = [];
+    
+    if (request.type === "response" as LLMRequestType) {
+      const personaId = request.data.personaId as string | undefined;
+      if (personaId && this.currentMessageFetcher) {
+        messages = this.currentMessageFetcher(personaId);
+      }
+    }
+
+    let hydratedSystem = request.system;
+    let hydratedUser = request.user;
+    
+    if (this.currentRawMessageFetcher) {
+      const personaId = request.data.personaId as string | undefined;
+      if (personaId) {
+        const rawMessages = this.currentRawMessageFetcher(personaId);
+        const messageMap = new Map<string, Message>();
+        for (const msg of rawMessages) {
+          messageMap.set(msg.id, msg);
+        }
+        
+        const placeholderCount = (request.user.match(/\[mid:[^\]]+\]/g) || []).length;
+        console.log(`[QueueProcessor] Hydrating ${placeholderCount} placeholders with ${messageMap.size} messages for ${personaId}`);
+        
+        hydratedSystem = hydratePromptPlaceholders(request.system, messageMap);
+        hydratedUser = hydratePromptPlaceholders(request.user, messageMap);
+        
+        const hydratedPlaceholderCount = (hydratedUser.match(/\[mid:[^\]]+\]/g) || []).length;
+        if (hydratedPlaceholderCount > 0) {
+          console.log(`[QueueProcessor] WARNING: ${hydratedPlaceholderCount} placeholders not hydrated!`);
+        }
+      }
+    }
+
+    const { content, finishReason } = await callLLMRaw(
+      hydratedSystem,
+      hydratedUser,
+      messages,
+      request.model,
+      { signal: this.abortController?.signal },
+      this.currentAccounts
+    );
+
+    if (!content) {
+      return {
+        request,
+        success: false,
+        content: null,
+        error: "Empty response from LLM",
+        finish_reason: finishReason ?? undefined,
+      };
+    }
+
+    return this.handleResponseType(request, content, finishReason);
+  }
+
+  private handleResponseType(
+    request: LLMRequest,
+    content: string,
+    finishReason: string | null
+  ): LLMResponse {
+    switch (request.type) {
+      case "json" as LLMRequestType:
+        return this.handleJSONResponse(request, content, finishReason);
+      case "response" as LLMRequestType:
+        return this.handleConversationResponse(request, content, finishReason);
+      case "raw" as LLMRequestType:
+      default:
+        return {
+          request,
+          success: true,
+          content,
+          finish_reason: finishReason ?? undefined,
+        };
+    }
+  }
+
+  private handleJSONResponse(
+    request: LLMRequest,
+    content: string,
+    finishReason: string | null
+  ): LLMResponse {
+    try {
+      const parsed = parseJSONResponse(content);
+      return {
+        request,
+        success: true,
+        content,
+        parsed,
+        finish_reason: finishReason ?? undefined,
+      };
+    } catch (error) {
+      return {
+        request,
+        success: false,
+        content,
+        error: `JSON parse failed: ${error instanceof Error ? error.message : String(error)}`,
+        finish_reason: finishReason ?? undefined,
+      };
+    }
+  }
+
+  private handleConversationResponse(
+    request: LLMRequest,
+    content: string,
+    finishReason: string | null
+  ): LLMResponse {
+    const cleaned = cleanResponseContent(content);
+    
+    const noMessagePatterns = [
+      /^no\s*(new\s*)?(message|response)/i,
+      /^nothing\s+to\s+(say|add)/i,
+      /^\[no\s+message\]/i,
+    ];
+    
+    const isNoMessage = noMessagePatterns.some((p) => p.test(cleaned));
+    
+    return {
+      request,
+      success: true,
+      content: isNoMessage ? null : cleaned,
+      finish_reason: finishReason ?? undefined,
+    };
+  }
+}

package/src/core/state/checkpoints.ts

@@ -0,0 +1,68 @@
+import type { StorageState } from "../types.js";
+import type { Storage } from "../../storage/interface.js";
+
+const DEBOUNCE_MS = 100;
+
+export class PersistenceState {
+  private storage: Storage | null = null;
+  private saveTimeout: ReturnType<typeof setTimeout> | null = null;
+  private pendingState: StorageState | null = null;
+  private loadedExistingData = false;
+
+  setStorage(storage: Storage): void {
+    this.storage = storage;
+  }
+
+  scheduleSave(state: StorageState): void {
+    this.pendingState = state;
+    
+    if (this.saveTimeout) {
+      clearTimeout(this.saveTimeout);
+    }
+    
+    this.saveTimeout = setTimeout(async () => {
+      this.saveTimeout = null;
+      if (this.pendingState) {
+        await this.saveNow(this.pendingState);
+        this.pendingState = null;
+      }
+    }, DEBOUNCE_MS);
+  }
+
+  async saveNow(state: StorageState): Promise<void> {
+    if (!this.storage) throw new Error("Storage not initialized");
+    await this.storage.save(state);
+  }
+
+  async load(): Promise<StorageState | null> {
+    if (!this.storage) throw new Error("Storage not initialized");
+    const state = await this.storage.load();
+    this.loadedExistingData = state !== null;
+    return state;
+  }
+
+  hasExistingData(): boolean {
+    return this.loadedExistingData;
+  }
+
+  async flush(): Promise<void> {
+    if (this.saveTimeout) {
+      clearTimeout(this.saveTimeout);
+      this.saveTimeout = null;
+    }
+    if (this.pendingState && this.storage) {
+      await this.storage.save(this.pendingState);
+      this.pendingState = null;
+    }
+  }
+
+  async moveToBackup(): Promise<void> {
+    if (!this.storage) throw new Error("Storage not initialized");
+    await this.storage.moveToBackup();
+  }
+
+  async loadBackup(): Promise<StorageState | null> {
+    if (!this.storage) throw new Error("Storage not initialized");
+    return this.storage.loadBackup();
+  }
+}

package/src/core/state/human.ts

@@ -0,0 +1,176 @@
+import type { HumanEntity, Fact, Trait, Topic, Person, Quote } from "../types.js";
+
+export function createDefaultHumanEntity(): HumanEntity {
+  return {
+    entity: "human",
+    facts: [],
+    traits: [],
+    topics: [],
+    people: [],
+    quotes: [],
+    last_updated: new Date().toISOString(),
+    last_activity: new Date().toISOString(),
+    settings: {
+      ceremony: {
+        time: "09:00",
+      },
+      opencode: {
+        integration: false,
+        polling_interval_ms: 1800000,
+      },
+    },
+  };
+}
+
+export class HumanState {
+  private human: HumanEntity = createDefaultHumanEntity();
+
+  load(entity: HumanEntity): void {
+    this.human = entity;
+  }
+
+  get(): HumanEntity {
+    return this.human;
+  }
+
+  set(entity: HumanEntity): void {
+    this.human = entity;
+    this.human.last_updated = new Date().toISOString();
+  }
+
+  fact_upsert(fact: Fact): void {
+    const idx = this.human.facts.findIndex((f) => f.id === fact.id);
+    fact.last_updated = new Date().toISOString();
+    if (idx >= 0) {
+      this.human.facts[idx] = fact;
+    } else {
+      this.human.facts.push(fact);
+    }
+    this.human.last_updated = new Date().toISOString();
+  }
+
+  fact_remove(id: string): boolean {
+    const idx = this.human.facts.findIndex((f) => f.id === id);
+    if (idx >= 0) {
+      this.human.facts.splice(idx, 1);
+      // Clean up quote references
+      this.human.quotes.forEach((q) => {
+        q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== id);
+      });
+      this.human.last_updated = new Date().toISOString();
+      return true;
+    }
+    return false;
+  }
+
+  trait_upsert(trait: Trait): void {
+    const idx = this.human.traits.findIndex((t) => t.id === trait.id);
+    trait.last_updated = new Date().toISOString();
+    if (idx >= 0) {
+      this.human.traits[idx] = trait;
+    } else {
+      this.human.traits.push(trait);
+    }
+    this.human.last_updated = new Date().toISOString();
+  }
+
+  trait_remove(id: string): boolean {
+    const idx = this.human.traits.findIndex((t) => t.id === id);
+    if (idx >= 0) {
+      this.human.traits.splice(idx, 1);
+      // Clean up quote references
+      this.human.quotes.forEach((q) => {
+        q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== id);
+      });
+      this.human.last_updated = new Date().toISOString();
+      return true;
+    }
+    return false;
+  }
+
+  topic_upsert(topic: Topic): void {
+    const idx = this.human.topics.findIndex((t) => t.id === topic.id);
+    topic.last_updated = new Date().toISOString();
+    if (idx >= 0) {
+      this.human.topics[idx] = topic;
+    } else {
+      this.human.topics.push(topic);
+    }
+    this.human.last_updated = new Date().toISOString();
+  }
+
+  topic_remove(id: string): boolean {
+    const idx = this.human.topics.findIndex((t) => t.id === id);
+    if (idx >= 0) {
+      this.human.topics.splice(idx, 1);
+      // Clean up quote references
+      this.human.quotes.forEach((q) => {
+        q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== id);
+      });
+      this.human.last_updated = new Date().toISOString();
+      return true;
+    }
+    return false;
+  }
+
+  person_upsert(person: Person): void {
+    const idx = this.human.people.findIndex((p) => p.id === person.id);
+    person.last_updated = new Date().toISOString();
+    if (idx >= 0) {
+      this.human.people[idx] = person;
+    } else {
+      this.human.people.push(person);
+    }
+    this.human.last_updated = new Date().toISOString();
+  }
+
+   person_remove(id: string): boolean {
+     const idx = this.human.people.findIndex((p) => p.id === id);
+     if (idx >= 0) {
+       this.human.people.splice(idx, 1);
+       // Clean up quote references
+       this.human.quotes.forEach((q) => {
+         q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== id);
+       });
+       this.human.last_updated = new Date().toISOString();
+       return true;
+     }
+     return false;
+   }
+
+   quote_add(quote: Quote): void {
+     if (!quote.created_at) {
+       quote.created_at = new Date().toISOString();
+     }
+     this.human.quotes.push(quote);
+     this.human.last_updated = new Date().toISOString();
+   }
+
+   quote_update(id: string, updates: Partial<Quote>): boolean {
+     const idx = this.human.quotes.findIndex((q) => q.id === id);
+     if (idx >= 0) {
+       this.human.quotes[idx] = { ...this.human.quotes[idx], ...updates };
+       this.human.last_updated = new Date().toISOString();
+       return true;
+     }
+     return false;
+   }
+
+   quote_remove(id: string): boolean {
+     const idx = this.human.quotes.findIndex((q) => q.id === id);
+     if (idx >= 0) {
+       this.human.quotes.splice(idx, 1);
+       this.human.last_updated = new Date().toISOString();
+       return true;
+     }
+     return false;
+   }
+
+   quote_getForMessage(messageId: string): Quote[] {
+     return this.human.quotes.filter((q) => q.message_id === messageId);
+   }
+
+   quote_getForDataItem(dataItemId: string): Quote[] {
+     return this.human.quotes.filter((q) => q.data_item_ids.includes(dataItemId));
+   }
+}

package/src/core/state/index.ts

@@ -0,0 +1,5 @@
+export { HumanState, createDefaultHumanEntity } from "./human.js";
+export { PersonaState } from "./personas.js";
+export type { PersonaData } from "./personas.js";
+export { QueueState } from "./queue.js";
+export { PersistenceState } from "./checkpoints.js";

package/src/core/state/personas.ts

@@ -0,0 +1,217 @@
+import type { PersonaEntity, Message, ContextStatus } from "../types.js";
+
+export interface PersonaData {
+  entity: PersonaEntity;
+  messages: Message[];
+  contextWindow?: { start: string; end: string };
+}
+
+export class PersonaState {
+  private personas: Map<string, PersonaData> = new Map();
+
+  load(personas: Record<string, { entity: PersonaEntity; messages: Message[] }>): void {
+    this.personas = new Map(
+      Object.entries(personas).map(([id, data]) => [
+        id,
+        { entity: data.entity, messages: data.messages },
+      ])
+    );
+  }
+
+  export(): Record<string, { entity: PersonaEntity; messages: Message[] }> {
+    const result: Record<string, { entity: PersonaEntity; messages: Message[] }> = {};
+    for (const [id, data] of this.personas) {
+      result[id] = { entity: data.entity, messages: data.messages };
+    }
+    return result;
+  }
+
+  getAll(): PersonaEntity[] {
+    return Array.from(this.personas.values()).map((p) => p.entity);
+  }
+
+  getById(id: string): PersonaEntity | null {
+    return this.personas.get(id)?.entity ?? null;
+  }
+
+  getByName(nameOrAlias: string): PersonaEntity | null {
+    const searchTerm = nameOrAlias.toLowerCase();
+    
+    // Priority 1: Exact display_name match
+    for (const data of this.personas.values()) {
+      if (data.entity.display_name.toLowerCase() === searchTerm) {
+        return data.entity;
+      }
+    }
+    
+    // Priority 2: Exact alias match
+    for (const data of this.personas.values()) {
+      if (data.entity.aliases?.some(alias => alias.toLowerCase() === searchTerm)) {
+        return data.entity;
+      }
+    }
+    
+    // Priority 3: Unambiguous partial match
+    const partialMatches: PersonaEntity[] = [];
+    for (const data of this.personas.values()) {
+      const displayNameLower = data.entity.display_name.toLowerCase();
+      const aliasesLower = data.entity.aliases?.map(a => a.toLowerCase()) ?? [];
+      
+      const matchesDisplayName = displayNameLower.includes(searchTerm);
+      const matchesAlias = aliasesLower.some(alias => alias.includes(searchTerm));
+      
+      if (matchesDisplayName || matchesAlias) {
+        partialMatches.push(data.entity);
+      }
+    }
+    
+    return partialMatches.length === 1 ? partialMatches[0] : null;
+  }
+
+  add(entity: PersonaEntity): void {
+    this.personas.set(entity.id, { entity, messages: [] });
+  }
+
+  update(personaId: string, updates: Partial<PersonaEntity>): boolean {
+    const data = this.personas.get(personaId);
+    if (!data) return false;
+    data.entity = { ...data.entity, ...updates, last_updated: new Date().toISOString() };
+    return true;
+  }
+
+  archive(personaId: string): boolean {
+    const data = this.personas.get(personaId);
+    if (!data) return false;
+    data.entity.is_archived = true;
+    data.entity.archived_at = new Date().toISOString();
+    data.entity.last_updated = new Date().toISOString();
+    return true;
+  }
+
+  unarchive(personaId: string): boolean {
+    const data = this.personas.get(personaId);
+    if (!data) return false;
+    data.entity.is_archived = false;
+    data.entity.archived_at = undefined;
+    data.entity.last_updated = new Date().toISOString();
+    return true;
+  }
+
+  delete(personaId: string): boolean {
+    return this.personas.delete(personaId);
+  }
+
+  messages_get(personaId: string): Message[] {
+    const messages = this.personas.get(personaId)?.messages ?? [];
+    return messages.map(m => ({ ...m }));
+  }
+
+  messages_append(personaId: string, message: Message): void {
+    const data = this.personas.get(personaId);
+    if (!data) return;
+    data.messages.push(message);
+    data.entity.last_activity = message.timestamp;
+    data.entity.last_updated = new Date().toISOString();
+  }
+
+  messages_sort(personaId: string): void {
+    const data = this.personas.get(personaId);
+    if (!data) return;
+    data.messages.sort((a, b) => 
+      new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+    );
+  }
+
+  messages_setContextStatus(
+    personaId: string,
+    messageId: string,
+    status: ContextStatus
+  ): boolean {
+    const data = this.personas.get(personaId);
+    if (!data) return false;
+    const msg = data.messages.find((m) => m.id === messageId);
+    if (!msg) return false;
+    msg.context_status = status;
+    return true;
+  }
+
+  messages_markRead(personaId: string, messageId: string): boolean {
+    const data = this.personas.get(personaId);
+    if (!data) return false;
+    const msg = data.messages.find((m) => m.id === messageId);
+    if (!msg) return false;
+    msg.read = true;
+    return true;
+  }
+
+  messages_markPendingAsRead(personaId: string): number {
+    const data = this.personas.get(personaId);
+    if (!data) return 0;
+    let count = 0;
+    for (const msg of data.messages) {
+      if (msg.role === "human" && !msg.read) {
+        msg.read = true;
+        count++;
+      }
+    }
+    return count;
+  }
+
+  messages_countUnread(personaId: string): number {
+    const data = this.personas.get(personaId);
+    if (!data) return 0;
+    return data.messages.filter(m => m.role === "system" && !m.read).length;
+  }
+
+  messages_markAllRead(personaId: string): number {
+    const data = this.personas.get(personaId);
+    if (!data) return 0;
+    let count = 0;
+    for (const msg of data.messages) {
+      if (!msg.read) {
+        msg.read = true;
+        count++;
+      }
+    }
+    return count;
+  }
+
+  messages_remove(personaId: string, messageIds: string[]): Message[] {
+    const data = this.personas.get(personaId);
+    if (!data) return [];
+    const idsSet = new Set(messageIds);
+    const removed: Message[] = [];
+    data.messages = data.messages.filter((m) => {
+      if (idsSet.has(m.id)) {
+        removed.push(m);
+        return false;
+      }
+      return true;
+    });
+    return removed;
+  }
+
+  messages_getUnextracted(personaId: string, flag: "f" | "r" | "p" | "o", limit?: number): Message[] {
+    const data = this.personas.get(personaId);
+    if (!data) return [];
+    const unextracted = data.messages.filter(m => m[flag] !== true);
+    if (limit && unextracted.length > limit) {
+      return unextracted.slice(0, limit).map(m => ({ ...m }));
+    }
+    return unextracted.map(m => ({ ...m }));
+  }
+
+  messages_markExtracted(personaId: string, messageIds: string[], flag: "f" | "r" | "p" | "o"): number {
+    const data = this.personas.get(personaId);
+    if (!data) return 0;
+    const idsSet = new Set(messageIds);
+    let count = 0;
+    for (const msg of data.messages) {
+      if (idsSet.has(msg.id) && msg[flag] !== true) {
+        msg[flag] = true;
+        count++;
+      }
+    }
+    return count;
+  }
+}