ei-tui 0.1.3 → 0.1.5

package/src/core/state/queue.ts

@@ -1,4 +1,5 @@
-import type { LLMRequest, LLMNextStep, QueueFailResult } from "../types.js";
+import type { LLMRequest, QueueFailResult } from "../types.js";
+import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -27,35 +28,55 @@ export class QueueState {
   private paused = false;
 
   load(queue: LLMRequest[]): void {
-    this.queue = queue;
+    // Reset any items stuck in 'processing' from a previous session
+    this.queue = queue.map(r =>
+      r.state === "processing" ? { ...r, state: "pending" } : r
+    );
   }
 
   export(): LLMRequest[] {
     return this.queue;
   }
 
-  enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts">): string {
+  enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts" | "state">): string {
     const id = crypto.randomUUID();
     const fullRequest: LLMRequest = {
       ...request,
       id,
       created_at: new Date().toISOString(),
       attempts: 0,
+      state: "pending",
     };
     this.queue.push(fullRequest);
     return id;
   }
 
-  peekHighest(): LLMRequest | null {
+  claimHighest(): LLMRequest | null {
     if (this.paused || this.queue.length === 0) return null;
-    const now = new Date().toISOString();
-    const available = this.queue.filter(r => !r.retry_after || r.retry_after <= now);
+    const available = this.queue.filter(r => r.state === "pending");
     if (available.length === 0) return null;
     const priorityOrder = { high: 0, normal: 1, low: 2 };
     const sorted = [...available].sort(
       (a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]
     );
-    return sorted[0] ?? null;
+    const item = sorted[0] ?? null;
+    if (item) {
+      item.state = "processing";
+    }
+    return item;
+  }
+
+  // Returns the retry_after of the highest-priority pending item, or null if ready now.
+  // Used by the processor loop to decide whether to sleep instead of claiming.
+  nextItemRetryAfter(): string | null {
+    if (this.paused || this.queue.length === 0) return null;
+    const available = this.queue.filter(r => r.state === "pending");
+    if (available.length === 0) return null;
+    const priorityOrder = { high: 0, normal: 1, low: 2 };
+    const sorted = [...available].sort(
+      (a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]
+    );
+    return sorted[0]?.retry_after ?? null;
   }
 
   complete(id: string): void {
@@ -76,38 +97,32 @@ export class QueueState {
     }
 
     // No error string and not flagged permanent = just increment, no classification
+    // Still reset to pending in case it was claimed (processing state)
     if (!error && !permanent) {
+      request.state = "pending";
       return { dropped: false };
     }
 
     const shouldDrop = permanent || (error ? isPermanentError(error) : false);
 
     if (shouldDrop) {
-      this.queue.splice(idx, 1);
+      request.state = "dlq";
       return { dropped: true };
     }
 
-    // Transient error — apply exponential backoff, never drop
+    // Transient error — reset to pending with backoff timer
+    request.state = "pending";
     const delay = calculateBackoff(request.attempts);
     request.retry_after = new Date(Date.now() + delay).toISOString();
     return { dropped: false, retryDelay: delay };
   }
 
-  getValidations(): LLMRequest[] {
-    return this.queue.filter(
-      (r) => r.next_step === ("handleEiValidation" as LLMNextStep)
-    );
-  }
 
-  clearValidations(ids: string[]): void {
-    const idSet = new Set(ids);
-    this.queue = this.queue.filter((r) => !idSet.has(r.id));
-  }
 
   clearPersonaResponses(personaId: string, nextStep: string): string[] {
     const removedIds: string[] = [];
     this.queue = this.queue.filter((r) => {
-      if (r.next_step === nextStep && r.data.personaId === personaId) {
+      if (r.state !== "dlq" && r.next_step === nextStep && r.data.personaId === personaId) {
         removedIds.push(r.id);
         return false;
       }
@@ -117,7 +132,49 @@ export class QueueState {
   }
 
   length(): number {
-    return this.queue.length;
+    return this.queue.filter(r => r.state === "pending" || r.state === "processing").length;
+  }
+
+  dlqLength(): number {
+    return this.queue.filter(r => r.state === "dlq").length;
+  }
+
+  hasProcessingItem(): boolean {
+    return this.queue.some(r => r.state === "processing");
+  }
+
+  getDLQItems(): LLMRequest[] {
+    return this.queue.filter(r => r.state === "dlq");
+  }
+
+  getAllActiveItems(): LLMRequest[] {
+    return this.queue.filter(r => r.state !== "dlq");
+  }
+
+  updateItem(id: string, updates: Partial<LLMRequest>): boolean {
+    const idx = this.queue.findIndex(r => r.id === id);
+    if (idx < 0) return false;
+    this.queue[idx] = { ...this.queue[idx], ...updates };
+    return true;
+  }
+
+  trimDLQ(): number {
+    const dlqItems = this.queue.filter(r => r.state === "dlq");
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - DLQ_MAX_AGE_DAYS);
+    const cutoffISO = cutoff.toISOString();
+
+    // Remove by age first
+    const afterAgeTrim = dlqItems.filter(r => r.created_at >= cutoffISO);
+
+    // Then enforce count limit (oldest first)
+    const sorted = afterAgeTrim.sort((a, b) => a.created_at.localeCompare(b.created_at));
+    const kept = sorted.slice(-DLQ_MAX_COUNT);
+    const keptIds = new Set(kept.map(r => r.id));
+
+    const before = this.queue.length;
+    this.queue = this.queue.filter(r => r.state !== "dlq" || keptIds.has(r.id));
+    return before - this.queue.length;
   }
 
   pause(): void {
@@ -133,12 +190,12 @@ export class QueueState {
   }
 
   hasPendingCeremonies(): boolean {
-    return this.queue.some(r => r.data.ceremony_progress === true);
+    return this.queue.some(r => r.state !== "dlq" && r.data.ceremony_progress === true);
   }
 
   clear(): number {
-    const count = this.queue.length;
-    this.queue = [];
+    const count = this.queue.filter(r => r.state !== "dlq").length;
+    this.queue = this.queue.filter(r => r.state === "dlq");
     return count;
   }
 }

package/src/core/state-manager.ts

@@ -250,7 +250,7 @@ export class StateManager {
     return result;
   }
 
-  queue_enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts">): string {
+  queue_enqueue(request: Omit<LLMRequest, "id" | "created_at" | "attempts" | "state">): string {
     const requestWithModel = {
       ...request,
       model: request.model ?? this.humanState.get().settings?.default_model,
@@ -260,8 +260,8 @@ export class StateManager {
     return id;
   }
 
-  queue_peekHighest(): LLMRequest | null {
-    return this.queueState.peekHighest();
+  queue_claimHighest(): LLMRequest | null {
+    return this.queueState.claimHighest();
   }
 
   queue_complete(id: string): void {
@@ -275,14 +275,7 @@ export class StateManager {
     return result;
   }
 
-  queue_getValidations(): LLMRequest[] {
-    return this.queueState.getValidations();
-  }
 
-  queue_clearValidations(ids: string[]): void {
-    this.queueState.clearValidations(ids);
-    this.scheduleSave();
-  }
 
   queue_clearPersonaResponses(personaId: string, nextStep: string): string[] {
     const result = this.queueState.clearPersonaResponses(personaId, nextStep);
@@ -294,6 +287,14 @@ export class StateManager {
     return this.queueState.length();
   }
 
+  queue_hasProcessingItem(): boolean {
+    return this.queueState.hasProcessingItem();
+  }
+
+  queue_nextItemRetryAfter(): string | null {
+    return this.queueState.nextItemRetryAfter();
+  }
+
   queue_pause(): void {
     this.queueState.pause();
     this.scheduleSave();
@@ -318,6 +319,30 @@ export class StateManager {
     return result;
   }
 
+  queue_dlqLength(): number {
+    return this.queueState.dlqLength();
+  }
+
+  queue_getDLQItems(): LLMRequest[] {
+    return this.queueState.getDLQItems();
+  }
+
+  queue_getAllActiveItems(): LLMRequest[] {
+    return this.queueState.getAllActiveItems();
+  }
+
+  queue_updateItem(id: string, updates: Partial<LLMRequest>): boolean {
+    const result = this.queueState.updateItem(id, updates);
+    if (result) this.scheduleSave();
+    return result;
+  }
+
+  queue_trimDLQ(): number {
+    const result = this.queueState.trimDLQ();
+    if (result > 0) this.scheduleSave();
+    return result;
+  }
+
   async flush(): Promise<void> {
     await this.persistenceState.flush();
   }
@@ -338,6 +363,7 @@ export class StateManager {
     this.humanState.load(state.human);
     this.personaState.load(state.personas);
     this.queueState.load(state.queue);
+    this.persistenceState.markExistingData();
     this.scheduleSave();
   }
 

package/src/core/types.ts

@@ -18,7 +18,6 @@ export enum ValidationLevel {
   Ei = "ei",         // Ei mentioned it to user (don't mention again)
   Human = "human",   // User explicitly confirmed (locked)
 }
-
 export enum LLMRequestType {
   Response = "response",
   JSON = "json",
@@ -47,9 +46,7 @@ export enum LLMNextStep {
   HandlePersonaTopicUpdate = "handlePersonaTopicUpdate",
   HandleHeartbeatCheck = "handleHeartbeatCheck",
   HandleEiHeartbeat = "handleEiHeartbeat",
-  HandleEiValidation = "handleEiValidation",
   HandleOneShot = "handleOneShot",
-  // Ceremony handlers
   HandlePersonaExpire = "handlePersonaExpire",
   HandlePersonaExplore = "handlePersonaExplore",
   HandleDescriptionCheck = "handleDescriptionCheck",
@@ -83,6 +80,7 @@ export interface Topic extends DataItemBase {
   category?: string; // Interest, Goal, Dream, Conflict, Concern, Fear, Hope, Plan, Project
   exposure_current: number;
   exposure_desired: number;
+  last_ei_asked?: string | null;  // ISO timestamp of last time Ei proactively asked about this
 }
 
 /**
@@ -109,6 +107,7 @@ export interface Person extends DataItemBase {
   relationship: string;
   exposure_current: number;
   exposure_desired: number;
+  last_ei_asked?: string | null;  // ISO timestamp of last time Ei proactively asked about this
 }
 
 export interface Quote {
@@ -180,7 +179,8 @@ export interface OpenCodeSettings {
   integration?: boolean;
   polling_interval_ms?: number;  // Default: 1800000 (30 min)
   last_sync?: string;  // ISO timestamp
-  extraction_point?: string;  // ISO timestamp - earliest unprocessed message, gradual extraction advances this
+  extraction_point?: string;  // ISO timestamp - cursor for single-session archive scan
+  processed_sessions?: Record<string, string>;  // sessionId → ISO timestamp of last import
 }
 
 export interface CeremonyConfig {
@@ -238,7 +238,6 @@ export interface PersonaEntity {
   last_activity: string;
   last_heartbeat?: string;
   last_extraction?: string;
-  last_inactivity_ping?: string;
 }
 
 export interface PersonaCreationInput {
@@ -257,6 +256,10 @@ export interface PersonaCreationInput {
 export const MESSAGE_MIN_COUNT = 200;
 export const MESSAGE_MAX_AGE_DAYS = 14;
 
+// DLQ rolloff thresholds
+export const DLQ_MAX_COUNT = 50;
+export const DLQ_MAX_AGE_DAYS = 14;
+
 // Reserved persona names (command keywords that conflict with /persona subcommands)
 export const RESERVED_PERSONA_NAMES = ["new", "clone"] as const;
 export type ReservedPersonaName = typeof RESERVED_PERSONA_NAMES[number];
@@ -272,14 +275,19 @@ export function isReservedPersonaName(name: string): boolean {
 export interface Message {
   id: string;
   role: "human" | "system";
-  content: string;
+  verbal_response?: string;   // Human text or persona's spoken reply
+  action_response?: string;  // Stage direction / action the persona performs
+  silence_reason?: string;   // Why the persona chose not to respond (not shown to LLM)
   timestamp: string;
-  read: boolean;
+  read: boolean;               // Has human seen this system message?
   context_status: ContextStatus;
-  f?: boolean;  // fact extraction done
-  r?: boolean;  // trait extraction done
-  p?: boolean;  // person extraction done
-  o?: boolean;  // topic extraction done
+  
+  // Extraction completion flags (omit when false to save space)
+  // Single-letter names minimize storage overhead for large message histories
+  f?: boolean;                 // Fact extraction completed
+  r?: boolean;                 // tRait extraction completed
+  p?: boolean;                 // Person extraction completed
+  o?: boolean;                 // tOpic extraction completed
 }
 
 export interface ChatMessage {
@@ -291,12 +299,15 @@ export interface ChatMessage {
 // LLM TYPES
 // =============================================================================
 
+export type LLMRequestState = "pending" | "processing" | "dlq";
+
 export interface LLMRequest {
   id: string;
   created_at: string;
   attempts: number;
   last_attempt?: string;
   retry_after?: string;
+  state: LLMRequestState;
   type: LLMRequestType;
   priority: LLMPriority;
   system: string;
@@ -346,6 +357,7 @@ export interface MessageQueryOptions {
 export interface QueueStatus {
   state: "idle" | "busy" | "paused";
   pending_count: number;
+  dlq_count: number;
   current_operation?: string;
 }
 

package/src/core/utils/crossFind.ts

@@ -0,0 +1,44 @@
+import type { HumanEntity, PersonaEntity, Fact, Trait, Topic, Person, Quote, PersonaTopic } from "../types.ts";
+export type CrossFindResult =
+  | { type: "fact"        } & Fact
+  | { type: "trait"       } & Trait
+  | { type: "topic"       } & Topic
+  | { type: "person"      } & Person
+  | { type: "quote"       } & Quote
+  | { type: "persona"     } & PersonaEntity
+  | { type: "personaTopic"; personaId: string } & PersonaTopic
+  | { type: "personaTrait"; personaId: string } & Trait;
+
+export function crossFind(
+  id: string,
+  human: HumanEntity,
+  personas?: PersonaEntity[],
+): CrossFindResult | null {
+
+  const fact  = human.facts.find(f => f.id === id);
+  if (fact)  return { type: "fact",   ...fact };
+
+  const trait = human.traits.find(t => t.id === id);
+  if (trait) return { type: "trait",  ...trait };
+
+  const person = human.people.find(p => p.id === id);
+  if (person) return { type: "person", ...person };
+
+  const topic = human.topics.find(t => t.id === id);
+  if (topic) return { type: "topic",  ...topic };
+
+  const quote = human.quotes.find(q => q.id === id);
+  if (quote) return { type: "quote",  ...quote };
+
+  for (const persona of personas ?? []) {
+    if (persona.id === id) return { type: "persona", ...persona };
+
+    const pTopic = persona.topics.find(t => t.id === id);
+    if (pTopic) return { type: "personaTopic", personaId: persona.id, ...pTopic };
+
+    const pTrait = persona.traits.find(t => t.id === id);
+    if (pTrait) return { type: "personaTrait", personaId: persona.id, ...pTrait };
+  }
+
+  return null;
+}

package/src/core/utils/index.ts

@@ -0,0 +1,4 @@
+import { crossFind } from "./crossFind.js";
+import { applyDecayToValue } from "./decay.js";
+
+export { crossFind, applyDecayToValue };