ei-tui 0.1.17 → 0.1.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.1.17",
+  "version": "0.1.19",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/embedding-service.ts

@@ -201,8 +201,17 @@ function createBunService(): EmbeddingService {
     if (embedderPromise) return embedderPromise;
     
     embedderPromise = (async () => {
-      const mod = await import(/* @vite-ignore */ FASTEMBED_MODULE);
-      embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2 });
+      const [mod, os, path] = await Promise.all([
+        import(/* @vite-ignore */ FASTEMBED_MODULE),
+        import('os'),
+        import('path'),
+      ]);
+      // Use EI_DATA_PATH if set, otherwise fall back to ~/.local/share/ei/embeddings.
+      // Must be absolute so the cache is stable regardless of cwd.
+      const cacheDir = process.env.EI_DATA_PATH
+        ? path.join(process.env.EI_DATA_PATH, 'embeddings')
+        : path.join(os.homedir(), '.local', 'share', 'ei', 'embeddings');
+      embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2, cacheDir });
       return embedder;
     })();
     

package/src/core/handlers/index.ts

@@ -29,8 +29,13 @@ import type {
   PersonaExpireResult, 
   PersonaExploreResult,
   DescriptionCheckResult,
+  RewriteItemType,
+  RewriteScanResult,
+  RewriteResult,
+  RewriteSubjectMatch,
 } from "../../prompts/ceremony/types.js";
-import { 
+import { buildRewritePrompt } from "../../prompts/ceremony/rewrite.js";
+import {
   orchestratePersonaGeneration, 
   queueItemMatch, 
   queueItemUpdate,
@@ -59,6 +64,30 @@ import { crossFind } from "../utils/index.js";
 
 export type ResponseHandler = (response: LLMResponse, state: StateManager) => void | Promise<void>;
 
+// =============================================================================
+// REWRITE HANDLER INJECTION
+// searchHumanData lives on Processor — inject it at startup to avoid circular deps.
+// Same pattern as registerReadMemoryExecutor in tools/index.ts.
+// =============================================================================
+
+type SearchHumanDataFn = (
+  query: string,
+  options?: { types?: Array<'fact' | 'trait' | 'topic' | 'person' | 'quote'>; limit?: number }
+) => Promise<{
+  facts: Fact[];
+  traits: Trait[];
+  topics: Topic[];
+  people: Person[];
+  quotes: Quote[];
+}>;
+
+let _searchHumanData: SearchHumanDataFn | null = null;
+
+/** Called by Processor at startup to inject searchHumanData for rewrite handlers. */
+export function registerSearchHumanData(fn: SearchHumanDataFn): void {
+  _searchHumanData = fn;
+}
+
 function splitMessagesByTimestamp(
   messages: Message[], 
   analyzeFromTimestamp: string | null
@@ -1117,6 +1146,298 @@ function handlePersonaTopicUpdate(response: LLMResponse, state: StateManager): v
 
 
 
+// =============================================================================
+// REWRITE HANDLERS (Ceremony — fire-and-forget)
+// =============================================================================
+
+/**
+ * handleRewriteScan — Phase 1 of Rewrite.
+ * LLM returns an array of subject strings found in the bloated item.
+ * For each subject we search the knowledge base, then queue Phase 2.
+ */
+async function handleRewriteScan(response: LLMResponse, state: StateManager): Promise<void> {
+  const itemId = response.request.data.itemId as string;
+  const itemType = response.request.data.itemType as RewriteItemType;
+  const rewriteModel = response.request.data.rewriteModel as string;
+
+  if (!itemId || !itemType) {
+    console.error("[handleRewriteScan] Missing itemId or itemType in request data");
+    return;
+  }
+
+  const subjects = response.parsed as RewriteScanResult | undefined;
+  if (!subjects || !Array.isArray(subjects) || subjects.length === 0) {
+    console.log(`[handleRewriteScan] No extra subjects found for ${itemType} "${itemId}" — item is cohesive`);
+    return;
+  }
+
+  if (!_searchHumanData) {
+    console.error("[handleRewriteScan] searchHumanData not injected — cannot search for matches");
+    return;
+  }
+
+  // Re-read the item from current state (it may have changed since scan was queued)
+  const human = state.getHuman();
+  const allItems: DataItemBase[] = [
+    ...human.facts, ...human.traits, ...human.topics, ...human.people,
+  ];
+  const currentItem = allItems.find(i => i.id === itemId);
+  if (!currentItem) {
+    console.warn(`[handleRewriteScan] Item ${itemId} no longer exists — skipping rewrite`);
+    return;
+  }
+
+  // Search for matches per subject, excluding the original item
+  const subjectMatches: RewriteSubjectMatch[] = [];
+  for (const searchTerm of subjects) {
+    try {
+      const results = await _searchHumanData(searchTerm, {
+        types: ["fact", "trait", "topic", "person"],
+        limit: 4,  // fetch 4 so we can exclude original and still have 3
+      });
+      const allMatches: DataItemBase[] = [
+        ...results.facts, ...results.traits, ...results.topics, ...results.people,
+      ].filter(m => m.id !== itemId);  // exclude original
+      subjectMatches.push({ searchTerm, matches: allMatches.slice(0, 3) });
+    } catch (err) {
+      console.warn(`[handleRewriteScan] Search failed for "${searchTerm}":`, err);
+      subjectMatches.push({ searchTerm, matches: [] });
+    }
+  }
+
+  // Build Phase 2 prompt and queue it
+  const prompt = buildRewritePrompt({
+    item: currentItem,
+    itemType,
+    subjects: subjectMatches,
+  });
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Normal,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandleRewriteRewrite,
+    model: rewriteModel,
+    data: {
+      itemId,
+      itemType,
+    },
+  });
+
+  console.log(`[handleRewriteScan] Queued Phase 2 for ${itemType} "${currentItem.name}" with ${subjectMatches.length} subject(s)`);
+}
+
+/**
+ * handleRewriteRewrite — Phase 2 of Rewrite.
+ * LLM returns { existing: [...], new: [...] }.
+ * Upsert existing items by id, create new items with defensive defaults + embeddings.
+ */
+async function handleRewriteRewrite(response: LLMResponse, state: StateManager): Promise<void> {
+  const itemId = response.request.data.itemId as string;
+  const itemType = response.request.data.itemType as RewriteItemType;
+
+  if (!itemId || !itemType) {
+    console.error("[handleRewriteRewrite] Missing itemId or itemType in request data");
+    return;
+  }
+
+  const result = response.parsed as RewriteResult | undefined;
+  if (!result || (!result.existing?.length && !result.new?.length)) {
+    console.log(`[handleRewriteRewrite] No changes returned for ${itemType} "${itemId}"`);
+    return;
+  }
+
+  const human = state.getHuman();
+  const now = new Date().toISOString();
+
+  // Look up the original item to inherit persona_groups
+  const allItems: DataItemBase[] = [
+    ...human.facts, ...human.traits, ...human.topics, ...human.people,
+  ];
+  const originalItem = allItems.find(i => i.id === itemId);
+  const inheritedGroups = originalItem?.persona_groups;
+
+  // Helper: resolve actual type from existing records (don't trust LLM's type field)
+  const resolveExistingType = (id: string): RewriteItemType | null => {
+    if (human.facts.find(f => f.id === id)) return "fact";
+    if (human.traits.find(t => t.id === id)) return "trait";
+    if (human.topics.find(t => t.id === id)) return "topic";
+    if (human.people.find(p => p.id === id)) return "person";
+    return null;
+  };
+
+  let existingCount = 0;
+  let newCount = 0;
+
+  // --- Process existing items ---
+  for (const item of result.existing ?? []) {
+    if (!item.id || !item.name || !item.description) {
+      console.warn(`[handleRewriteRewrite] Skipping existing item with missing fields: ${JSON.stringify(item).slice(0, 100)}`);
+      continue;
+    }
+
+    // Resolve type from actual records, not from LLM response
+    const resolvedType = resolveExistingType(item.id);
+    if (!resolvedType) {
+      console.warn(`[handleRewriteRewrite] Existing item id "${item.id}" not found in human data — skipping`);
+      continue;
+    }
+
+    let embedding: number[] | undefined;
+    try {
+      const embeddingService = getEmbeddingService();
+      const text = getItemEmbeddingText({ name: item.name, description: item.description });
+      embedding = await embeddingService.embed(text);
+    } catch (err) {
+      console.warn(`[handleRewriteRewrite] Failed to compute embedding for existing ${resolvedType} "${item.name}":`, err);
+    }
+
+    switch (resolvedType) {
+      case "fact": {
+        const existing = human.facts.find(f => f.id === item.id)!;
+        state.human_fact_upsert({
+          ...existing,
+          name: item.name,
+          description: item.description,
+          sentiment: item.sentiment ?? existing.sentiment,
+          last_updated: now,
+          embedding,
+        });
+        break;
+      }
+      case "trait": {
+        const existing = human.traits.find(t => t.id === item.id)!;
+        state.human_trait_upsert({
+          ...existing,
+          name: item.name,
+          description: item.description,
+          sentiment: item.sentiment ?? existing.sentiment,
+          strength: item.strength ?? existing.strength,
+          last_updated: now,
+          embedding,
+        });
+        break;
+      }
+      case "topic": {
+        const existing = human.topics.find(t => t.id === item.id)!;
+        state.human_topic_upsert({
+          ...existing,
+          name: item.name,
+          description: item.description,
+          sentiment: item.sentiment ?? existing.sentiment,
+          last_updated: now,
+          embedding,
+        });
+        break;
+      }
+      case "person": {
+        const existing = human.people.find(p => p.id === item.id)!;
+        state.human_person_upsert({
+          ...existing,
+          name: item.name,
+          description: item.description,
+          sentiment: item.sentiment ?? existing.sentiment,
+          last_updated: now,
+          embedding,
+        });
+        break;
+      }
+    }
+    existingCount++;
+  }
+
+  // --- Process new items ---
+  for (const item of result.new ?? []) {
+    if (!item.type || !item.name || !item.description) {
+      console.warn(`[handleRewriteRewrite] Skipping new item with missing fields: ${JSON.stringify(item).slice(0, 100)}`);
+      continue;
+    }
+
+    let embedding: number[] | undefined;
+    try {
+      const embeddingService = getEmbeddingService();
+      const text = getItemEmbeddingText({ name: item.name, description: item.description });
+      embedding = await embeddingService.embed(text);
+    } catch (err) {
+      console.warn(`[handleRewriteRewrite] Failed to compute embedding for new ${item.type} "${item.name}":`, err);
+    }
+
+    const baseFields = {
+      id: crypto.randomUUID(),
+      name: item.name,
+      description: item.description,
+      sentiment: item.sentiment ?? 0,
+      last_updated: now,
+      learned_by: "ei",
+      persona_groups: inheritedGroups,
+      embedding,
+    };
+
+    switch (item.type) {
+      case "fact": {
+        const fact: Fact = {
+          ...baseFields,
+          validated: ValidationLevel.None,
+          validated_date: now,
+        };
+        state.human_fact_upsert(fact);
+        break;
+      }
+      case "trait": {
+        const trait: Trait = {
+          ...baseFields,
+          strength: item.strength ?? 0.5,
+        };
+        state.human_trait_upsert(trait);
+        break;
+      }
+      case "topic": {
+        if (!item.category) {
+          console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);
+        }
+        const topic: Topic = {
+          ...baseFields,
+          category: item.category ?? "Interest",
+          exposure_current: 0.5,
+          exposure_desired: 0.5,
+        };
+        state.human_topic_upsert(topic);
+        break;
+      }
+      case "person": {
+        if (!item.relationship) {
+          console.warn(`[handleRewriteRewrite] New person "${item.name}" missing relationship — defaulting to "Unknown"`);
+        }
+        const person: Person = {
+          ...baseFields,
+          relationship: item.relationship ?? "Unknown",
+          exposure_current: 0.5,
+          exposure_desired: 0.5,
+        };
+        state.human_person_upsert(person);
+        break;
+      }
+      default:
+        console.warn(`[handleRewriteRewrite] Unknown type "${item.type}" for new item "${item.name}" — skipping`);
+    }
+    newCount++;
+  }
+
+  console.log(`[handleRewriteRewrite] Complete for ${itemType} "${itemId}": ${existingCount} existing updated, ${newCount} new created`);
+}
+
+/**
+ * handleToolSynthesis — second LLM call in the tool flow.
+ * The QueueProcessor already injected tool history into messages and got the
+ * final persona response. Parse and store it exactly like handlePersonaResponse.
+ */
+function handleToolSynthesis(response: LLMResponse, state: StateManager): void {
+  console.log(`[handleToolSynthesis] Routing to handlePersonaResponse`);
+  handlePersonaResponse(response, state);
+}
+
+
 export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handlePersonaResponse,
   handlePersonaGeneration,
@@ -1137,4 +1458,7 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handlePersonaExpire,
   handlePersonaExplore,
   handleDescriptionCheck,
+  handleToolSynthesis,
+  handleRewriteScan,
+  handleRewriteRewrite,
 };

package/src/core/llm-client.ts

@@ -17,11 +17,17 @@ export interface ResolvedModel {
 export interface LLMCallOptions {
   signal?: AbortSignal;
   temperature?: number;
+  /** OpenAI-compatible tools array. When present and non-empty, sent with tool_choice: "auto". */
+  tools?: Record<string, unknown>[];
 }
 
 export interface LLMRawResponse {
   content: string | null;
   finishReason: string | null;
+  /** Raw tool_calls array from the API response, present when finishReason is "tool_calls". */
+  rawToolCalls?: unknown[];
+  /** The full assistant message object (needed to inject into history for the tool loop). */
+  assistantMessage?: Record<string, unknown>;
 }
 
 let llmCallCount = 0;
@@ -170,14 +176,21 @@ export async function callLLMRaw(
     headers["anthropic-dangerous-direct-browser-access"] = "true";
   }
   
+  const requestBody: Record<string, unknown> = {
+    model,
+    messages: finalMessages,
+    temperature,
+  };
+
+  if (options.tools && options.tools.length > 0) {
+    requestBody.tools = options.tools;
+    requestBody.tool_choice = "auto";
+  }
+
   const response = await fetch(`${normalizedBaseURL}/chat/completions`, {
     method: "POST",
     headers,
-    body: JSON.stringify({
-      model,
-      messages: finalMessages,
-      temperature,
-    }),
+    body: JSON.stringify(requestBody),
     signal,
   });
   
@@ -189,9 +202,16 @@ export async function callLLMRaw(
   const data = await response.json();
   const choice = data.choices?.[0];
   
+  const assistantMessage = choice?.message as Record<string, unknown> | undefined;
+  const rawToolCalls = Array.isArray(choice?.message?.tool_calls)
+    ? (choice.message.tool_calls as unknown[])
+    : undefined;
+
   return {
-    content: choice?.message?.content ?? null,
+    content: (choice?.message?.content as string | null) ?? null,
     finishReason: choice?.finish_reason ?? null,
+    rawToolCalls,
+    assistantMessage,
   };
 }
 

package/src/core/orchestrators/ceremony.ts

@@ -10,7 +10,7 @@ import {
   type ExtractionOptions,
 } from "./human-extraction.js";
 import { queuePersonaTopicScan, type PersonaTopicContext } from "./persona-topics.js";
-import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt } from "../../prompts/ceremony/index.js";
+import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt, buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
 
 export function isNewDay(lastCeremony: string | undefined, now: Date): boolean {
   if (!lastCeremony) return true;
@@ -227,6 +227,10 @@ export function handleCeremonyProgress(state: StateManager): void {
 
   // Dedup phase: log near-duplicate human entities (dry-run only, no mutations)
   runDedupPhase(state);
+
+  // Rewrite phase: fire-and-forget scans for bloated human data items
+  // No ceremony_progress gating — Expire/Explore only touch persona topics, zero overlap
+  queueRewritePhase(state);
   
   // Expire phase: queue LLM calls for each active persona
   // handlePersonaExpire already chains to Explore → DescriptionCheck
@@ -574,3 +578,78 @@ export function runDedupPhase(state: StateManager): void {
 
   console.log(`[ceremony:dedup] Done. ${totalCandidates} total candidate pair(s) found.`);
 }
+
+// =============================================================================
+// REWRITE PHASE (fire-and-forget — queues Low-priority Phase 1 scans)
+// =============================================================================
+
+const REWRITE_DESCRIPTION_THRESHOLD = 750;
+
+/**
+ * Queue Phase 1 "scan" for every human data item whose description exceeds the
+ * threshold. Gated on rewrite_model being set in HumanSettings.
+ * 
+ * Fire-and-forget: no ceremony_progress, no blocking. Expire/Explore proceed
+ * immediately since they only touch persona topics (zero overlap with human data).
+ * Phase 2 items enqueue at Normal priority, naturally processing before more
+ * Low-priority Phase 1 scans.
+ */
+export function queueRewritePhase(state: StateManager): void {
+  const human = state.getHuman();
+  const rewriteModel = human.settings?.rewrite_model;
+
+  if (!rewriteModel) {
+    console.log("[ceremony:rewrite] rewrite_model not set — skipping rewrite phase");
+    return;
+  }
+
+  const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
+
+  for (const fact of human.facts) {
+    if ((fact.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+      itemsToScan.push({ item: fact, type: "fact" });
+    }
+  }
+  for (const trait of human.traits) {
+    if ((trait.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+      itemsToScan.push({ item: trait, type: "trait" });
+    }
+  }
+  for (const topic of human.topics) {
+    if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+      itemsToScan.push({ item: topic, type: "topic" });
+    }
+  }
+  for (const person of human.people) {
+    if ((person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
+      itemsToScan.push({ item: person, type: "person" });
+    }
+  }
+
+  if (itemsToScan.length === 0) {
+    console.log("[ceremony:rewrite] No items above threshold — nothing to rewrite");
+    return;
+  }
+
+  console.log(`[ceremony:rewrite] Found ${itemsToScan.length} item(s) above ${REWRITE_DESCRIPTION_THRESHOLD} chars — queueing Phase 1 scans`);
+
+  for (const { item, type } of itemsToScan) {
+    const prompt = buildRewriteScanPrompt({ item, itemType: type });
+
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleRewriteScan,
+      model: rewriteModel,
+      data: {
+        itemId: item.id,
+        itemType: type,
+        rewriteModel,  // pass through so Phase 1 handler can queue Phase 2 with the same model
+      },
+    });
+  }
+
+  console.log(`[ceremony:rewrite] Queued ${itemsToScan.length} Phase 1 scan(s) at Low priority`);
+}

package/src/core/orchestrators/human-extraction.ts

@@ -281,11 +281,14 @@ export async function queueItemMatch(
       break;
   }
 
+  // Traits are personality patterns — they must only match against other traits.
+  // Non-trait candidates (facts, topics, people) must never absorb trait content,
+  // and trait candidates must never cross-match into facts/topics/people.
   const allItemsWithEmbeddings = [
-    ...human.facts.map(f => ({ ...f, data_type: "fact" as DataItemType })),
+    ...(dataType !== "trait" ? human.facts.map(f => ({ ...f, data_type: "fact" as DataItemType })) : []),
     ...human.traits.map(t => ({ ...t, data_type: "trait" as DataItemType })),
-    ...human.topics.map(t => ({ ...t, data_type: "topic" as DataItemType })),
-    ...human.people.map(p => ({ ...p, data_type: "person" as DataItemType })),
+    ...(dataType !== "trait" ? human.topics.map(t => ({ ...t, data_type: "topic" as DataItemType })) : []),
+    ...(dataType !== "trait" ? human.people.map(p => ({ ...p, data_type: "person" as DataItemType })) : []),
   ].filter(item => item.embedding && item.embedding.length > 0);
 
   let topKItems: Array<{
@@ -321,15 +324,18 @@ export async function queueItemMatch(
   }
 
   if (topKItems.length === 0) {
-    console.log(`[queueItemMatch] No embeddings available, using all ${human.facts.length + human.traits.length + human.topics.length + human.people.length} items`);
-    
-    for (const fact of human.facts) {
-      topKItems.push({
-        data_type: "fact",
-        data_id: fact.id,
-        data_name: fact.name,
-        data_description: dataType === "fact" ? fact.description : truncateDescription(fact.description),
-      });
+
+    console.log(`[queueItemMatch] No embeddings available, using filtered items (dataType=${dataType})`);
+
+    if (dataType !== "trait") {
+      for (const fact of human.facts) {
+        topKItems.push({
+          data_type: "fact",
+          data_id: fact.id,
+          data_name: fact.name,
+          data_description: dataType === "fact" ? fact.description : truncateDescription(fact.description),
+        });
+      }
     }
 
     for (const trait of human.traits) {
@@ -341,22 +347,24 @@ export async function queueItemMatch(
       });
     }
 
-    for (const topic of human.topics) {
-      topKItems.push({
-        data_type: "topic",
-        data_id: topic.id,
-        data_name: topic.name,
-        data_description: dataType === "topic" ? topic.description : truncateDescription(topic.description),
-      });
-    }
-
-    for (const person of human.people) {
-      topKItems.push({
-        data_type: "person",
-        data_id: person.id,
-        data_name: person.name,
-        data_description: dataType === "person" ? person.description : truncateDescription(person.description),
-      });
+    if (dataType !== "trait") {
+      for (const topic of human.topics) {
+        topKItems.push({
+          data_type: "topic",
+          data_id: topic.id,
+          data_name: topic.name,
+          data_description: dataType === "topic" ? topic.description : truncateDescription(topic.description),
+        });
+      }
+
+      for (const person of human.people) {
+        topKItems.push({
+          data_type: "person",
+          data_id: person.id,
+          data_name: person.name,
+          data_description: dataType === "person" ? person.description : truncateDescription(person.description),
+        });
+      }
     }
   }
 

package/src/core/orchestrators/index.ts

@@ -19,6 +19,7 @@ export {
   queueExplorePhase,
   queueDescriptionCheck,
   runHumanCeremony,
+  queueRewritePhase,
 } from "./ceremony.js";
 export {
   queuePersonaTopicScan,