ei-tui 0.5.4 → 0.6.1

package/src/core/handlers/utils.ts

@@ -1,6 +1,14 @@
 import type { Message, RoomMessage, LLMResponse } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 
+export function getMessageContent(msg: { content?: string; verbal_response?: string; action_response?: string }): string {
+  if (msg.content) return msg.content;
+  const parts: string[] = [];
+  if (msg.action_response) parts.push(`_${msg.action_response}_`);
+  if (msg.verbal_response) parts.push(msg.verbal_response);
+  return parts.join('\n\n');
+}
+
 export function normalizeRoomMessages(messages: RoomMessage[], state: StateManager): Message[] {
   const human = state.getHuman();
   const humanName = human.settings?.name_display ?? "Human";
@@ -12,6 +20,7 @@ export function normalizeRoomMessages(messages: RoomMessage[], state: StateManag
       id: m.id,
       role: m.role === "human" ? "human" as const : "system" as const,
       speaker_name: speakerName,
+      content: m.content,
       verbal_response: m.verbal_response,
       action_response: m.action_response,
       silence_reason: m.silence_reason,
@@ -111,13 +120,6 @@ export function markMessagesExtracted(
   }
 }
 
-/**
- * Returns the combined display text of a message for quote indexing.
- * Mirrors the rendering logic used in the frontends.
- */
 export function getMessageText(message: Message): string {
-  const parts: string[] = [];
-  if (message.action_response) parts.push(`_${message.action_response}_`);
-  if (message.verbal_response) parts.push(message.verbal_response);
-  return parts.join('\n\n');
+  return getMessageContent(message);
 }

package/src/core/heartbeat-manager.ts

@@ -6,6 +6,7 @@ import {
   type Message,
 } from "./types.js";
 import { StateManager } from "./state-manager.js";
+import { getMessageContent } from "./handlers/utils.js";
 import {
   buildHeartbeatCheckPrompt,
   buildEiHeartbeatPrompt,
@@ -15,6 +16,10 @@ import {
 } from "../prompts/index.js";
 import { filterMessagesForContext } from "./context-utils.js";
 import { filterHumanDataByVisibility } from "./prompt-context-builder.js";
+import { cosineSimilarity, computePersonaDescriptionEmbedding } from "./embedding-service.js";
+
+const REFLECTION_SIMILARITY_THRESHOLD = 0.80;
+const REFLECTION_COOLDOWN_MS = 7 * 24 * 60 * 60 * 1000; // 1 week between drift prompts
 
 // =============================================================================
 // MODEL HELPERS
@@ -43,7 +48,7 @@ export function countTrailingPersonaMessages(history: Message[]): number {
   for (let i = history.length - 1; i >= 0; i--) {
     const msg = history[i];
     if (msg.role === "human") break;
-    if (msg.role === "system" && msg.verbal_response && msg.silence_reason === undefined) {
+    if (msg.role === "system" && getMessageContent(msg) && msg.silence_reason === undefined) {
       count++;
     }
   }
@@ -84,6 +89,20 @@ export async function queueEiHeartbeat(
     });
   }
 
+  const newPeople = human.people
+    .filter(p => !p.validated_date)
+    .slice(0, 3);
+  for (const person of newPeople) {
+    const quote = human.quotes.find((q) => q.data_item_ids.includes(person.id));
+    items.push({
+      id: person.id,
+      type: "New Person",
+      name: person.name,
+      description: person.description ?? '',
+      quote: quote?.text,
+    });
+  }
+
   const underEngagedPeople = human.people
     .filter(
       (p) =>
@@ -174,7 +193,7 @@ export async function queueEiHeartbeat(
     user: prompt.user,
     next_step: LLMNextStep.HandleEiHeartbeat,
     model: getModelForPersona(sm, "ei"),
-    data: { personaId: "ei", isTUI },
+    data: { personaId: "ei", isTUI, newPersonIds: newPeople.map(p => p.id) },
   });
 }
 
@@ -213,6 +232,44 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
         b.exposure_desired - b.exposure_current - (a.exposure_desired - a.exposure_current)
     );
 
+  let driftContext: HeartbeatCheckPromptData["drift_context"];
+  const personRecord = sm.human_person_getByIdentifier("ei_persona", personaId);
+
+  if (personRecord?.embedding) {
+    let currentPersona = persona;
+
+    if (!currentPersona.description_embedding) {
+      const embedding = await computePersonaDescriptionEmbedding(currentPersona);
+      if (embedding) {
+        sm.persona_update(personaId, { description_embedding: embedding });
+        currentPersona = { ...currentPersona, description_embedding: embedding };
+      }
+    }
+
+    if (currentPersona.description_embedding) {
+      const lastAsked = currentPersona.reflection_last_asked
+        ? new Date(currentPersona.reflection_last_asked).getTime()
+        : 0;
+
+      // Gate: person must have been updated at least 1 week AFTER reflection was last asked.
+      // This handles both the cooldown AND the extraction echo — the ceremony extraction
+      // that fires right after a persona surfaces drift updates last_updated by minutes,
+      // which can never satisfy the 1-week offset requirement.
+      if (new Date(personRecord.last_updated).getTime() > lastAsked + REFLECTION_COOLDOWN_MS) {
+        const similarity = cosineSimilarity(personRecord.embedding, currentPersona.description_embedding);
+        if (similarity < REFLECTION_SIMILARITY_THRESHOLD) {
+          driftContext = {
+            people_description: personRecord.description ?? '',
+            persona_description: currentPersona.long_description ?? '',
+          };
+          console.log(`[HeartbeatCheck ${persona.display_name}] Drift detected (similarity: ${similarity.toFixed(3)}) - including reflection context`);
+        } else {
+          console.log(`[HeartbeatCheck ${persona.display_name}] Person updated but no drift (similarity: ${similarity.toFixed(3)})`);
+        }
+      }
+    }
+  }
+
   const promptData: HeartbeatCheckPromptData = {
     persona: {
       name: persona.display_name,
@@ -225,6 +282,7 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
     },
     recent_history: contextHistory.slice(-10),
     inactive_days: inactiveDays,
+    drift_context: driftContext,
   };
 
   const prompt = buildHeartbeatCheckPrompt(promptData);

package/src/core/llm-client.ts

@@ -337,7 +337,7 @@ export async function callLLMRaw(
   }
 
   let finalToolCalls = rawToolCalls;
-  if ((!rawToolCalls || rawToolCalls.length === 0) && choice?.finish_reason === "stop" && typeof textContent === "string") {
+  if ((!rawToolCalls || rawToolCalls.length === 0) && choice?.finish_reason === "stop" && typeof textContent === "string" && textContent.trimStart().startsWith("<|tool_call>")) {
     const rescued = rescueGemmaToolCalls(textContent);
     if (rescued.length > 0) {
       console.log(`[LLM] Rescued ${rescued.length} tool call(s) from content (Gemma native format)`);

package/src/core/message-manager.ts

@@ -8,6 +8,7 @@ import {
   type LLMRequest,
 } from "./types.js";
 import { formatTimestamp } from "./format-utils.js";
+import { getMessageContent } from "./handlers/utils.js";
 import { StateManager } from "./state-manager.js";
 import { QueueProcessor } from "./queue-processor.js";
 import {
@@ -113,7 +114,7 @@ export async function recallPendingMessages(
     .map((m) => m.id);
   if (pendingIds.length === 0) return "";
   const removed = sm.messages_remove(personaId, pendingIds);
-  const recalledContent = removed.map((m) => m.verbal_response ?? "").join("\n\n");
+  const recalledContent = removed.map((m) => getMessageContent(m)).join("\n\n");
   onMessageAdded(personaId);
   onMessageRecalled(personaId, recalledContent);
   return recalledContent;
@@ -164,7 +165,7 @@ export async function sendMessage(
   const prompt = buildResponsePrompt(promptData);
 
   sm.queue_enqueue({
-    type: LLMRequestType.Response,
+    type: LLMRequestType.Raw,
     priority: LLMPriority.High,
     system: prompt.system,
     user: prompt.user,

package/src/core/orchestrators/ceremony.ts

@@ -1,5 +1,6 @@
-import { LLMRequestType, LLMPriority, LLMNextStep, type CeremonyConfig, type PersonaTopic, type Topic, type Message, type DataItemBase } from "../types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep, RoomMode, type CeremonyConfig, type PersonaTopic, type Topic, type DataItemBase } from "../types.js";
 import type { StateManager } from "../state-manager.js";
+import { normalizeRoomMessages } from "../handlers/utils.js";
 import { applyDecayToValue } from "../utils/index.js";
 import {
   queueFactFind,
@@ -9,9 +10,9 @@ import {
   type ExtractionContext,
   type ExtractionOptions,
 } from "./human-extraction.js";
-import { queuePersonaTopicScan, type PersonaTopicContext } from "./persona-topics.js";
-import { queueDedupPhase } from "./dedup-phase.js";
-import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt, buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
+import { queuePersonaTopicRating, type PersonaTopicContext, type PersonaTopicOptions } from "./persona-topics.js";
+import { queuePersonMigration } from "./person-migration.js";
+import { buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
 
 export function isNewDay(lastCeremony: string | undefined, now: Date): boolean {
   if (!lastCeremony) return true;
@@ -70,14 +71,14 @@ export function startCeremony(state: StateManager): void {
     },
   });
   
-  // PHASE 1: Deduplication (runs BEFORE Expose)
-  console.log("[ceremony] Starting Phase 1: Deduplication");
-  queueDedupPhase(state);
-  
-  // Check if dedup work was queued
+  // PHASE 1: Person Migration
+  console.log("[ceremony] Starting Phase 1: Person Migration");
+  queuePersonMigration(state);
+
+  // Check if migration work was queued
   if (!state.queue_hasPendingCeremonies()) {
-    // No dedup work found → immediately advance to Expose phase
-    console.log("[ceremony] No dedup work, advancing to Expose phase");
+    // No migration work found → immediately advance to Expose phase
+    console.log("[ceremony] No migration work, advancing to Expose phase");
     handleCeremonyProgress(state, 1);
   }
   
@@ -142,16 +143,21 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
     console.log(`[ceremony:exposure] Queued human extraction scans (f:${unextractedFacts.length}, t:${unextractedTopics.length}, p:${unextractedPeople.length})`);
   }
 
-  const unextractedForPersonaTopics = state.messages_getUnextracted(personaId, "t");
-  if (unextractedForPersonaTopics.length > 0) {
+  const human = state.getHuman();
+  const lastCeremony = human.settings?.ceremony?.last_ceremony;
+  const shortId = personaId.slice(0, 8);
+  const forPersonaTopics = state.messages_getUnextractedForPersona(personaId, shortId, lastCeremony ?? undefined);
+  if (forPersonaTopics.length > 0) {
     const personaTopicContext: PersonaTopicContext = {
       personaId,
       personaDisplayName: persona.display_name,
-      messages_context: allMessages.filter(m => m.t === true),
-      messages_analyze: unextractedForPersonaTopics,
+      messages_context: allMessages.filter(m => !!m.persona_extracted?.[shortId]),
+      messages_analyze: forPersonaTopics,
+      topics: persona.topics,
     };
-    queuePersonaTopicScan(personaTopicContext, state);
-    console.log(`[ceremony:exposure] Queued persona topic scan for ${persona.display_name}`);
+    const personaTopicOptions: PersonaTopicOptions = { ceremony_progress: options?.ceremony_progress };
+    queuePersonaTopicRating(personaTopicContext, state, personaTopicOptions);
+    console.log(`[ceremony:exposure] Queued persona topic rating for ${persona.display_name} (${forPersonaTopics.length} messages)`);
   }
 }
 
@@ -194,6 +200,32 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
     for (const persona of personasWithUnprocessed) {
       queueExposurePhase(persona.id, state, options);
     }
+
+    const rooms = state.getRoomList();
+    for (const room of rooms) {
+      if (room.mode === RoomMode.ChooseYourPath) continue;
+      for (const personaId of room.persona_ids) {
+        const shortId = personaId.slice(0, 8);
+        const unprocessedRaw = state.getRoomUnextractedMessagesForPersona(room.id, shortId);
+        if (unprocessedRaw.length === 0) continue;
+        const personaForRoom = state.persona_getById(personaId);
+        if (!personaForRoom) continue;
+        const allRoomMessagesRaw = state.getRoomActivePath(room.id);
+        const processedIds = new Set(allRoomMessagesRaw.filter(m => !!m.persona_extracted?.[shortId]).map(m => m.id));
+        const allNormalized = normalizeRoomMessages(allRoomMessagesRaw, state);
+        const unprocessedNormalized = normalizeRoomMessages(unprocessedRaw, state);
+        const personaTopicContext: PersonaTopicContext = {
+          personaId,
+          personaDisplayName: personaForRoom.display_name,
+          messages_context: allNormalized.filter(m => processedIds.has(m.id)),
+          messages_analyze: unprocessedNormalized,
+          topics: personaForRoom.topics,
+        };
+        const roomScanOptions: PersonaTopicOptions = { ceremony_progress: 2, roomId: room.id };
+        queuePersonaTopicRating(personaTopicContext, state, roomScanOptions);
+        console.log(`[ceremony:expose] Queued room persona topic rating: ${personaForRoom.display_name} in "${room.display_name}" (${unprocessedRaw.length} messages)`);
+      }
+    }
     return;
   }
 
@@ -238,16 +270,9 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
   runHumanCeremony(state);
 
   // Rewrite phase: fire-and-forget scans for bloated human data items
-  // No ceremony_progress gating — Expire/Explore only touch persona topics, zero overlap
   queueRewritePhase(state);
   
-  // Expire phase: queue LLM calls for each active persona
-  // handlePersonaExpire already chains to Explore → DescriptionCheck
-  for (const persona of activePersonas) {
-    queueExpirePhase(persona.id, state);
-  }
-  
-  console.log("[ceremony:progress] Ceremony Decay complete, Expire queued");
+  console.log("[ceremony:progress] Ceremony Decay complete");
 }
 
 // =============================================================================
@@ -334,124 +359,6 @@ export function prunePersonaMessages(personaId: string, state: StateManager): vo
   }
 }
 
-// =============================================================================
-// EXPIRE PHASE (queues LLM calls)
-// =============================================================================
-
-export function queueExpirePhase(personaId: string, state: StateManager): void {
-  const persona = state.persona_getById(personaId);
-  if (!persona) {
-    console.error(`[ceremony:expire] Persona not found: ${personaId}`);
-    return;
-  }
-  
-  console.log(`[ceremony:expire] Queueing for ${persona.display_name}`);
-  
-  if (persona.topics.length === 0) {
-    console.log(`[ceremony:expire] ${persona.display_name} has no topics, skipping to description check`);
-    queueDescriptionCheck(personaId, state);
-    return;
-  }
-  
-  const prompt = buildPersonaExpirePrompt({
-    persona_name: persona.display_name,
-    topics: persona.topics,
-  });
-  
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonaExpire,
-    data: { personaId, personaDisplayName: persona.display_name },
-  });
-}
-
-// =============================================================================
-// EXPLORE PHASE (queues LLM calls — chained from handlePersonaExpire in handlers)
-// =============================================================================
-
-export function queueExplorePhase(personaId: string, state: StateManager): void {
-  const persona = state.persona_getById(personaId);
-  if (!persona) {
-    console.error(`[ceremony:explore] Persona not found: ${personaId}`);
-    queueDescriptionCheck(personaId, state);
-    return;
-  }
-  
-  console.log(`[ceremony:explore] Queueing for ${persona.display_name}`);
-  
-  const messages = state.messages_get(personaId);
-  const recentMessages = messages.slice(-20);
-  const themes = extractConversationThemes(recentMessages);
-  
-  const prompt = buildPersonaExplorePrompt({
-    persona_name: persona.display_name,
-    traits: persona.traits,
-    remaining_topics: persona.topics,
-    recent_conversation_themes: themes,
-  });
-  
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandlePersonaExplore,
-    data: { personaId, personaDisplayName: persona.display_name },
-  });
-}
-
-function extractConversationThemes(messages: Message[]): string[] {
-  const humanMessages = messages.filter(m => m.role === "human");
-  if (humanMessages.length === 0) return [];
-  
-  const words = humanMessages
-    .map(m => (m.verbal_response ?? '').toLowerCase())
-    .join(" ")
-    .split(/\s+/)
-    .filter(w => w.length > 4);
-  
-  const frequency: Record<string, number> = {};
-  for (const word of words) {
-    frequency[word] = (frequency[word] || 0) + 1;
-  }
-  
-  return Object.entries(frequency)
-    .filter(([_, count]) => count >= 2)
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 5)
-    .map(([word]) => word);
-}
-
-export function queueDescriptionCheck(personaId: string, state: StateManager): void {
-  const persona = state.persona_getById(personaId);
-  if (!persona) {
-    console.error(`[ceremony:description] Persona not found: ${personaId}`);
-    return;
-  }
-  
-  console.log(`[ceremony:description] Queueing for ${persona.display_name}`);
-  
-  const prompt = buildDescriptionCheckPrompt({
-    persona_name: persona.display_name,
-    current_short_description: persona.short_description,
-    current_long_description: persona.long_description,
-    traits: persona.traits,
-    topics: persona.topics,
-  });
-  
-  state.queue_enqueue({
-    type: LLMRequestType.JSON,
-    priority: LLMPriority.Low,
-    system: prompt.system,
-    user: prompt.user,
-    next_step: LLMNextStep.HandleDescriptionCheck,
-    data: { personaId, personaDisplayName: persona.display_name },
-  });
-}
-
 // =============================================================================
 // HUMAN CEREMONY (synchronous — runs during Decay phase)
 // =============================================================================
@@ -550,7 +457,10 @@ export function queueRewritePhase(state: StateManager): void {
     }
   }
   for (const person of human.people) {
-    if ((person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !person.rewrite_checked) {
+    const isPersonaLinked = (person.identifiers ?? []).some(
+      i => i.type.toLowerCase() === 'ei persona'
+    );
+    if (!isPersonaLinked && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !person.rewrite_checked) {
       itemsToScan.push({ item: person, type: "person" });
     }
   }

package/src/core/orchestrators/dedup-phase.ts

@@ -1,208 +1,9 @@
 import { StateManager } from "../state-manager.js";
 import { LLMRequestType, LLMPriority, LLMNextStep, type DataItemBase } from "../types.js";
-import type { DataItemType } from "../types/data-items.js";
-import { buildDedupPrompt } from "../../prompts/ceremony/dedup.js";
 import { buildUserDedupPrompt } from "../../prompts/ceremony/user-dedup.js";
 
-// =============================================================================
-// TYPES
-// =============================================================================
-
 type DedupableItem = DataItemBase & { relationship?: string };
 
-interface Cluster {
-  ids: string[];
-  minSim: number;
-  maxSim: number;
-  size: number;
-}
-
-// =============================================================================
-// DEDUP CANDIDATE FINDING (copied from ceremony.ts)
-// =============================================================================
-
-const DEDUP_DEFAULT_THRESHOLD = 0.95;  // Raised from 0.90: Ei's topic corpus is a single dense project domain — mega-cluster persists all the way to 0.92. At 0.95, max cluster drops to 7 items.
-
-function findDedupCandidates<T extends DedupableItem>(
-  items: T[],
-  threshold: number
-): Array<{ a: T; b: T; similarity: number }> {
-  const withEmbeddings = items.filter(item =>
-    item.embedding && item.embedding.length > 0 &&
-    item.relationship !== "Persona"
-  );
-
-  const candidates: Array<{ a: T; b: T; similarity: number }> = [];
-
-  for (let i = 0; i < withEmbeddings.length; i++) {
-    for (let j = i + 1; j < withEmbeddings.length; j++) {
-      const a = withEmbeddings[i];
-      const b = withEmbeddings[j];
-      const dot = a.embedding!.reduce((sum, v, k) => sum + v * b.embedding![k], 0);
-      const normA = Math.sqrt(a.embedding!.reduce((sum, v) => sum + v * v, 0));
-      const normB = Math.sqrt(b.embedding!.reduce((sum, v) => sum + v * v, 0));
-      const similarity = normA && normB ? dot / (normA * normB) : 0;
-
-      if (similarity >= threshold) {
-        candidates.push({ a, b, similarity });
-      }
-    }
-  }
-
-  return candidates.sort((x, y) => y.similarity - x.similarity);
-}
-
-// =============================================================================
-// UNION-FIND CLUSTERING
-// =============================================================================
-
-function clusterPairs<T extends DedupableItem>(
-  pairs: Array<{ a: T; b: T; similarity: number }>
-): Cluster[] {
-  const parent = new Map<string, string>();
-  
-  function find(x: string): string {
-    if (!parent.has(x)) parent.set(x, x);
-    if (parent.get(x) !== x) parent.set(x, find(parent.get(x)!));
-    return parent.get(x)!;
-  }
-  
-  function union(x: string, y: string): void {
-    const px = find(x), py = find(y);
-    if (px !== py) parent.set(px, py);
-  }
-  
-  // Union all pairs
-  for (const pair of pairs) {
-    union(pair.a.id, pair.b.id);
-  }
-  
-  // Group by root to create clusters
-  const clusters = new Map<string, { ids: string[]; sims: number[] }>();
-  for (const pair of pairs) {
-    const root = find(pair.a.id);
-    if (!clusters.has(root)) {
-      clusters.set(root, { ids: [], sims: [] });
-    }
-    const cluster = clusters.get(root)!;
-    if (!cluster.ids.includes(pair.a.id)) cluster.ids.push(pair.a.id);
-    if (!cluster.ids.includes(pair.b.id)) cluster.ids.push(pair.b.id);
-    cluster.sims.push(pair.similarity);
-  }
-  
-  // Convert to Cluster objects
-  return Array.from(clusters.values()).map(c => ({
-    ids: c.ids,
-    minSim: Math.min(...c.sims),
-    maxSim: Math.max(...c.sims),
-    size: c.ids.length
-  }));
-}
-
-// =============================================================================
-// QUALITY GATES
-// =============================================================================
-
-function filterClusters(clusters: Cluster[]): Cluster[] {
-  return clusters
-    .filter(c => {
-      if (c.size > 50) {
-        console.warn(`[Dedup] Cluster rejected (size too large): ${c.size} items`);
-        return false;
-      }
-      return true;
-    })
-    .filter(c => {
-      const spread = c.maxSim - c.minSim;
-      if (spread > 0.10) {  // 10% threshold
-        console.warn(`[Dedup] Cluster rejected (high spread): ${spread.toFixed(3)} range`);
-        return false;
-      }
-      return true;
-    });
-}
-
-// =============================================================================
-// MAIN QUEUEING FUNCTION
-// =============================================================================
-
-export function queueDedupPhase(state: StateManager): void {
-  const human = state.getHuman();
-  const rewriteModel = human.settings?.rewrite_model;
-  
-  if (!rewriteModel) {
-    console.log("[Dedup] rewrite_model not set — skipping dedup phase");
-    return;
-  }
-  
-  const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
-  
-  console.log(`[Dedup] Starting deduplication phase (threshold: ${threshold})`);
-  
-  const entityTypes: Array<{ type: DataItemType; items: DedupableItem[] }> = [
-    { type: "topic", items: human.topics },
-    { type: "person", items: human.people },
-  ];
-  
-  let totalClusters = 0;
-  
-  for (const { type, items } of entityTypes) {
-    // Find dedup candidates
-    const pairs = findDedupCandidates(items, threshold);
-    
-    if (pairs.length === 0) {
-      console.log(`[Dedup] ${type}: No duplicates found`);
-      continue;
-    }
-    
-    // Cluster pairs via union-find
-    const clusters = clusterPairs(pairs);
-    
-    // Apply quality gates
-    const vettedClusters = filterClusters(clusters);
-    
-    console.log(`[Dedup] ${type}: ${pairs.length} pairs → ${clusters.length} clusters → ${vettedClusters.length} vetted`);
-    
-    // Queue Opus curation for each vetted cluster
-    for (const cluster of vettedClusters) {
-      // Hydrate cluster with full entity data
-      const clusterEntities = cluster.ids
-        .map(id => items.find(item => item.id === id))
-        .filter((item): item is DedupableItem => item !== undefined);
-      
-      if (clusterEntities.length === 0) {
-        console.warn(`[Dedup] Cluster hydration failed - no entities found`);
-        continue;
-      }
-      
-      // Build prompt
-      const prompt = buildDedupPrompt({
-        cluster: clusterEntities,
-        itemType: type as "topic" | "person",
-        similarityRange: { min: cluster.minSim, max: cluster.maxSim }
-      });
-      
-      // Queue LLM request
-      state.queue_enqueue({
-        type: LLMRequestType.JSON,
-        priority: LLMPriority.Normal,
-        system: prompt.system,
-        user: prompt.user,
-        next_step: LLMNextStep.HandleDedupCurate,
-        model: rewriteModel,
-        data: {
-          entity_type: type,
-          entity_ids: cluster.ids,
-          ceremony_progress: 1
-        }
-      });
-      totalClusters++;
-    }
-  }
-  
-  console.log(`[Dedup] Queued ${totalClusters} clusters for curation`);
-}
-
 // =============================================================================
 // USER-TRIGGERED DEDUP
 // =============================================================================

package/src/core/orchestrators/extraction-chunker.ts

@@ -1,5 +1,7 @@
 import type { Message } from "../types.js";
 import type { ExtractionContext } from "./human-extraction.js";
+import { getMessageContent } from "../handlers/utils.js";
+import { getMessageDisplayText } from "../../prompts/message-utils.js";
 
 const DEFAULT_MAX_TOKENS = 10000;
 const CHARS_PER_TOKEN = 4;
@@ -12,7 +14,10 @@ function estimateTokens(text: string): number {
 }
 
 function estimateMessageTokens(messages: Message[]): number {
-  return messages.reduce((sum, msg) => sum + estimateTokens(msg.verbal_response ?? '') + 4, 0);
+  return messages.reduce((sum, msg) => {
+    const text = getMessageDisplayText(msg) ?? getMessageContent(msg);
+    return sum + estimateTokens(text) + 4;
+  }, 0);
 }
 
 function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
@@ -20,7 +25,7 @@ function fitMessagesFromEnd(messages: Message[], maxTokens: number): Message[] {
   let tokens = 0;
 
   for (let i = messages.length - 1; i >= 0; i--) {
-    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
+    const msgTokens = estimateTokens(getMessageContent(messages[i])) + 4;
     if (tokens + msgTokens > maxTokens) break;
     result.unshift(messages[i]);
     tokens += msgTokens;
@@ -39,7 +44,7 @@ function pullMessagesFromStart(
   let i = startIndex;
 
   while (i < messages.length) {
-    const msgTokens = estimateTokens(messages[i].verbal_response ?? '') + 4;
+    const msgTokens = estimateTokens(getMessageContent(messages[i])) + 4;
     if (tokens + msgTokens > maxTokens && pulled.length > 0) break;
     pulled.push(messages[i]);
     tokens += msgTokens;