ei-tui 0.1.25 → 0.2.0

package/src/core/handlers/human-matching.ts

@@ -1,239 +1,312 @@
 import {
-  ValidationLevel,
   type LLMResponse,
   type Message,
-  type Trait,
   type Topic,
-  type Fact,
   type Person,
   type Quote,
-  type DataItemType,
-  type DataItemBase,
 } from "../types.js";
 import type { StateManager } from "../state-manager.js";
-import type { ItemMatchResult, ItemUpdateResult, ExposureImpact } from "../../prompts/human/types.js";
-import { queueItemUpdate, type ExtractionContext } from "../orchestrators/index.js";
-import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
-import { crossFind } from "../utils/index.js";
-import { splitMessagesByTimestamp, getMessageText } from "./utils.js";
+import type { ItemMatchResult, ExposureImpact, TopicUpdateResult, PersonUpdateResult } from "../../prompts/human/types.js";
+import { queueTopicUpdate, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
+import { getEmbeddingService, getTopicEmbeddingText, getPersonEmbeddingText } from "../embedding-service.js";
 
-export function handleHumanItemMatch(response: LLMResponse, state: StateManager): void {
+function mergeGroups(personaGroup: string | null, isNewItem: boolean, existing: string[] | undefined): string[] | undefined {
+  if (!personaGroup) return existing;
+  if (isNewItem) return [personaGroup];
+  const groups = new Set(existing ?? []);
+  groups.add(personaGroup);
+  return Array.from(groups);
+}
+import { resolveMessageWindow, getMessageText } from "./utils.js";
+
+export function handleTopicMatch(response: LLMResponse, state: StateManager): void {
   const result = response.parsed as ItemMatchResult | undefined;
   if (!result) {
-    console.error("[handleHumanItemMatch] No parsed result");
+    console.error("[handleTopicMatch] No parsed result");
     return;
   }
 
-  const candidateType = response.request.data.candidateType as DataItemType;
   const personaId = response.request.data.personaId as string;
   const personaDisplayName = response.request.data.personaDisplayName as string;
-  const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
-  const allMessages = state.messages_get(personaId);
+  const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
 
-  let messages_context: Message[];
-  let messages_analyze: Message[];
-
-  if (messageIdsToMark && messageIdsToMark.length > 0) {
-    const messageIdSet = new Set(messageIdsToMark);
-    messages_analyze = allMessages.filter(m => messageIdSet.has(m.id));
-    const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
-    messages_context = allMessages.filter(m => 
-      !messageIdSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
-    );
-  } else {
-    // Fallback to existing behavior
-    const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
-    const split = splitMessagesByTimestamp(allMessages, analyzeFrom);
-    messages_context = split.messages_context;
-    messages_analyze = split.messages_analyze;
+  let matched_guid = result.matched_guid;
+  if (matched_guid === "new") {
+    matched_guid = null;
+  } else if (matched_guid) {
+    const human = state.getHuman();
+    const found = human.topics.find(t => t.id === matched_guid);
+    if (!found) {
+      console.warn(`[handleTopicMatch] matched_guid "${matched_guid}" not found in topics — treating as new`);
+      matched_guid = null;
+    }
   }
+  result.matched_guid = matched_guid;
 
-  const context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string } = {
+  const context: ExtractionContext & {
+    candidateName: string;
+    candidateDescription: string;
+    candidateCategory: string;
+    extraction_model?: string;
+  } = {
     personaId,
     personaDisplayName,
     messages_context,
     messages_analyze,
-    itemName: response.request.data.itemName as string,
-    itemValue: response.request.data.itemValue as string,
-    itemCategory: response.request.data.itemCategory as string | undefined,
+    candidateName: response.request.data.candidateName as string,
+    candidateDescription: response.request.data.candidateDescription as string,
+    candidateCategory: response.request.data.candidateCategory as string,
+    extraction_model: response.request.data.extraction_model as string | undefined,
   };
 
-  let resolvedType: DataItemType = candidateType;
+  queueTopicUpdate(result, context, state);
+  const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new topic)";
+  console.log(`[handleTopicMatch] topic "${context.candidateName}": ${matched}`);
+}
+
+export function handlePersonMatch(response: LLMResponse, state: StateManager): void {
+  const result = response.parsed as ItemMatchResult | undefined;
+  if (!result) {
+    console.error("[handlePersonMatch] No parsed result");
+    return;
+  }
+
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+  const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
+
   let matched_guid = result.matched_guid;
   if (matched_guid === "new") {
     matched_guid = null;
   } else if (matched_guid) {
-    const found = crossFind(matched_guid, state.getHuman());
+    const human = state.getHuman();
+    const found = human.people.find(p => p.id === matched_guid);
     if (!found) {
-      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" not found in human data — treating as new item`);
+      console.warn(`[handlePersonMatch] matched_guid "${matched_guid}" not found in people — treating as new`);
       matched_guid = null;
-    } else if (found.type === "fact" && found.validated === ValidationLevel.Human) {
-      console.log(`[handleHumanItemMatch] Skipping locked fact "${found.name}" (human-validated)`);
-      return;
-    } else if (!(found.type === "fact" || found.type === "trait" || found.type === "topic" || found.type === "person")) {
-      console.warn(`[handleHumanItemMatch] matched_guid "${matched_guid}" resolved to non-human type "${found.type}" - Ignoring`);
-      return;
-    } else {
-      resolvedType = found.type;
-      context.itemName = found.name || context.itemName;
-      context.itemValue = found.description || context.itemValue;
     }
   }
   result.matched_guid = matched_guid;
-  queueItemUpdate(resolvedType, result, context, state);
-  const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new item)";
-  console.log(`[handleHumanItemMatch] ${resolvedType} "${context.itemName}": ${matched}`);
+
+  const context: ExtractionContext & {
+    candidateName: string;
+    candidateDescription: string;
+    candidateRelationship: string;
+    extraction_model?: string;
+  } = {
+    personaId,
+    personaDisplayName,
+    messages_context,
+    messages_analyze,
+    candidateName: response.request.data.candidateName as string,
+    candidateDescription: response.request.data.candidateDescription as string,
+    candidateRelationship: response.request.data.candidateRelationship as string,
+    extraction_model: response.request.data.extraction_model as string | undefined,
+  };
+
+  queuePersonUpdate(result, context, state);
+  const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new person)";
+  console.log(`[handlePersonMatch] person "${context.candidateName}": ${matched}`);
 }
 
-export async function handleHumanItemUpdate(response: LLMResponse, state: StateManager): Promise<void> {
-  const result = response.parsed as ItemUpdateResult | undefined;
-  
+export async function handleTopicUpdate(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as (TopicUpdateResult & { quotes?: Array<{ text: string; reason: string }> }) | undefined;
+
   if (!result || Object.keys(result).length === 0) {
-    console.log("[handleHumanItemUpdate] No changes needed (empty result)");
+    console.log("[handleTopicUpdate] No changes needed (empty result)");
     return;
   }
 
-  const candidateType = response.request.data.candidateType as DataItemType;
   const isNewItem = response.request.data.isNewItem as boolean;
   const existingItemId = response.request.data.existingItemId as string | undefined;
   const personaId = response.request.data.personaId as string;
   const personaDisplayName = response.request.data.personaDisplayName as string;
+  const candidateCategory = response.request.data.candidateCategory as string | undefined;
 
   if (!result.name || !result.description || result.sentiment === undefined) {
-    console.error("[handleHumanItemUpdate] Missing required fields in result");
+    console.error("[handleTopicUpdate] Missing required fields in result");
     return;
   }
 
   const now = new Date().toISOString();
+  const human = state.getHuman();
+
   const resolveItemId = (): string => {
     if (isNewItem || !existingItemId) return crypto.randomUUID();
-    const h = state.getHuman();
-    const arr = candidateType === "fact" ? h.facts : candidateType === "trait" ? h.traits : candidateType === "topic" ? h.topics : h.people;
-    // Guard: if existingItemId isn't in the correct type array, treat as new
-    // (prevents cross-type ID reuse when LLM matches against a different type's UUID)
-    return arr.find((x: DataItemBase) => x.id === existingItemId) ? existingItemId : crypto.randomUUID();
+    return human.topics.find(t => t.id === existingItemId) ? existingItemId : crypto.randomUUID();
   };
   const itemId = resolveItemId();
 
   const persona = state.persona_getById(personaId);
   const personaGroup = persona?.group_primary ?? null;
-  const isEi = personaDisplayName.toLowerCase() === "ei";
 
-  const human = state.getHuman();
-  const getExistingItem = (): { learned_by?: string; last_changed_by?: string; persona_groups?: string[] } | undefined => {
-    if (isNewItem) return undefined;
-    switch (candidateType) {
-      case "fact": return human.facts.find(f => f.id === existingItemId);
-      case "trait": return human.traits.find(t => t.id === existingItemId);
-      case "topic": return human.topics.find(t => t.id === existingItemId);
-      case "person": return human.people.find(p => p.id === existingItemId);
-    }
+  const existingTopic = isNewItem ? undefined : human.topics.find(t => t.id === existingItemId);
+
+  let embedding: number[] | undefined;
+  try {
+    const embeddingService = getEmbeddingService();
+    const category = result.category ?? candidateCategory ?? existingTopic?.category;
+    const text = getTopicEmbeddingText({ name: result.name, category, description: result.description });
+    embedding = await embeddingService.embed(text);
+  } catch (err) {
+    console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${result.name}":`, err);
+  }
+
+  const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
+  const topic: Topic = {
+    id: itemId,
+    name: result.name,
+    description: result.description,
+    sentiment: result.sentiment,
+    category: result.category ?? candidateCategory ?? existingTopic?.category,
+    exposure_current: calculateExposureCurrent(exposureImpact),
+    exposure_desired: result.exposure_desired ?? 0.5,
+    last_updated: now,
+    learned_by: isNewItem ? personaId : existingTopic?.learned_by,
+    last_changed_by: personaId,
+    persona_groups: mergeGroups(personaGroup, isNewItem, existingTopic?.persona_groups),
+    embedding,
   };
-  const existingItem = getExistingItem();
-
-  const mergeGroups = (existing: string[] | undefined): string[] | undefined => {
-    if (!personaGroup) return existing;
-    if (isNewItem) return [personaGroup];
-    const groups = new Set(existing ?? []);
-    groups.add(personaGroup);
-    return Array.from(groups);
+  state.human_topic_upsert(topic);
+
+  const allMessages = state.messages_get(personaId);
+  await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
+
+  console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${result.name}"`);
+}
+
+export async function handlePersonUpdate(response: LLMResponse, state: StateManager): Promise<void> {
+  const result = response.parsed as (PersonUpdateResult & { quotes?: Array<{ text: string; reason: string }> }) | undefined;
+
+  if (!result || Object.keys(result).length === 0) {
+    console.log("[handlePersonUpdate] No changes needed (empty result)");
+    return;
+  }
+
+  const isNewItem = response.request.data.isNewItem as boolean;
+  const existingItemId = response.request.data.existingItemId as string | undefined;
+  const personaId = response.request.data.personaId as string;
+  const personaDisplayName = response.request.data.personaDisplayName as string;
+  const candidateRelationship = response.request.data.candidateRelationship as string | undefined;
+
+  if (!result.name || !result.description || result.sentiment === undefined) {
+    console.error("[handlePersonUpdate] Missing required fields in result");
+    return;
+  }
+
+  const now = new Date().toISOString();
+  const human = state.getHuman();
+
+  const resolveItemId = (): string => {
+    if (isNewItem || !existingItemId) return crypto.randomUUID();
+    return human.people.find(p => p.id === existingItemId) ? existingItemId : crypto.randomUUID();
   };
+  const itemId = resolveItemId();
+
+  const persona = state.persona_getById(personaId);
+  const personaGroup = persona?.group_primary ?? null;
+
+  const existingPerson = isNewItem ? undefined : human.people.find(p => p.id === existingItemId);
 
   let embedding: number[] | undefined;
   try {
     const embeddingService = getEmbeddingService();
-    const text = getItemEmbeddingText({ name: result.name, description: result.description });
+    const relationship = result.relationship ?? candidateRelationship ?? existingPerson?.relationship;
+    const text = getPersonEmbeddingText({ name: result.name, relationship, description: result.description });
     embedding = await embeddingService.embed(text);
   } catch (err) {
-    console.warn(`[handleHumanItemUpdate] Failed to compute embedding for ${candidateType} "${result.name}":`, err);
+    console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${result.name}":`, err);
   }
 
-  switch (candidateType) {
-    case "fact": {
-      const fact: Fact = {
-        id: itemId,
-        name: result.name,
-        description: result.description,
-        sentiment: result.sentiment,
-        validated: ValidationLevel.None,
-        validated_date: now,
-        last_updated: now,
-        learned_by: isNewItem ? personaId : existingItem?.learned_by,
-        last_changed_by: personaId,
-        persona_groups: mergeGroups(existingItem?.persona_groups),
-        embedding,
-      };
-      applyOrValidate(state, "fact", fact, personaDisplayName, isEi, personaGroup);
-      break;
-    }
-    case "trait": {
-      const trait: Trait = {
-        id: itemId,
-        name: result.name,
-        description: result.description,
-        sentiment: result.sentiment,
-        strength: (result as any).strength ?? 0.5,
-        last_updated: now,
-        learned_by: isNewItem ? personaId : existingItem?.learned_by,
-        last_changed_by: personaId,
-        persona_groups: mergeGroups(existingItem?.persona_groups),
-        embedding,
-      };
-      applyOrValidate(state, "trait", trait, personaDisplayName, isEi, personaGroup);
-      break;
-    }
-    case "topic": {
-      const exposureImpact = (result as any).exposure_impact as ExposureImpact | undefined;
-      const itemCategory = response.request.data.itemCategory as string | undefined;
-      const existingTopic = human.topics.find(t => t.id === existingItemId);
-      const topic: Topic = {
-        id: itemId,
-        name: result.name,
-        description: result.description,
-        sentiment: result.sentiment,
-        category: (result as any).category ?? itemCategory ?? existingTopic?.category,
-        exposure_current: calculateExposureCurrent(exposureImpact),
-        exposure_desired: (result as any).exposure_desired ?? 0.5,
-        last_updated: now,
-        learned_by: isNewItem ? personaId : existingItem?.learned_by,
-        last_changed_by: personaId,
-        persona_groups: mergeGroups(existingItem?.persona_groups),
-        embedding,
-      };
-      applyOrValidate(state, "topic", topic, personaDisplayName, isEi, personaGroup);
-      break;
-    }
-    case "person": {
-      const exposureImpact = (result as any).exposure_impact as ExposureImpact | undefined;
-      const person: Person = {
-        id: itemId,
-        name: result.name,
-        description: result.description,
-        sentiment: result.sentiment,
-        relationship: (result as any).relationship ?? "Unknown",
-        exposure_current: calculateExposureCurrent(exposureImpact),
-        exposure_desired: (result as any).exposure_desired ?? 0.5,
-        last_updated: now,
-        learned_by: isNewItem ? personaId : existingItem?.learned_by,
-        last_changed_by: personaId,
-        persona_groups: mergeGroups(existingItem?.persona_groups),
-        embedding,
-      };
-      applyOrValidate(state, "person", person, personaDisplayName, isEi, personaGroup);
-      break;
-    }
-  }
+  const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
+  const person: Person = {
+    id: itemId,
+    name: result.name,
+    description: result.description,
+    sentiment: result.sentiment,
+    relationship: result.relationship ?? candidateRelationship ?? existingPerson?.relationship ?? "Unknown",
+    exposure_current: calculateExposureCurrent(exposureImpact),
+    exposure_desired: result.exposure_desired ?? 0.5,
+    last_updated: now,
+    learned_by: isNewItem ? personaId : existingPerson?.learned_by,
+    last_changed_by: personaId,
+    persona_groups: mergeGroups(personaGroup, isNewItem, existingPerson?.persona_groups),
+    embedding,
+  };
+  state.human_person_upsert(person);
 
   const allMessages = state.messages_get(personaId);
   await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
 
-  console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
+  console.log(`[handlePersonUpdate] ${isNewItem ? "Created" : "Updated"} person "${result.name}"`);
+}
+
+function normalizeText(text: string): string {
+  return text
+    .replace(/[\u201C\u201D]/g, '"')              // curly double quotes
+    .replace(/[\u2018\u2019\u0060\u00B4]/g, "'")  // curly single, backtick, acute accent
+    .replace(/[\u2014\u2013\u2012]/g, '-')         // em-dash, en-dash, figure dash
+    .replace(/\u00A0/g, ' ')                       // non-breaking space
+    .replace(/[\u2000-\u200F]/g, ' ')              // unicode space variants
+    .replace(/\u2026|\.\.\./g, '\u2026');           // normalize both ellipsis forms → unicode ellipsis (1:1)
 }
 
-function normalizeQuotes(text: string): string {
+function stripPunctuation(text: string): string {
+  // Remove characters LLMs commonly mangle, keep spaces and alphanumeric
+  // Strip: punctuation, unicode punctuation variants, curly quotes, dashes, etc.
+  // Keep: letters, digits, spaces
   return text
-    .replace(/[\u201C\u201D]/g, '"')  // Curly double quotes to straight
-    .replace(/[\u2018\u2019]/g, "'"); // Curly single quotes to straight
+    .replace(/[^\w\s]/gu, ' ')   // replace non-word, non-space with space
+    .replace(/\s+/g, ' ')        // collapse multiple spaces
+    .trim()
+    .toLowerCase();
+}
+
+interface WordBoundaryMatch {
+  start: number;
+  end: number;
+  text: string;
+}
+
+function findQuoteByWords(quoteText: string, msgText: string): WordBoundaryMatch | null {
+  const strippedQuote = stripPunctuation(quoteText);
+  const quoteWords = strippedQuote.split(' ').filter(w => w.length > 0);
+
+  if (quoteWords.length < 3) return null;  // Too short to trust — require at least 3 words
+
+  // Build word token list from original message with original positions
+  const wordTokens: Array<{ word: string; start: number; end: number }> = [];
+  const wordRegex = /\S+/g;
+  let match: RegExpExecArray | null;
+  while ((match = wordRegex.exec(msgText)) !== null) {
+    wordTokens.push({
+      word: stripPunctuation(match[0]),
+      start: match.index,
+      end: match.index + match[0].length,
+    });
+  }
+
+  // Find contiguous sequence of words matching the quote words
+  for (let i = 0; i <= wordTokens.length - quoteWords.length; i++) {
+    let allMatch = true;
+    for (let j = 0; j < quoteWords.length; j++) {
+      if (wordTokens[i + j].word !== quoteWords[j]) {
+        allMatch = false;
+        break;
+      }
+    }
+    if (allMatch) {
+      const startToken = wordTokens[i];
+      const endToken = wordTokens[i + quoteWords.length - 1];
+      return {
+        start: startToken.start,
+        end: endToken.end,
+        text: msgText.slice(startToken.start, endToken.end),
+      };
+    }
+  }
+
+  return null;
 }
 
 async function validateAndStoreQuotes(
@@ -250,88 +323,107 @@ async function validateAndStoreQuotes(
     let found = false;
     for (const message of messages) {
       const msgText = getMessageText(message);
-      const normalizedMsg = normalizeQuotes(msgText);
-      const normalizedQuote = normalizeQuotes(candidate.text);
+
+      // Level 1: normalized exact match
+      const normalizedMsg = normalizeText(msgText);
+      const normalizedQuote = normalizeText(candidate.text);
       const start = normalizedMsg.indexOf(normalizedQuote);
+
+      let matchStart: number;
+      let matchEnd: number;
+      let matchText: string;
+      let matchLevel: string;
+
       if (start !== -1) {
-        const end = start + candidate.text.length;
-        
-        // Check for ANY overlapping quote in this message (not just exact match)
-        const existing = state.human_quote_getForMessage(message.id);
-        const overlapping = existing.find(q =>
-          q.start !== null && q.end !== null &&
-          start < q.end && end > q.start  // ranges overlap
-        );
-        
-        if (overlapping) {
-          // Merge: expand to the union of both ranges
-          const mergedStart = Math.min(start, overlapping.start!);
-          const mergedEnd = Math.max(end, overlapping.end!);
-          const mergedText = msgText.slice(mergedStart, mergedEnd);
-          
-          // Merge data_item_ids and persona_groups (deduplicated)
-          const mergedDataItemIds = overlapping.data_item_ids.includes(dataItemId)
-            ? overlapping.data_item_ids
-            : [...overlapping.data_item_ids, dataItemId];
-          const group = personaGroup || "General";
-          const mergedGroups = overlapping.persona_groups.includes(group)
-            ? overlapping.persona_groups
-            : [...overlapping.persona_groups, group];
-          
-          // Only recompute embedding if the text actually changed
-          let embedding = overlapping.embedding;
-          if (mergedText !== overlapping.text) {
-            try {
-              const embeddingService = getEmbeddingService();
-              embedding = await embeddingService.embed(mergedText);
-            } catch (err) {
-              console.warn(`[extraction] Failed to recompute embedding for merged quote: "${mergedText.slice(0, 30)}..."`, err);
-            }
+        matchStart = start;
+        matchEnd = start + candidate.text.length;
+        matchText = candidate.text;
+        matchLevel = "exact";
+      } else {
+        // Level 2: word-boundary fallback
+        const wordMatch = findQuoteByWords(candidate.text, msgText);
+        if (!wordMatch) continue;
+        matchStart = wordMatch.start;
+        matchEnd = wordMatch.end;
+        matchText = wordMatch.text;
+        matchLevel = "word-boundary";
+      }
+
+      const existing = state.human_quote_getForMessage(message.id);
+      const overlapping = existing.find(q =>
+        q.start !== null && q.end !== null &&
+        matchStart < q.end && matchEnd > q.start
+      );
+
+      if (overlapping) {
+        const mergedStart = Math.min(matchStart, overlapping.start!);
+        const mergedEnd = Math.max(matchEnd, overlapping.end!);
+        const mergedText = msgText.slice(mergedStart, mergedEnd);
+
+        const mergedDataItemIds = overlapping.data_item_ids.includes(dataItemId)
+          ? overlapping.data_item_ids
+          : [...overlapping.data_item_ids, dataItemId];
+        const group = personaGroup || "General";
+        const mergedGroups = overlapping.persona_groups.includes(group)
+          ? overlapping.persona_groups
+          : [...overlapping.persona_groups, group];
+
+        let embedding = overlapping.embedding;
+        if (mergedText !== overlapping.text) {
+          try {
+            const embeddingService = getEmbeddingService();
+            embedding = await embeddingService.embed(mergedText);
+          } catch (err) {
+            console.warn(`[extraction] Failed to recompute embedding for merged quote: "${mergedText.slice(0, 30)}..."`, err);
           }
-          
-          state.human_quote_update(overlapping.id, {
-            start: mergedStart,
-            end: mergedEnd,
-            text: mergedText,
-            data_item_ids: mergedDataItemIds,
-            persona_groups: mergedGroups,
-            embedding,
-          });
-          console.log(`[extraction] Merged overlapping quote: "${mergedText.slice(0, 50)}..." (${mergedStart}-${mergedEnd})`);
-          found = true;
-          break;
         }
-        
-        let embedding: number[] | undefined;
-        try {
-          const embeddingService = getEmbeddingService();
-          embedding = await embeddingService.embed(candidate.text);
-        } catch (err) {
-          console.warn(`[extraction] Failed to compute embedding for quote: "${candidate.text.slice(0, 30)}..."`, err);
-        }
-        
-        const quote: Quote = {
-          id: crypto.randomUUID(),
-          message_id: message.id,
-          data_item_ids: [dataItemId],
-          persona_groups: [personaGroup || "General"],
-          text: candidate.text,
-          speaker: message.role === "human" ? "human" : personaName,
-          timestamp: message.timestamp,
-          start: start,
-          end: end,
-          created_at: new Date().toISOString(),
-          created_by: "extraction",
+
+        state.human_quote_update(overlapping.id, {
+          start: mergedStart,
+          end: mergedEnd,
+          text: mergedText,
+          data_item_ids: mergedDataItemIds,
+          persona_groups: mergedGroups,
           embedding,
-        };
-        state.human_quote_add(quote);
-        console.log(`[extraction] Captured quote: "${candidate.text.slice(0, 50)}..."`);
+        });
+        console.log(`[extraction] Merged overlapping quote: "${mergedText.slice(0, 50)}..." (${mergedStart}-${mergedEnd})`);
         found = true;
         break;
       }
+
+      let embedding: number[] | undefined;
+      try {
+        const embeddingService = getEmbeddingService();
+        embedding = await embeddingService.embed(matchText);
+      } catch (err) {
+        console.warn(`[extraction] Failed to compute embedding for quote: "${matchText.slice(0, 30)}..."`, err);
+      }
+
+      const quote: Quote = {
+        id: crypto.randomUUID(),
+        message_id: message.id,
+        data_item_ids: [dataItemId],
+        persona_groups: [personaGroup || "General"],
+        text: matchText,
+        speaker: message.role === "human" ? "human" : personaName,
+        timestamp: message.timestamp,
+        start: matchStart,
+        end: matchEnd,
+        created_at: new Date().toISOString(),
+        created_by: "extraction",
+        embedding,
+      };
+      state.human_quote_add(quote);
+      if (matchLevel === "word-boundary") {
+        console.log(`[extraction] Captured quote (word-boundary match): "${matchText.slice(0, 50)}..."`);
+      } else {
+        console.log(`[extraction] Captured quote: "${matchText.slice(0, 50)}..."`);
+      }
+      found = true;
+      break;
     }
     if (!found) {
-      console.log(`[extraction] Quote not found in messages, skipping: "${candidate.text?.slice(0, 50)}..."`);
+      console.warn(`[extraction] Quote not found in messages (both levels), skipping: "${candidate.text?.slice(0, 50)}..."`);
     }
   }
 }
@@ -346,18 +438,4 @@ function calculateExposureCurrent(impact: ExposureImpact | undefined): number {
   }
 }
 
-function applyOrValidate(
-  state: StateManager,
-  dataType: DataItemType,
-  item: Fact | Trait | Topic | Person,
-  _personaName: string,
-  _isEi: boolean,
-  _personaGroup: string | null
-): void {
-  switch (dataType) {
-    case "fact": state.human_fact_upsert(item as Fact); break;
-    case "trait": state.human_trait_upsert(item as Trait); break;
-    case "topic": state.human_topic_upsert(item as Topic); break;
-    case "person": state.human_person_upsert(item as Person); break;
-  }
-}
+