ei-tui 0.6.4 → 0.6.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ei-tui",
-  "version": "0.6.4",
+  "version": "0.6.6",
   "author": "Flare576",
   "repository": {
     "type": "git",

package/src/core/handlers/dedup.ts

@@ -114,7 +114,9 @@ export async function handleDedupCurate(
   // =========================================================================
   // PHASE 1: Update Quote foreign keys FIRST (before deletions)
   // =========================================================================
-  
+
+  const liveEntityIds = new Set(entityList.map((e: Fact | Topic | Person) => e.id));
+
   for (const removal of decisions.remove) {
     const quotes = state.quotes.filter((q: Quote) =>
       q.data_item_ids.includes(removal.to_be_removed)
@@ -123,6 +125,7 @@ export async function handleDedupCurate(
     for (const quote of quotes) {
       const updatedIds = quote.data_item_ids
         .map((id: string) => id === removal.to_be_removed ? removal.replaced_by : id)
+        .filter((id: string) => liveEntityIds.has(id))         // Drop links to already-merged entities
         .filter((id: string, idx: number, arr: string[]) => arr.indexOf(id) === idx);  // Dedupe
       
       stateManager.human_quote_update(quote.id, {

package/src/core/handlers/human-matching.ts

@@ -8,7 +8,7 @@ import {
 import type { PersonIdentifier } from "../types/data-items.js";
 import type { StateManager } from "../state-manager.js";
 import type { ItemMatchResult, ExposureImpact, TopicUpdateResult, PersonUpdateResult } from "../../prompts/human/types.js";
-import { queueTopicUpdate, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
+import { queueTopicUpdate, queuePersonUpdate, queueTopicValidate, type ExtractionContext } from "../orchestrators/index.js";
 import { getEmbeddingService, getTopicEmbeddingText, getPersonEmbeddingText } from "../embedding-service.js";
 import { calculateExposureCurrent } from "../utils/exposure.js";
 
@@ -28,12 +28,13 @@ export function handleTopicMatch(response: LLMResponse, state: StateManager): vo
   const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
 
   let matched_guid = result.matched_guid;
+  let resolvedTopic: import('../types/data-items.js').Topic | null = null;
   if (matched_guid === "new") {
     matched_guid = null;
   } else if (matched_guid) {
     const human = state.getHuman();
-    const found = human.topics.find(t => t.id === matched_guid);
-    if (!found) {
+    resolvedTopic = human.topics.find(t => t.id === matched_guid) ?? null;
+    if (!resolvedTopic) {
       console.warn(`[handleTopicMatch] matched_guid "${matched_guid}" not found in topics — treating as new`);
       matched_guid = null;
     }
@@ -57,7 +58,7 @@ export function handleTopicMatch(response: LLMResponse, state: StateManager): vo
     extraction_model: response.request.data.extraction_model as string | undefined,
   };
 
-  queueTopicUpdate(result, context, state);
+  queueTopicUpdate(result, context, state, resolvedTopic);
   const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new topic)";
   console.log(`[handleTopicMatch] topic "${context.candidateName}": ${matched}`);
 }
@@ -74,12 +75,13 @@ export function handlePersonMatch(response: LLMResponse, state: StateManager): v
   const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
 
   let matched_guid = result.matched_guid;
+  let resolvedPerson: import('../types/data-items.js').Person | null = null;
   if (matched_guid === "new") {
     matched_guid = null;
   } else if (matched_guid) {
     const human = state.getHuman();
-    const found = human.people.find(p => p.id === matched_guid);
-    if (!found) {
+    resolvedPerson = human.people.find(p => p.id === matched_guid) ?? null;
+    if (!resolvedPerson) {
       console.warn(`[handlePersonMatch] matched_guid "${matched_guid}" not found in people — treating as new`);
       matched_guid = null;
     }
@@ -103,7 +105,7 @@ export function handlePersonMatch(response: LLMResponse, state: StateManager): v
     extraction_model: response.request.data.extraction_model as string | undefined,
   };
 
-  queuePersonUpdate(result, context, state);
+  queuePersonUpdate(result, context, state, resolvedPerson);
   const matched = matched_guid ? `matched GUID "${matched_guid}"` : "no match (new person)";
   console.log(`[handlePersonMatch] person "${context.candidateName}": ${matched}`);
 }
@@ -199,6 +201,11 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
     : state.messages_get(personaId);
   await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
 
+  if (isNewItem && embedding) {
+    const extractionModel = (response.request.data as Record<string, unknown>).extraction_model as string | undefined;
+    await queueTopicValidate(topic, state, extractionModel);
+  }
+
   console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${resolvedName}"`);
 }
 

package/src/core/handlers/index.ts

@@ -88,4 +88,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
   handleRoomJudge,
   handlePersonaPreview,
   [LLMNextStep.HandlePersonIdentifierMigration]: handlePersonIdentifierMigration,
+  [LLMNextStep.HandleTopicValidate]: handleDedupCurate,
 };

package/src/core/llm-client.ts

@@ -51,9 +51,10 @@ function isGuid(str: string): boolean {
 }
 
 function buildResolvedModel(account: ProviderAccount, model: ModelConfig): ResolvedModel {
+  const apiModelId = model.model_id ?? model.name;
   return {
     provider: account.name,
-    model: model.name === "(default)" ? undefined : model.name,
+    model: apiModelId === "(default)" ? undefined : apiModelId,
     config: {
       name: account.name,
       baseURL: account.url,
@@ -164,10 +165,16 @@ function findModelAndAccount(
     const model = account?.models?.find((m) => m.name === modelName);
     return { model, account };
   }
+  // Try matching by model UUID first
   for (const account of accounts) {
     const model = account.models?.find((m) => m.id === spec);
     if (model) return { model, account };
   }
+  // Fall back to matching by account name (bare spec like "EG" or "RnP")
+  const accountByName = accounts.find(
+    (a) => a.name.toLowerCase() === spec.toLowerCase() && a.enabled
+  );
+  if (accountByName) return { model: undefined, account: accountByName };
   return { model: undefined, account: undefined };
 }
 
@@ -265,6 +272,10 @@ export async function callLLMRaw(
     max_tokens: modelConfig?.max_output_tokens ?? DEFAULT_MAX_OUTPUT_TOKENS,
   };
 
+  if (modelConfig?.thinking_budget !== undefined) {
+    requestBody.think = { budget_tokens: modelConfig.thinking_budget };
+  }
+
   if (options.tools && options.tools.length > 0) {
     requestBody.tools = options.tools;
     requestBody.tool_choice = "auto";

package/src/core/orchestrators/human-extraction.ts

@@ -13,6 +13,7 @@ import {
   type ItemMatchResult,
   type ParticipantContext,
 } from "../../prompts/human/index.js";
+import { buildValidatePrompt } from "../../prompts/ceremony/dedup.js";
 import { chunkExtractionContext } from "./extraction-chunker.js";
 import { getEmbeddingService, findTopK, getTopicEmbeddingText } from "../embedding-service.js";
 import { resolveTokenLimit } from "../llm-client.js";
@@ -290,6 +291,13 @@ export function queueDirectTopicUpdate(
 const EMBEDDING_TOP_K = 20;
 const EMBEDDING_MIN_SIMILARITY = 0.3;
 
+/**
+ * Minimum cosine similarity to trigger the post-create validate step.
+ * Higher than EMBEDDING_MIN_SIMILARITY (0.3) because we need near-duplicates,
+ * not just vague thematic overlap.
+ */
+export const VALIDATE_MIN_SIMILARITY = 0.85;
+
 /**
  * Queue a topic match request using embedding-based similarity (topics only).
  */
@@ -380,15 +388,15 @@ export function queueTopicUpdate(
     candidateCategory: string;
     extraction_model?: string;
   },
-  state: StateManager
+  state: StateManager,
+  resolvedItem?: Topic | null
 ): number {
-  const human = state.getHuman();
   const matchedGuid = matchResult.matched_guid;
   const isNewItem = matchedGuid === null;
 
-  let existingItem: Topic | null = null;
-  if (!isNewItem && matchedGuid) {
-    existingItem = human.topics.find(t => t.id === matchedGuid) ?? null;
+  let existingItem: Topic | null = resolvedItem ?? null;
+  if (!isNewItem && matchedGuid && existingItem === null) {
+    existingItem = state.getHuman().topics.find(t => t.id === matchedGuid) ?? null;
   }
 
   const extractionOptions: ExtractionOptions = { extraction_model: context.extraction_model };
@@ -523,15 +531,15 @@ export function queuePersonUpdate(
     candidateIdentifiers?: PersonIdentifier[];
     extraction_model?: string;
   },
-  state: StateManager
+  state: StateManager,
+  resolvedItem?: Person | null
 ): number {
-  const human = state.getHuman();
   const matchedGuid = matchResult.matched_guid;
   const isNewItem = matchedGuid === null;
 
-  let existingItem: Person | null = null;
-  if (!isNewItem && matchedGuid) {
-    existingItem = human.people.find(p => p.id === matchedGuid) ?? null;
+  let existingItem: Person | null = resolvedItem ?? null;
+  if (!isNewItem && matchedGuid && existingItem === null) {
+    existingItem = state.getHuman().people.find(p => p.id === matchedGuid) ?? null;
   }
 
   const candidateIdentifiers = context.candidateIdentifiers ?? [];
@@ -548,7 +556,7 @@ export function queuePersonUpdate(
   }
 
   const userIdentifierTypes = [...new Set(
-    human.people
+    state.getHuman().people
       .flatMap(p => (p.identifiers ?? []).map(i => i.type))
       .filter(Boolean)
   )];
@@ -598,4 +606,55 @@ export function queuePersonUpdate(
   return chunks.length;
 }
 
+export async function queueTopicValidate(
+  newTopic: Topic,
+  state: StateManager,
+  extractionModel?: string
+): Promise<void> {
+  if (!newTopic.embedding || newTopic.embedding.length === 0) {
+    console.log(`[queueTopicValidate] Skipping "${newTopic.name}" — no embedding available`);
+    return;
+  }
+
+  const human = state.getHuman();
+  const candidates = human.topics.filter(t => t.id !== newTopic.id && t.embedding && t.embedding.length > 0);
+
+  if (candidates.length === 0) {
+    console.log(`[queueTopicValidate] No existing topics with embeddings to compare against`);
+    return;
+  }
+
+  const topResult = findTopK(newTopic.embedding, candidates, 1);
+  if (topResult.length === 0 || topResult[0].similarity < VALIDATE_MIN_SIMILARITY) {
+    const best = topResult[0];
+    console.log(`[queueTopicValidate] "${newTopic.name}" is genuinely new (best match: ${best ? `"${best.item.name}" @ ${best.similarity.toFixed(3)}` : "none"})`);
+    return;
+  }
+
+  const existingTopic = topResult[0].item;
+  const similarity = topResult[0].similarity;
+
+  console.log(`[queueTopicValidate] Near-duplicate candidate: "${newTopic.name}" ↔ "${existingTopic.name}" (${similarity.toFixed(3)}) — queuing validate`);
+
+  const prompt = buildValidatePrompt({
+    established: existingTopic,
+    newcomer: newTopic,
+    itemType: "topic",
+    similarity,
+  });
+
+  state.queue_enqueue({
+    type: LLMRequestType.JSON,
+    priority: LLMPriority.Normal,
+    model: extractionModel,
+    system: prompt.system,
+    user: prompt.user,
+    next_step: LLMNextStep.HandleTopicValidate,
+    data: {
+      entity_type: "topic",
+      entity_ids: [existingTopic.id, newTopic.id],
+    },
+  });
+}
+
 

package/src/core/orchestrators/index.ts

@@ -8,6 +8,8 @@ export {
   queueTopicUpdate,
   queuePersonUpdate,
   queueEventSummary,
+  queueTopicValidate,
+  VALIDATE_MIN_SIMILARITY,
   type ExtractionContext,
   type ExtractionOptions,
 } from "./human-extraction.js";

package/src/core/processor.ts

@@ -1130,6 +1130,9 @@ const toolNextSteps = new Set([
                     });
                   }
                 },
+                onUsageUpdate: (modelId, usage) => {
+                  this.stateManager.model_update_usage(modelId, usage);
+                },
               }
             );
 

package/src/core/queue-processor.ts

@@ -37,9 +37,10 @@ export interface QueueProcessorStartOptions {
   onEnqueue?: EnqueueCallback;
   /**
    * Called when a tool executor updates its provider config (e.g. Spotify refresh token rotation).
-   * Injected by Processor to persist the updated config back to storage.
+   * Injected by Processor pointing to stateManager.queue_enqueue.
    */
   onProviderConfigUpdate?: (providerId: string, updates: Record<string, string>) => void;
+  onUsageUpdate?: (modelId: string, usage: { calls: number; tokens_in: number; tokens_out: number }) => void;
 }
 
 export class QueueProcessor {
@@ -52,6 +53,7 @@ export class QueueProcessor {
   private currentTools: ToolDefinition[] | undefined;
   private currentOnEnqueue: EnqueueCallback | undefined;
   private currentOnProviderConfigUpdate: ((providerId: string, updates: Record<string, string>) => void) | undefined;
+  private currentOnUsageUpdate: ((modelId: string, usage: { calls: number; tokens_in: number; tokens_out: number }) => void) | undefined;
 
   getState(): QueueProcessorState {
     return this.state;
@@ -70,6 +72,7 @@ export class QueueProcessor {
     this.currentTools = options?.tools;
     this.currentOnEnqueue = options?.onEnqueue;
     this.currentOnProviderConfigUpdate = options?.onProviderConfigUpdate;
+    this.currentOnUsageUpdate = options?.onUsageUpdate;
     this.abortController = new AbortController();
 
     this.processRequest(request)
@@ -197,7 +200,7 @@ export class QueueProcessor {
         hydratedUser,
         messages,
         request.model,
-        { signal: this.abortController?.signal, tools: openAITools },
+        { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
         this.currentAccounts
       );
 
@@ -304,7 +307,7 @@ export class QueueProcessor {
       hydratedUser,
       messages,
       request.model,
-      { signal: this.abortController?.signal, tools: openAITools },
+      { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
       this.currentAccounts
     );
     if (thinking) {
@@ -496,7 +499,7 @@ export class QueueProcessor {
         reformatUserPrompt,
         messages, // existing tool history — gives full context without duplicating the ask
         request.model,
-        { signal: this.abortController?.signal },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
         this.currentAccounts
       );
 
@@ -553,7 +556,7 @@ export class QueueProcessor {
         reformatUserPrompt,
         [], // no message history needed — schema is already in the system prompt
         request.model,
-        { signal: this.abortController?.signal },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
         this.currentAccounts
       );
 

package/src/core/state-manager.ts

@@ -1064,6 +1064,24 @@ export class StateManager {
     return { success: true, cleared };
   }
 
+  model_update_usage(modelId: string, delta: { calls: number; tokens_in: number; tokens_out: number }): void {
+    const human = this.humanState.get();
+    const accounts = human.settings?.accounts;
+    if (!accounts) return;
+
+    for (const account of accounts) {
+      const model = account.models?.find(m => m.id === modelId);
+      if (model) {
+        model.total_calls = (model.total_calls ?? 0) + delta.calls;
+        model.total_tokens_in = (model.total_tokens_in ?? 0) + delta.tokens_in;
+        model.total_tokens_out = (model.total_tokens_out ?? 0) + delta.tokens_out;
+        model.last_used = new Date().toISOString();
+        this.scheduleSave();
+        return;
+      }
+    }
+  }
+
   async flush(): Promise<void> {
     await this.persistenceState.flush();
   }

package/src/core/types/entities.ts

@@ -44,9 +44,11 @@ export interface BackupConfig {
  */
 export interface ModelConfig {
   id: string;                       // GUID (crypto.randomUUID())
-  name: string;                     // Model identifier, e.g. "claude-haiku-4-5", "(default)"
+  name: string;                     // Display name shown in UI, e.g. "Gemma4 (thinking)", "(default)"
+  model_id?: string;                // Actual model identifier sent to API — falls back to name if absent
   token_limit?: number;             // Input token limit (user sets effective limit)
   max_output_tokens?: number;       // Output token limit (API-enforced)
+  thinking_budget?: number;         // Thinking token budget: 0 = disabled, N = enable with N tokens, undefined = don't send
   total_calls?: number;             // Usage counter
   total_tokens_in?: number;         // Usage counter
   total_tokens_out?: number;        // Usage counter

package/src/core/types/enums.ts

@@ -51,6 +51,7 @@ export enum LLMNextStep {
   HandleRoomJudge = "handleRoomJudge",
   HandlePersonaPreview = "handlePersonaPreview",
   HandlePersonIdentifierMigration = "handlePersonIdentifierMigration",
+  HandleTopicValidate = "handleTopicValidate",
 }
 
 export enum ProviderType {

package/src/prompts/ceremony/dedup.ts

@@ -1,4 +1,4 @@
-import type { DedupPromptData } from "./types.js";
+import type { DedupPromptData, ValidatePromptData } from "./types.js";
 
 // =============================================================================
 // DEDUP CURATOR — Merge duplicate entities with data preservation
@@ -140,6 +140,94 @@ ${schemaReminder}`;
   return { system, user };
 }
 
+// =============================================================================
+// VALIDATE — Binary merge decision for a newly created record
+// =============================================================================
+
+export function buildValidatePrompt(data: ValidatePromptData): { system: string; user: string } {
+  const typeLabel = data.itemType.charAt(0).toUpperCase() + data.itemType.slice(1);
+  const pct = Math.round(data.similarity * 100);
+
+  const system = `# Your Task
+
+A new ${typeLabel} record was just created from a real conversation. The moment it landed in the system, we checked it against everything already stored and found one record with a similarity score of ${pct}% — high enough that they might be the same thing under different words.
+
+You are the last gate before a duplicate takes root.
+
+**Established record**: Has been in the system. Learned from prior conversations.
+**Newcomer**: Just synthesized from the most recent conversation. Description is current-state, not a log.
+
+## What You're Deciding
+
+Are these the same thing — the same interest, concern, goal, or moment — described twice? Or are they genuinely distinct, and both deserve to exist?
+
+Similarity of meaning is not the same as identity. "Concern about job security" and "Fear of career stagnation" share semantic space. They are not the same record.
+
+Ask yourself: *If a persona referenced the established record in conversation, would the newcomer feel like a repeat? Or would it feel like something different being said?*
+
+If they are the same thing: **merge**. Preserve every unique detail from both. The newcomer's description is synthesized and current — weight it, but don't discard what the established record learned first.
+
+If they are distinct: **keep both**. Return them both in \`update\` unchanged. Leave \`remove\` and \`add\` empty.
+
+## Output Format
+
+\`\`\`json
+{
+  "update": [ /* one or both records — include ALL fields from whichever survive */ ],
+  "remove": [ /* { "to_be_removed": "uuid", "replaced_by": "uuid" } — only if merging */ ],
+  "add": []
+}
+\`\`\`
+
+Rules:
+- \`add\` is always empty here. We are not creating new records from this decision.
+- If merging: the merged record goes in \`update\`, the absorbed record goes in \`remove\`.
+- If keeping both: return both in \`update\` exactly as received. Do not modify either.
+- Descriptions must stay concise — under 300 characters, never over 500. Synthesize; don't concatenate.
+- When merging numeric fields: take the HIGHER value for \`exposure_current\`, \`exposure_desired\`, \`strength\`, \`confidence\`. Average \`sentiment\`.
+- Do NOT invent information. Only what exists in these two records.
+
+Return raw JSON only. No markdown fencing, no commentary.`;
+
+  const payload = JSON.stringify({
+    established: stripEmbedding(data.established),
+    newcomer: stripEmbedding(data.newcomer),
+    item_type: data.itemType,
+    similarity_score: data.similarity,
+  }, null, 2);
+
+  const schemaReminder = `**Return JSON:**
+\n\`\`\`json
+{
+  "update": [
+    {
+      "id": "uuid-of-surviving-record",
+      "type": "${data.itemType}",
+      "name": "canonical name",
+      "description": "merged or unchanged description"
+    }
+  ],
+  "remove": [
+    {
+      "to_be_removed": "uuid-of-absorbed-record",
+      "replaced_by": "uuid-of-surviving-record"
+    }
+  ],
+  "add": []
+}
+\`\`\`
+
+If keeping both, return both in \`update\` unchanged with empty \`remove\` and \`add\`.`;
+
+  const user = `${payload}
+
+---
+
+${schemaReminder}`;
+
+  return { system, user };
+}
+
 // =============================================================================
 // Helpers
 // =============================================================================

package/src/prompts/ceremony/index.ts

@@ -2,7 +2,7 @@ export { buildPersonaExpirePrompt } from "./expire.js";
 export { buildPersonaExplorePrompt } from "./explore.js";
 export { buildDescriptionCheckPrompt } from "./description-check.js";
 export { buildRewriteScanPrompt, buildRewritePrompt } from "./rewrite.js";
-export { buildDedupPrompt } from "./dedup.js";
+export { buildDedupPrompt, buildValidatePrompt } from "./dedup.js";
 export { buildUserDedupPrompt } from "./user-dedup.js";
 export { buildPersonMigrationPrompt, type PersonMigrationPromptData } from "./person-migration.js";
 export type {
@@ -20,4 +20,5 @@ export type {
   RewriteResult,
   DedupPromptData,
   DedupResult,
+  ValidatePromptData,
 } from "./types.js";

package/src/prompts/ceremony/types.ts

@@ -107,6 +107,14 @@ export interface DedupPromptData {
   similarityRange: { min: number; max: number };  // e.g., { min: 0.90, max: 0.98 }
 }
 
+/** Input: exactly 2 records — one established, one just created — for binary merge decision. */
+export interface ValidatePromptData {
+  established: DataItemBase;
+  newcomer: DataItemBase;
+  itemType: RewriteItemType;
+  similarity: number;
+}
+
 /** Output: merge decisions (update/remove/add). */
 export interface DedupResult {
   update: Array<{

package/tui/src/commands/editor.tsx

@@ -26,16 +26,19 @@ export const editorCommand: Command = {
     });
     
     if (result.aborted) {
+      ctx.setInputText("");
       ctx.showNotification("Editor cancelled", "info");
       return;
     }
     
     if (!result.success) {
+      ctx.setInputText("");
       ctx.showNotification("Editor failed to open", "error");
       return;
     }
     
     if (result.content === null) {
+      ctx.setInputText("");
       ctx.showNotification("No changes made", "info");
       return;
     }

package/tui/src/commands/help.tsx

@@ -7,6 +7,6 @@ export const helpCommand: Command = {
   description: "Show help screen",
   usage: "/help or /h",
   execute: async (_args, ctx) => {
-    ctx.showOverlay((_hideOverlay, _hideForEditor) => <HelpOverlay onDismiss={_hideOverlay} />, ctx.renderer);
+    ctx.showOverlay((_hideOverlay, _hideForEditor) => <HelpOverlay onDismiss={_hideOverlay} renderer={ctx.renderer} />, ctx.renderer);
   },
 };