ei-tui 1.0.0 → 1.1.0

package/src/core/handlers/human-matching.ts

@@ -85,6 +85,10 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
   const primaryId = personaIds[0] ?? personaId;
 
   const now = new Date().toISOString();
+  const { messages_analyze } = resolveMessageWindow(response, state);
+  const earliestMessageTimestamp = messages_analyze.length > 0
+    ? messages_analyze.reduce((a, b) => a.timestamp < b.timestamp ? a : b).timestamp
+    : now;
   const human = state.getHuman();
 
   const resolveItemId = (): string => {
@@ -144,7 +148,7 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
     exposure_current: calculateExposureCurrent(exposureImpact, existingTopic?.exposure_current ?? 0),
     exposure_desired: result.exposure_desired ?? 0.5,
     last_updated: now,
-    learned_on: isNewItem ? now : existingTopic?.learned_on,
+    learned_on: isNewItem ? earliestMessageTimestamp : existingTopic?.learned_on,
     last_mentioned: now,
     learned_by: isNewItem ? primaryId : existingTopic?.learned_by,
     last_changed_by: primaryId,
@@ -168,6 +172,26 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
   console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${resolvedName}"`);
 }
 
+function ensureEiPersonaHasNickname(identifiers: PersonIdentifier[], state: StateManager): PersonIdentifier[] {
+  const eiPersonaId = identifiers.find(i => i.type === 'Ei Persona')?.value;
+  if (!eiPersonaId) return identifiers;
+
+  const persona = state.persona_getById(eiPersonaId);
+  if (!persona) return identifiers;
+
+  const hasNickname = identifiers.some(i => i.type === 'Nickname' && i.value === persona.display_name);
+  if (hasNickname) return identifiers;
+
+  const withoutPrimary = identifiers.map(i =>
+    i.type === 'Ei Persona' ? { ...i, is_primary: undefined } : i
+  ).map(({ is_primary, ...rest }) => is_primary ? { ...rest, is_primary } : rest);
+
+  return [
+    { type: 'Nickname', value: persona.display_name, is_primary: true as const },
+    ...withoutPrimary.map(i => i.type === 'Ei Persona' ? { type: i.type, value: i.value } : i),
+  ];
+}
+
 export async function handlePersonUpdate(response: LLMResponse, state: StateManager): Promise<void> {
   const result = response.parsed as (PersonUpdateResult & {
     identifiers?: PersonIdentifier[];
@@ -194,6 +218,10 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
   const primaryId = personaIds[0] ?? personaId;
 
   const now = new Date().toISOString();
+  const { messages_analyze } = resolveMessageWindow(response, state);
+  const earliestMessageTimestamp = messages_analyze.length > 0
+    ? messages_analyze.reduce((a, b) => a.timestamp < b.timestamp ? a : b).timestamp
+    : now;
   const human = state.getHuman();
 
   const resolveItemId = (): string => {
@@ -264,7 +292,7 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
         deduped.push(id);
       }
     }
-    resolvedIdentifiers = deduped;
+    resolvedIdentifiers = ensureEiPersonaHasNickname(deduped, state);
   } else {
     const base = [...(existingPerson?.identifiers ?? [])];
     const sanitizedToAdd = sanitizeEiPersonaIdentifiers(
@@ -279,12 +307,16 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
         base.push({ type: id.type, value: id.value, ...(id.is_primary ? { is_primary: id.is_primary } : {}) });
       }
     }
-    resolvedIdentifiers = base;
+    resolvedIdentifiers = ensureEiPersonaHasNickname(base, state);
   }
 
+  const personName = resolvedIdentifiers.find(i => i.is_primary && i.type !== 'Ei Persona')?.value
+    ?? resolvedIdentifiers.find(i => i.type !== 'Ei Persona')?.value
+    ?? candidateName;
+
   const person: Person = {
     id: itemId,
-    name: candidateName,
+    name: personName,
     description: resolvedDescription,
     sentiment: resolvedSentiment,
     relationship: result.relationship ?? candidateRelationship ?? existingPerson?.relationship ?? "Unknown",
@@ -293,7 +325,7 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
     identifiers: resolvedIdentifiers,
     validated_date: isNewItem ? '' : (existingPerson?.validated_date ?? ''),
     last_updated: now,
-    learned_on: isNewItem ? now : existingPerson?.learned_on,
+    learned_on: isNewItem ? earliestMessageTimestamp : existingPerson?.learned_on,
     last_mentioned: now,
     learned_by: isNewItem ? primaryId : existingPerson?.learned_by,
     last_changed_by: primaryId,
@@ -323,14 +355,13 @@ function normalizeText(text: string): string {
     .replace(/[\u2018\u2019\u0060\u00B4]/g, "'")  // curly single, backtick, acute accent
     .replace(/[\u2014\u2013\u2012]/g, '-')         // em-dash, en-dash, figure dash
     .replace(/\u00A0/g, ' ')                       // non-breaking space
-    .replace(/[\u2000-\u200F]/g, ' ');              // unicode space variants
+    .replace(/[\u2000-\u200F]/g, ' ')              // unicode space variants
+    .replace(/[*_`~]/g, '');                       // Markdown emphasis/code chars
 }
 
 function stripPunctuation(text: string): string {
-  // Remove characters LLMs commonly mangle, keep spaces and alphanumeric
-  // Strip: punctuation, unicode punctuation variants, curly quotes, dashes, etc.
-  // Keep: letters, digits, spaces
   return text
+    .replace(/[*_`~]/g, ' ')     // Markdown chars (kept by \w, must strip explicitly)
     .replace(/[^\w\s]/gu, ' ')   // replace non-word, non-space with space
     .replace(/\s+/g, ' ')        // collapse multiple spaces
     .trim()
@@ -406,6 +437,10 @@ async function validateAndStoreQuotes(
   if (!candidates || candidates.length === 0) return;
   
   for (const candidate of candidates) {
+    if (!candidate.text) {
+      console.warn('[extraction] Skipping quote candidate with missing text field');
+      continue;
+    }
     let found = false;
     for (const message of messages) {
       const msgText = getMessageText(message);
@@ -511,7 +546,7 @@ async function validateAndStoreQuotes(
       break;
     }
     if (!found) {
-      console.warn(`[extraction] Quote not found in messages (both levels), skipping: "${candidate.text?.slice(0, 50)}..."`);
+      console.warn(`[extraction] Quote not found in messages (both levels), skipping: "${candidate.text}"`);
     }
   }
 }

package/src/core/llm-client.ts

@@ -2,6 +2,36 @@ import type { ChatMessage, ProviderAccount, ModelConfig } from "./types.js";
 const DEFAULT_TOKEN_LIMIT = 8192;
 const DEFAULT_MAX_OUTPUT_TOKENS = 8000;
 
+// Lazy verbose network dump — only active when EI_DEBUG_NETWORK_VERBOSE=1.
+// Uses dynamic import so the web bundle never pulls in node:fs.
+async function writeNetworkDump(
+  callNumber: number,
+  nextStep: string,
+  meta: { model: string; provider: string; latency_ms: number; status_code: number; tokens_in: number; tokens_out: number },
+  request: unknown,
+  response: unknown
+): Promise<void> {
+  const dataPath = (typeof process !== "undefined" && process.env?.EI_DATA_PATH) ||
+    (typeof Bun !== "undefined" && (Bun as Record<string, unknown>).env && ((Bun as { env: Record<string, string> }).env.EI_DATA_PATH));
+  if (!dataPath) return;
+
+  try {
+    const { mkdirSync, writeFileSync } = await import("node:fs");
+    const { join } = await import("node:path");
+    const logsDir = join(dataPath as string, "logs");
+    mkdirSync(logsDir, { recursive: true });
+
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const safeName = nextStep.replace(/[^a-zA-Z0-9_-]/g, "_");
+    const filename = join(logsDir, `${timestamp}_call${callNumber}_${safeName}.json`);
+
+    const payload = JSON.stringify({ meta, request, response }, null, 2);
+    writeFileSync(filename, payload);
+  } catch {
+    // Silent — verbose dump failures must never crash the main path
+  }
+}
+
 export interface ProviderConfig {
   baseURL: string;
   apiKey: string;
@@ -22,6 +52,8 @@ export interface LLMCallOptions {
   tools?: Record<string, unknown>[];
   /** Fire-and-forget callback invoked after a successful response to increment usage counters. */
   onUsageUpdate?: (modelId: string, usage: { calls: number; tokens_in: number; tokens_out: number }) => void;
+  /** Queue step name passed through to EI_DEBUG_NETWORK_VERBOSE file dumps. */
+  nextStep?: string;
 }
 
 export interface LLMRawResponse {
@@ -212,7 +244,7 @@ function logTokenLimit(model: string, source: string, tokens: number): void {
   if (source === "default") {
     console.warn(`[TokenLimit] Unknown model "${model}" — using conservative default (${DEFAULT_TOKEN_LIMIT})`);
   } else {
-    console.log(`[TokenLimit] ${model}: ${source} → ${tokens} tokens (extraction budget: ${budget})`);
+    console.debug(`[TokenLimit] ${model}: ${source} → ${tokens} tokens (extraction budget: ${budget})`);
   }
 }
 
@@ -226,7 +258,7 @@ export async function callLLMRaw(
 ): Promise<LLMRawResponse> {
   llmCallCount++;
   
-  const { signal, temperature = 0.7, onUsageUpdate } = options;
+  const { signal, temperature = 0.7, onUsageUpdate, nextStep = "unknown" } = options;
   
   if (signal?.aborted) {
     throw new Error("LLM call aborted");
@@ -251,7 +283,9 @@ export async function callLLMRaw(
   
   const totalChars = finalMessages.reduce((sum, m) => sum + (m.content?.length ?? 0), 0);
   const estimatedTokens = Math.ceil(totalChars / 4);
-  console.log(`[LLM] Call #${llmCallCount} - ~${estimatedTokens} tokens (${totalChars} chars)`);
+  const modelLabel = model ?? "default";
+  console.log(`[LLM] Call #${llmCallCount} — ${config.name}:${modelLabel}, ~${estimatedTokens} tokens est.`);
+  const _llmCallStart = Date.now();
   
   const normalizedBaseURL = config.baseURL.replace(/\/+$/, "");
   
@@ -275,14 +309,15 @@ export async function callLLMRaw(
 
   if (modelConfig?.thinking_budget !== undefined) {
     if (modelConfig.thinking_budget === 0) {
-      // Universal kill switch — works on Ollama, LM Studio, and all OpenAI-compat providers.
-      requestBody.reasoning_effort = "none";
+      // Universal kill switch across all known providers. Non-conflicting — each reads
+      // whichever field it understands and ignores the rest.
+      requestBody.reasoning_effort = "none";  // Ollama, OpenAI-compat
+      requestBody.enable_thinking = false;    // Rapid-MLX
     } else {
-      // Pass both signals: providers that honor the token budget get it (Qwen3 via Ollama,
-      // Anthropic), providers that reduce thinking to on/off use reasoning_effort as the
-      // on-signal (Gemma4 via Ollama/LM Studio). Non-conflicting — each provider reads
-      // whichever field it understands.
+      // Pass all on-signals: providers that honor the token budget get it (Qwen3, Anthropic),
+      // providers that reduce thinking to on/off use reasoning_effort or enable_thinking.
       requestBody.reasoning_effort = "high";
+      requestBody.enable_thinking = true;
       requestBody.think = { budget_tokens: modelConfig.thinking_budget };
     }
   }
@@ -306,9 +341,24 @@ export async function callLLMRaw(
   
   const data = await response.json();
 
+  const _llmLatency = Date.now() - _llmCallStart;
+  const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
+  const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
+  console.log(`[LLM] Response #${llmCallCount} — ${response.status} ${_llmLatency}ms | in: ${tokensIn} out: ${tokensOut}`);
+
+  const isVerbose = (typeof process !== "undefined" && process.env?.EI_DEBUG_NETWORK_VERBOSE === "1") ||
+    (typeof Bun !== "undefined" && (Bun as { env: Record<string, string> }).env?.EI_DEBUG_NETWORK_VERBOSE === "1");
+  if (isVerbose) {
+    void writeNetworkDump(
+      llmCallCount,
+      nextStep,
+      { model: modelLabel, provider: config.name, latency_ms: _llmLatency, status_code: response.status, tokens_in: tokensIn, tokens_out: tokensOut },
+      requestBody,
+      data
+    );
+  }
+
   if (onUsageUpdate && modelConfig) {
-    const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
-    const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
     onUsageUpdate(modelConfig.id, { calls: 1, tokens_in: tokensIn, tokens_out: tokensOut });
   }
 
@@ -400,6 +450,7 @@ const JSON_REPAIR_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [
   { pattern: /:\s*(\d{4}-\d{2}-\d{2}T[^"}\],\n]+)/g, replacement: ': "$1"' },
   { pattern: /:\s*0([1-9][0-9]*)([,\s\n\r\]}])/g, replacement: ": 0.$1$2" },
   { pattern: /,(\s*[\]}])/g, replacement: "$1" },
+  { pattern: /"(\s*\n[ \t]+"[a-zA-Z_][a-zA-Z0-9_]*"\s*:)/g, replacement: '",$1' },
 ];
 
 export function repairJSON(jsonStr: string): string {
@@ -479,6 +530,41 @@ export function rescueGemmaToolCalls(content: string): unknown[] {
   return rescued;
 }
 
+function findOutermostObject(str: string): string | null {
+  const start = str.indexOf('{');
+  if (start === -1) return null;
+
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let i = start; i < str.length; i++) {
+    const ch = str[i];
+
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (ch === '\\' && inString) {
+      escaped = true;
+      continue;
+    }
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+
+    if (ch === '{') depth++;
+    else if (ch === '}') {
+      depth--;
+      if (depth === 0) return str.slice(start, i + 1);
+    }
+  }
+
+  return null;
+}
+
 export function parseJSONResponse(content: string): unknown {
   const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
   const jsonStr = jsonMatch ? jsonMatch[1].trim() : content.trim();
@@ -491,10 +577,10 @@ export function parseJSONResponse(content: string): unknown {
       return JSON.parse(repaired);
     } catch {
       // Last resort: extract the outermost {...} block from mixed prose/JSON content.
-      // Handles 'thinking prose...\n{...json...}' responses from extended-thinking models.
-      const outerMatch = jsonStr.match(/\{[\s\S]*\}/);
-      if (outerMatch) {
-        const extracted = outerMatch[0];
+      // Bracket-depth scan (not greedy regex) stops at the first valid close so extra
+      // trailing braces from models like Gemma are excluded from the extracted slice.
+      const extracted = findOutermostObject(jsonStr);
+      if (extracted) {
         try {
           return JSON.parse(extracted);
         } catch {

package/src/core/orchestrators/human-extraction.ts

@@ -311,6 +311,34 @@ const EMBEDDING_MIN_SIMILARITY = 0.3;
  */
 export const VALIDATE_MIN_SIMILARITY = 0.92;
 
+/**
+ * Returns the best cosine similarity between a topic candidate and any existing
+ * topic in state. Used by queueTopicValidate to detect near-duplicates after
+ * a new topic is created.
+ * Returns 0 if no topics exist or embedding fails.
+ */
+export async function getBestTopicSimilarity(
+  candidate: TopicScanCandidate,
+  state: StateManager
+): Promise<number> {
+  const human = state.getHuman();
+  const topicsWithEmbeddings = human.topics.filter(t => t.embedding && t.embedding.length > 0);
+  if (topicsWithEmbeddings.length === 0) return 0;
+  try {
+    const embeddingService = getEmbeddingService();
+    const candidateText = getTopicEmbeddingText({
+      name: candidate.name,
+      category: candidate.category,
+      description: candidate.description,
+    });
+    const candidateVector = await embeddingService.embed(candidateText);
+    const topK = findTopK(candidateVector, topicsWithEmbeddings, 1);
+    return topK.length > 0 ? topK[0].similarity : 0;
+  } catch {
+    return 0;
+  }
+}
+
 /**
  * Queue a topic match request using embedding-based similarity (topics only).
  */

package/src/core/orchestrators/index.ts

@@ -12,6 +12,7 @@ export {
   queueTargetedPersonUpdate,
   queueTargetedTopicUpdate,
   VALIDATE_MIN_SIMILARITY,
+  getBestTopicSimilarity,
   type ExtractionContext,
   type ExtractionOptions,
 } from "./human-extraction.js";

package/src/core/processor.ts

@@ -36,8 +36,10 @@ import { handlers } from "./handlers/index.js";
 import { normalizeRoomMessages, getMessageContent } from "./handlers/utils.js";
 import { sanitizeEiPersonaIdentifiers } from "./utils/identifier-utils.js";
 import { ContextStatus as ContextStatusEnum, RoomMode } from "./types.js";
-import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
-import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
+import { registerFindMemoryExecutor, registerFetchMemoryExecutor, registerFetchMessageExecutor, registerFileReadExecutor, SYSTEM_TOOLS } from "./tools/index.js";
+import { createFindMemoryExecutor } from "./tools/builtin/find-memory.js";
+import { createFetchMemoryExecutor } from "./tools/builtin/fetch-memory.js";
+import { createFetchMessageExecutor } from "./tools/builtin/fetch-message.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
 import { EMMETT_PERSONA_DEFINITION } from "../templates/emmett.js";
 import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueReflectionDrain, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction, queueTargetedPersonUpdate, queueTargetedTopicUpdate } from "./orchestrators/index.js";
@@ -240,7 +242,15 @@ export class Processor {
     this.seedBuiltinFacts();
     this.migrateLearnedOn();
     this.seedSettings();
-    registerReadMemoryExecutor(createReadMemoryExecutor(this.searchHumanData.bind(this), this.getPersonaList.bind(this)));
+    registerFindMemoryExecutor(createFindMemoryExecutor(this.searchHumanData.bind(this), this.getPersonaList.bind(this), this.stateManager.getHuman.bind(this.stateManager)));
+    registerFetchMemoryExecutor(createFetchMemoryExecutor(this.stateManager.getHuman.bind(this.stateManager)));
+    registerFetchMessageExecutor(createFetchMessageExecutor(
+      this.stateManager.persona_getAll.bind(this.stateManager),
+      this.stateManager.messages_get.bind(this.stateManager),
+      this.stateManager.getRoomList.bind(this.stateManager),
+      this.stateManager.getRoomMessages.bind(this.stateManager),
+      (roomId: string) => this.stateManager.getRoom(roomId)?.display_name ?? null
+    ));
     if (this.isTUI) {
       await registerFileReadExecutor();
     }
@@ -294,13 +304,12 @@ export class Processor {
       }
       return;
     }
-    const readMemoryTool = this.stateManager.tools_getByName("read_memory");
     const emmettEntity: PersonaEntity = {
       ...EMMETT_PERSONA_DEFINITION,
       id: "emmet",
       display_name: "Emmett",
       last_updated: new Date().toISOString(),
-      tools: readMemoryTool ? [readMemoryTool.id] : [],
+      tools: [],
     };
     this.stateManager.persona_add(emmettEntity);
     this.interface.onPersonaAdded?.();
@@ -334,6 +343,11 @@ export class Processor {
   private bootstrapTools(): void {
     const now = new Date().toISOString();
 
+    for (const name of ["find_memory", "fetch_memory", "fetch_message", "read_memory"]) {
+      const tool = this.stateManager.tools_getByName(name);
+      if (tool) this.stateManager.tools_remove(tool.id);
+    }
+
     // --- Ei built-in provider ---
     if (!this.stateManager.tools_getProviderById("ei")) {
       const eiProvider: ToolProvider = {
@@ -349,35 +363,6 @@ export class Processor {
       this.stateManager.tools_addProvider(eiProvider);
     }
 
-    // read_memory tool
-    this.stateManager.tools_upsertBuiltin({
-        id: crypto.randomUUID(),
-        provider_id: "ei",
-        name: "read_memory",
-        display_name: "Read Memory",
-        description:
-          "Search Ei's persistent knowledge base — facts, topics, people, and quotes learned across ALL conversations over time, not just this one. Use this when you need context about the user, their life, relationships, or interests that may not be visible in the current exchange. Use `recent: true` to retrieve what's been discussed recently.",
-        input_schema: {
-          type: "object",
-          properties: {
-            query: { type: "string", description: "What to search for — a person, topic, fact, or anything Ei has learned about the user" },
-            types: {
-              type: "array",
-              items: { type: "string", enum: ["fact", "topic", "person", "quote"] },
-              description: "Limit search to specific memory types (default: all types)",
-            },
-            limit: { type: "number", description: "Max results to return (default: 10, max: 20)" },
-            recent: { type: "boolean", description: "If true, return recently-mentioned results sorted by last_mentioned date instead of relevance. Combine with a query to filter recent results by topic." },
-          },
-          required: [],
-        },
-        runtime: "any",
-        builtin: true,
-        enabled: true,
-        created_at: now,
-        max_calls_per_interaction: 6,  // Dedup needs to verify relationships before irreversible merges. Typical cluster (3-8 items) requires: parent concept lookup + 2 relationship verifications + context validation. Still under HARD_TOOL_CALL_LIMIT (10).
-    });
-
     // file_read tool (TUI only)
     this.stateManager.tools_upsertBuiltin({
         id: crypto.randomUUID(),
@@ -817,6 +802,20 @@ export class Processor {
         max_calls_per_interaction: 1,
         created_at: now,
     });
+
+    // --- Reconcile pass: prune stale tool references from persona tool lists ---
+    // Build manifest of all tool IDs currently in state (everything seeded above).
+    const manifestIds = new Set(this.stateManager.tools_getAll().map(t => t.id));
+
+    for (const persona of this.stateManager.persona_getAll()) {
+      if (!persona.tools?.length) continue;
+      const pruned = persona.tools.filter(id => manifestIds.has(id));
+      if (pruned.length !== persona.tools.length) {
+        const removed = persona.tools.length - pruned.length;
+        this.stateManager.persona_update(persona.id, { tools: pruned });
+        console.log(`[Processor] Pruned ${removed} stale tool reference(s) from persona "${persona.display_name}"`);
+      }
+    }
   }
 
   /**
@@ -1093,10 +1092,10 @@ const toolNextSteps = new Set([
               personaId ??
               (request.next_step === LLMNextStep.HandleEiHeartbeat ? "ei" : undefined);
             
-            // Dedup operates on Human data, not persona data - provide read_memory directly.
+            // Dedup operates on Human data, not persona data — provide find_memory from SYSTEM_TOOLS directly.
             // Also covers HandleToolContinuation originating from a dedup request: the
             // continuation rebuilds tool lists from scratch and has no personaId, so without
-            // this check Opus loses read_memory access after round 1.
+            // this check Opus loses find_memory access after round 1.
             const isDedupRequest =
               request.next_step === LLMNextStep.HandleDedupCurate ||
               (request.next_step === LLMNextStep.HandleToolContinuation &&
@@ -1104,12 +1103,9 @@ const toolNextSteps = new Set([
 
             let tools: ToolDefinition[] = [];
             if (isDedupRequest) {
-              const readMemory = this.stateManager.tools_getByName("read_memory");
-              if (readMemory?.enabled) {
-                tools = [readMemory];
-              }
+              tools = SYSTEM_TOOLS.filter(t => t.name === "find_memory");
             } else if (toolNextSteps.has(request.next_step) && toolPersonaId) {
-              tools = this.stateManager.tools_getForPersona(toolPersonaId, this.isTUI);
+              tools = [...SYSTEM_TOOLS, ...this.stateManager.tools_getForPersona(toolPersonaId, this.isTUI)];
             }
 
             // Auto-inject each handler's dedicated submit tool — infrastructure, not user-visible.

package/src/core/prompt-context-builder.ts

@@ -274,6 +274,7 @@ export async function buildResponsePromptData(
 
   const alwaysMessages = sm.messages_getAlways(persona.id);
   const temporalAnchors = alwaysMessages.map(m => ({
+    id: m.id,
     role: m.role === "human" ? "human" as const : "system" as const,
     content: m.content,
     silence_reason: m.silence_reason,

package/src/core/queue-processor.ts

@@ -200,7 +200,7 @@ export class QueueProcessor {
         hydratedUser,
         messages,
         request.model,
-        { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.data.originalNextStep ?? request.next_step}+tool_continuation` },
         this.currentAccounts
       );
 
@@ -219,7 +219,7 @@ export class QueueProcessor {
             if (!args.should_respond && args.content) {
               args.should_respond = true;
             }
-            console.log(`[QueueProcessor] submit tool "${submitCall.name}" called — returning arguments as parsed response`);
+      console.debug(`[QueueProcessor] submit tool "${submitCall.name}" called — returning arguments as parsed response`);
             return {
               request,
               success: true,
@@ -297,9 +297,9 @@ export class QueueProcessor {
     const isHeartbeat = request.next_step === LLMNextStep.HandleHeartbeatCheck || request.next_step === LLMNextStep.HandleEiHeartbeat;
     if (isHeartbeat) {
       const personaName = request.data.personaDisplayName as string | undefined ?? 'Ei';
-      console.log(`[${personaName} Heartbeat] LLM call - tools offered: ${openAITools.length} (${activeTools.map(t => t.name).join(', ') || 'none'})`);
+      console.debug(`[${personaName} Heartbeat] LLM call - tools offered: ${openAITools.length} (${activeTools.map(t => t.name).join(', ') || 'none'})`);
     } else {
-      console.log(`[QueueProcessor] LLM call for ${request.next_step}, tools=${openAITools.length}`);
+      console.debug(`[QueueProcessor] LLM call for ${request.next_step}, tools=${openAITools.length}`);
     }
 
     const { content, finishReason, rawToolCalls, assistantMessage, thinking } = await callLLMRaw(
@@ -307,18 +307,18 @@ export class QueueProcessor {
       hydratedUser,
       messages,
       request.model,
-      { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
+      { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate, nextStep: request.next_step },
       this.currentAccounts
     );
     if (thinking) {
-      console.log(`[QueueProcessor] Extended thinking on ${request.next_step} (${thinking.length} chars) — TODO(#13): stream to TUI`);
+      console.debug(`[QueueProcessor] Extended thinking on ${request.next_step} (${thinking.length} chars) — TODO(#13): stream to TUI`);
     }
 
     // =========================================================================
     // Tool call path: execute tools, enqueue HandleToolContinuation, done.
     // =========================================================================
     if (finishReason === "tool_calls" && rawToolCalls?.length) {
-      console.log(`[QueueProcessor] finish_reason=tool_calls — executing tools, will enqueue HandleToolContinuation`);
+      console.debug(`[QueueProcessor] finish_reason=tool_calls — executing tools, will enqueue HandleToolContinuation`);
 
       const toolCalls = parseToolCalls(rawToolCalls);
       if (toolCalls.length === 0) {
@@ -364,7 +364,7 @@ export class QueueProcessor {
         });
       }
 
-      console.log(`[QueueProcessor] Tool execution complete: ${results.length} result(s). Enqueueing HandleToolContinuation.`);
+      console.debug(`[QueueProcessor] Tool execution complete: ${results.length} result(s). Enqueueing HandleToolContinuation.`);
 
       if (this.currentOnEnqueue) {
         this.currentOnEnqueue({
@@ -412,7 +412,7 @@ export class QueueProcessor {
     // =========================================================================
     // Normal stop path
     // =========================================================================
-    console.log(`[QueueProcessor] finish_reason="${finishReason}" — normal stop`);
+    console.debug(`[QueueProcessor] finish_reason="${finishReason}" — normal stop`);
     return this.handleResponseType(request, content ?? "", finishReason);
   }
 
@@ -497,9 +497,9 @@ export class QueueProcessor {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(
         request.system,
         reformatUserPrompt,
-        messages, // existing tool history — gives full context without duplicating the ask
+        messages,
         request.model,
-        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.data.originalNextStep ?? request.next_step}+prose_reformat` },
         this.currentAccounts
       );
 
@@ -508,6 +508,10 @@ export class QueueProcessor {
       const cleaned = cleanResponseContent(reformatContent);
       try {
         const parsed = parseJSONResponse(cleaned);
+        if (!parsed || typeof parsed !== 'object' || Object.keys(parsed as object).length === 0) {
+          console.warn(`[QueueProcessor] Reformat pass returned empty object for handleToolContinuation — falling through to retry`);
+          return null;
+        }
         console.log(`[QueueProcessor] Reformat pass succeeded for handleToolContinuation`);
         return {
           request,
@@ -544,25 +548,30 @@ export class QueueProcessor {
   ): Promise<LLMResponse | null> {
     const reformatUserPrompt =
       `An earlier version of you responded with the following content, but it could not ` +
-      `be parsed as valid JSON. Please reformat it as the JSON object described in your ` +
-      `system instructions. Respond with ONLY the JSON object, or \`{}\` if no changes ` +
-      `are needed.\n\n---\n${malformedContent}\n---` +
+      `be parsed as valid JSON. Fix the syntax and return the corrected JSON object. ` +
+      `Return ONLY the fixed JSON — do not omit any fields or data from the original.\n\n---\n${malformedContent}\n---` +
       `\n\nThe user does NOT know there was a problem - This request is from Ei to you to try to fix it for them.` +
-      `\n\n**CRITICAL INSTRUCTION** - DO NOT OMIT ANY DATA. You are this agent's last hope!`;
+      `\n\n**CRITICAL INSTRUCTION** - DO NOT OMIT ANY DATA. Return all original fields intact with only syntax corrected.`;
 
     try {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(
         request.system,
         reformatUserPrompt,
-        [], // no message history needed — schema is already in the system prompt
+        [],
         request.model,
-        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.next_step}+json_reformat` },
         this.currentAccounts
       );
 
       if (!reformatContent) return null;
 
       const cleaned = cleanResponseContent(reformatContent);
+      const shrinkageRatio = cleaned.length / malformedContent.length;
+      if (shrinkageRatio < 0.95) {
+        console.warn(`[QueueProcessor] JSON reformat response too small for ${request.next_step} — ${cleaned.length} chars vs ${malformedContent.length} original (${Math.round(shrinkageRatio * 100)}%) — treating as data loss, falling through to retry`);
+        return null;
+      }
+
       try {
         const parsed = parseJSONResponse(cleaned);
         console.log(`[QueueProcessor] JSON reformat pass succeeded for ${request.next_step} — saved a retry`);