ei-tui 1.0.1 → 1.2.0

package/src/core/orchestrators/ceremony.ts

@@ -137,10 +137,7 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
       messages_analyze: unextractedPeople,
       extraction_flag: "p",
     };
-    const personScanOptions = persona.pending_update
-      ? { ...options, reflection_progress: 1 }
-      : options;
-    queuePersonScan(context, state, personScanOptions);
+    queuePersonScan(context, state, options);
   }
   
   const totalUnextracted = unextractedFacts.length + unextractedTopics.length + unextractedPeople.length;
@@ -287,7 +284,7 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
   // Person Rewrite phase (phase 4): scan bloated Person records, extract Topics from them.
   // Gated via ceremony_progress so Topic Rewrite can run after — Topics created here
   // need to be visible before Topic Rewrite snapshots the threshold.
-  queuePersonRewritePhase(state);
+  queuePersonRewritePhase(state, { ceremonyProgress: 4 });
 
   // Zero-work guard: if no person rewrites queued, advance to topic rewrite immediately
   if (!state.queue_hasPendingCeremonies()) {
@@ -482,7 +479,7 @@ export function queueReflectionDrain(personaId: string, state: StateManager): vo
     messages_analyze: unextractedPeople,
     extraction_flag: "p",
   };
-  queuePersonScan(context, state, { reflection_progress: 1 });
+  queuePersonScan(context, state);
   console.log(`[reflection:drain] Queued Person scan for ${persona.display_name} (${unextractedPeople.length} messages) — clears on completion`);
 }
 
@@ -490,7 +487,7 @@ function getRewriteModel(state: StateManager): string | undefined {
   return state.getHuman().settings?.rewrite_model;
 }
 
-export function queuePersonRewritePhase(state: StateManager): void {
+export function queuePersonRewritePhase(state: StateManager, options?: { ceremonyProgress?: number }): void {
   const rewriteModel = getRewriteModel(state);
   if (!rewriteModel) {
     console.log("[ceremony:rewrite] rewrite_model not set — skipping person rewrite phase");
@@ -498,13 +495,30 @@ export function queuePersonRewritePhase(state: StateManager): void {
   }
 
   const human = state.getHuman();
-  const personsToScan = human.people.filter(person => {
+  const allCandidates = human.people.filter(person => {
     const isPersonaLinked = (person.identifiers ?? []).some(
       i => i.type.toLowerCase() === 'ei persona'
     );
     return !isPersonaLinked
-      && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
-      && !person.rewrite_checked;
+      && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD;
+  });
+
+  const alreadyChecked = allCandidates.filter(p => {
+    const descLen = p.description?.length ?? 0;
+    return p.rewrite_length_floor !== undefined && descLen < p.rewrite_length_floor;
+  });
+  if (alreadyChecked.length > 0) {
+    for (const person of alreadyChecked) {
+      console.log(
+        `[ceremony:rewrite] Person "${person.name}" is ${person.description?.length ?? 0} chars ` +
+        `(floor: ${person.rewrite_length_floor}) — already reviewed, skipping`
+      );
+    }
+  }
+
+  const personsToScan = allCandidates.filter(p => {
+    if (p.rewrite_length_floor === undefined) return true;
+    return (p.description?.length ?? 0) >= p.rewrite_length_floor;
   });
 
   if (personsToScan.length === 0) {
@@ -527,7 +541,7 @@ export function queuePersonRewritePhase(state: StateManager): void {
         itemId: person.id,
         itemType: "person" as RewriteItemType,
         rewriteModel,
-        ceremony_progress: 4,
+        ...(options?.ceremonyProgress !== undefined && { ceremony_progress: options.ceremonyProgress }),
       },
     });
   }
@@ -543,11 +557,28 @@ export function queueTopicRewritePhase(state: StateManager): void {
   }
 
   const human = state.getHuman();
-  const topicsToScan = human.topics.filter(topic =>
+  const allCandidateTopics = human.topics.filter(topic =>
     (topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
-    && !topic.rewrite_checked
   );
 
+  const alreadyCheckedTopics = allCandidateTopics.filter(t => {
+    const descLen = t.description?.length ?? 0;
+    return t.rewrite_length_floor !== undefined && descLen < t.rewrite_length_floor;
+  });
+  if (alreadyCheckedTopics.length > 0) {
+    for (const topic of alreadyCheckedTopics) {
+      console.log(
+        `[ceremony:rewrite] Topic "${topic.name}" is ${topic.description?.length ?? 0} chars ` +
+        `(floor: ${topic.rewrite_length_floor}) — already reviewed, skipping`
+      );
+    }
+  }
+
+  const topicsToScan = allCandidateTopics.filter(t => {
+    if (t.rewrite_length_floor === undefined) return true;
+    return (t.description?.length ?? 0) >= t.rewrite_length_floor;
+  });
+
   if (topicsToScan.length === 0) {
     console.log("[ceremony:rewrite] No topics above threshold — skipping topic rewrite phase");
     return;

package/src/core/orchestrators/human-extraction.ts

@@ -195,6 +195,15 @@ export function queuePersonScan(context: ExtractionContext, state: StateManager,
   
   if (chunks.length === 0) return 0;
 
+  // If the persona has a pending_update (reflection in progress), gate person
+  // scans so handleHumanPersonScan won't queue updates for persona-linked people.
+  // This prevents importers and other callers from bypassing the reflection lock
+  // — they don't know about pending_update, so we enforce it here centrally.
+  const persona = state.persona_getById(context.personaId);
+  const effectiveOptions: ExtractionOptions | undefined = persona?.pending_update
+    ? { ...options, reflection_progress: 1 }
+    : options;
+
   // Pre-mark messages before enqueuing — prevents duplicate scans if the
   // queue check fires again during LLM latency (100ms loop × 5s call = 50 dupes)
   for (const chunk of chunks) {
@@ -225,7 +234,7 @@ export function queuePersonScan(context: ExtractionContext, state: StateManager,
       user: prompt.user,
       next_step: LLMNextStep.HandleHumanPersonScan,
       data: {
-        ...options,
+        ...effectiveOptions,
         personaId: chunk.personaId,
         personaDisplayName: chunk.channelDisplayName,
         analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
@@ -311,6 +320,34 @@ const EMBEDDING_MIN_SIMILARITY = 0.3;
  */
 export const VALIDATE_MIN_SIMILARITY = 0.92;
 
+/**
+ * Returns the best cosine similarity between a topic candidate and any existing
+ * topic in state. Used by queueTopicValidate to detect near-duplicates after
+ * a new topic is created.
+ * Returns 0 if no topics exist or embedding fails.
+ */
+export async function getBestTopicSimilarity(
+  candidate: TopicScanCandidate,
+  state: StateManager
+): Promise<number> {
+  const human = state.getHuman();
+  const topicsWithEmbeddings = human.topics.filter(t => t.embedding && t.embedding.length > 0);
+  if (topicsWithEmbeddings.length === 0) return 0;
+  try {
+    const embeddingService = getEmbeddingService();
+    const candidateText = getTopicEmbeddingText({
+      name: candidate.name,
+      category: candidate.category,
+      description: candidate.description,
+    });
+    const candidateVector = await embeddingService.embed(candidateText);
+    const topK = findTopK(candidateVector, topicsWithEmbeddings, 1);
+    return topK.length > 0 ? topK[0].similarity : 0;
+  } catch {
+    return 0;
+  }
+}
+
 /**
  * Queue a topic match request using embedding-based similarity (topics only).
  */

package/src/core/orchestrators/index.ts

@@ -12,6 +12,7 @@ export {
   queueTargetedPersonUpdate,
   queueTargetedTopicUpdate,
   VALIDATE_MIN_SIMILARITY,
+  getBestTopicSimilarity,
   type ExtractionContext,
   type ExtractionOptions,
 } from "./human-extraction.js";

package/src/core/processor.ts

@@ -36,12 +36,15 @@ import { handlers } from "./handlers/index.js";
 import { normalizeRoomMessages, getMessageContent } from "./handlers/utils.js";
 import { sanitizeEiPersonaIdentifiers } from "./utils/identifier-utils.js";
 import { ContextStatus as ContextStatusEnum, RoomMode } from "./types.js";
-import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
-import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
+import { registerFindMemoryExecutor, registerFetchMemoryExecutor, registerFetchMessageExecutor, registerFileReadExecutor, SYSTEM_TOOLS } from "./tools/index.js";
+import { createFindMemoryExecutor } from "./tools/builtin/find-memory.js";
+import { createFetchMemoryExecutor } from "./tools/builtin/fetch-memory.js";
+import { createFetchMessageExecutor } from "./tools/builtin/fetch-message.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
 import { EMMETT_PERSONA_DEFINITION } from "../templates/emmett.js";
 import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueReflectionDrain, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction, queueTargetedPersonUpdate, queueTargetedTopicUpdate } from "./orchestrators/index.js";
 import { finishDocumentBatch } from "./handlers/document-segmentation.js";
+import { buildSynthesisPrompt } from "../prompts/synthesis/index.js";
 import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
 import { DEFAULT_SEED_TRAITS } from "./constants/seed-traits.js";
 
@@ -240,7 +243,15 @@ export class Processor {
     this.seedBuiltinFacts();
     this.migrateLearnedOn();
     this.seedSettings();
-    registerReadMemoryExecutor(createReadMemoryExecutor(this.searchHumanData.bind(this), this.getPersonaList.bind(this)));
+    registerFindMemoryExecutor(createFindMemoryExecutor(this.searchHumanData.bind(this), this.getPersonaList.bind(this), this.stateManager.getHuman.bind(this.stateManager)));
+    registerFetchMemoryExecutor(createFetchMemoryExecutor(this.stateManager.getHuman.bind(this.stateManager)));
+    registerFetchMessageExecutor(createFetchMessageExecutor(
+      this.stateManager.persona_getAll.bind(this.stateManager),
+      this.stateManager.messages_get.bind(this.stateManager),
+      this.stateManager.getRoomList.bind(this.stateManager),
+      this.stateManager.getRoomMessages.bind(this.stateManager),
+      (roomId: string) => this.stateManager.getRoom(roomId)?.display_name ?? null
+    ));
     if (this.isTUI) {
       await registerFileReadExecutor();
     }
@@ -294,13 +305,12 @@ export class Processor {
       }
       return;
     }
-    const readMemoryTool = this.stateManager.tools_getByName("read_memory");
     const emmettEntity: PersonaEntity = {
       ...EMMETT_PERSONA_DEFINITION,
       id: "emmet",
       display_name: "Emmett",
       last_updated: new Date().toISOString(),
-      tools: readMemoryTool ? [readMemoryTool.id] : [],
+      tools: [],
     };
     this.stateManager.persona_add(emmettEntity);
     this.interface.onPersonaAdded?.();
@@ -326,6 +336,140 @@ export class Processor {
     return executeUnsource(preview, this.stateManager);
   }
 
+  async generateDocument(subject: string): Promise<{ slug: string }> {
+    this.bootstrapEmmett();
+    const slugBase = subject
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, "-")
+      .slice(0, 40);
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const slug = `${slugBase}_${timestamp}`;
+
+    const primary = await this.searchHumanData(subject, { limit: 20 });
+    if (
+      primary.facts.length === 0 &&
+      primary.topics.length === 0 &&
+      primary.people.length === 0 &&
+      primary.quotes.length === 0
+    ) {
+      throw new Error(`No knowledge found about '${subject}'`);
+    }
+
+    const seenQuoteIds = new Set<string>();
+    const seenItemIds = new Set<string>(
+      [...primary.topics, ...primary.people, ...primary.facts].map(i => i.id)
+    );
+
+    const MAX_QUOTES_PER_ENTITY = 3;
+
+    const enrichTopic = (topic: import("../prompts/synthesis/types.js").EnrichedTopic["topic"]) => {
+      const linked = this.stateManager.human_quote_getForDataItem(topic.id)
+        .sort((a, b) => b.timestamp.localeCompare(a.timestamp))
+        .slice(0, MAX_QUOTES_PER_ENTITY);
+      linked.forEach(q => seenQuoteIds.add(q.id));
+      return { topic, quotes: linked };
+    };
+
+    const enrichPerson = (person: import("../prompts/synthesis/types.js").EnrichedPerson["person"]) => {
+      const linked = this.stateManager.human_quote_getForDataItem(person.id)
+        .sort((a, b) => b.timestamp.localeCompare(a.timestamp))
+        .slice(0, MAX_QUOTES_PER_ENTITY);
+      linked.forEach(q => seenQuoteIds.add(q.id));
+      return { person, quotes: linked };
+    };
+
+    const enrichedTopics = primary.topics.map(enrichTopic);
+    const enrichedPeople = primary.people.map(enrichPerson);
+
+    const human = this.stateManager.getHuman();
+    const allItems = [...human.facts, ...human.topics, ...human.people];
+
+    const MAX_SECONDARY_ENTITIES = 10;
+
+    const secondaryTopics: typeof enrichedTopics = [];
+    const secondaryPeople: typeof enrichedPeople = [];
+    const secondaryFacts: typeof primary.facts = [];
+
+    outer: for (const quote of [...enrichedTopics.flatMap(e => e.quotes), ...enrichedPeople.flatMap(e => e.quotes)]) {
+      for (const itemId of quote.data_item_ids) {
+        if (secondaryTopics.length + secondaryPeople.length + secondaryFacts.length >= MAX_SECONDARY_ENTITIES) break outer;
+        if (seenItemIds.has(itemId)) continue;
+        seenItemIds.add(itemId);
+        const item = allItems.find(i => i.id === itemId);
+        if (!item) continue;
+        if (human.topics.find(t => t.id === itemId)) {
+          secondaryTopics.push(enrichTopic(item as typeof primary.topics[0]));
+        } else if (human.people.find(p => p.id === itemId)) {
+          secondaryPeople.push(enrichPerson(item as typeof primary.people[0]));
+        } else if (human.facts.find(f => f.id === itemId)) {
+          secondaryFacts.push(item as typeof primary.facts[0]);
+        }
+      }
+    }
+
+    const standaloneQuotes = primary.quotes.filter(q => !seenQuoteIds.has(q.id));
+
+    const allLoadedFacts = [...primary.facts, ...secondaryFacts];
+    const allLoadedTopics = [...enrichedTopics, ...secondaryTopics];
+    const allLoadedPeople = [...enrichedPeople, ...secondaryPeople];
+
+    const loadedEntityNames = new Map<string, string>();
+    for (const f of allLoadedFacts) loadedEntityNames.set(f.id, f.name);
+    for (const { topic } of allLoadedTopics) loadedEntityNames.set(topic.id, topic.name);
+    for (const { person } of allLoadedPeople) loadedEntityNames.set(person.id, person.name);
+
+    const prompt = buildSynthesisPrompt({
+      subject,
+      facts: allLoadedFacts,
+      topics: allLoadedTopics,
+      people: allLoadedPeople,
+      standaloneQuotes,
+      loadedEntityNames,
+    });
+
+    const model = this.stateManager.getHuman().settings?.rewrite_model
+      ?? this.stateManager.getHuman().settings?.default_model;
+
+    this.stateManager.queue_enqueue({
+      type: LLMRequestType.Raw,
+      priority: LLMPriority.Normal,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleKnowledgeSynthesis,
+      model,
+      data: { slug, subject },
+    });
+
+    return { slug };
+  }
+
+  checkGenerationModel(): { model: string; isRewriteModel: boolean } {
+    const settings = this.stateManager.getHuman().settings;
+    if (settings?.rewrite_model) {
+      return { model: settings.rewrite_model, isRewriteModel: true };
+    }
+    return { model: settings?.default_model ?? "unknown", isRewriteModel: false };
+  }
+
+  async getGeneratedDocumentContent(slug: string): Promise<string | null> {
+    const messages = this.stateManager.messages_get("emmet");
+    const target = `generate:document:${slug}`;
+    const message = messages.find(m => m.source_tag === target);
+    return message?.content ?? null;
+  }
+
+  async reRunDocument(slug: string): Promise<{ slug: string }> {
+    const docs = this.stateManager.getHuman().settings?.document?.processed_documents ?? {};
+    const entry = docs[slug];
+    if (!entry || entry.type !== "generated" || !entry.subject) {
+      throw new Error(`No generated document found for slug "${slug}"`);
+    }
+    const subject = entry.subject;
+    const preview = this.getUnsourcePreview(`generate:document:${slug}`);
+    await this.executeUnsource(preview);
+    return this.generateDocument(subject);
+  }
+
   /**
    * Seed built-in tool providers and tools if they don't exist yet.
    * Called on every startup (after state load/restore) — safe to call repeatedly.
@@ -334,6 +478,11 @@ export class Processor {
   private bootstrapTools(): void {
     const now = new Date().toISOString();
 
+    for (const name of ["find_memory", "fetch_memory", "fetch_message", "read_memory"]) {
+      const tool = this.stateManager.tools_getByName(name);
+      if (tool) this.stateManager.tools_remove(tool.id);
+    }
+
     // --- Ei built-in provider ---
     if (!this.stateManager.tools_getProviderById("ei")) {
       const eiProvider: ToolProvider = {
@@ -349,35 +498,6 @@ export class Processor {
       this.stateManager.tools_addProvider(eiProvider);
     }
 
-    // read_memory tool
-    this.stateManager.tools_upsertBuiltin({
-        id: crypto.randomUUID(),
-        provider_id: "ei",
-        name: "read_memory",
-        display_name: "Read Memory",
-        description:
-          "Search Ei's persistent knowledge base — facts, topics, people, and quotes learned across ALL conversations over time, not just this one. Use this when you need context about the user, their life, relationships, or interests that may not be visible in the current exchange. Use `recent: true` to retrieve what's been discussed recently.",
-        input_schema: {
-          type: "object",
-          properties: {
-            query: { type: "string", description: "What to search for — a person, topic, fact, or anything Ei has learned about the user" },
-            types: {
-              type: "array",
-              items: { type: "string", enum: ["fact", "topic", "person", "quote"] },
-              description: "Limit search to specific memory types (default: all types)",
-            },
-            limit: { type: "number", description: "Max results to return (default: 10, max: 20)" },
-            recent: { type: "boolean", description: "If true, return recently-mentioned results sorted by last_mentioned date instead of relevance. Combine with a query to filter recent results by topic." },
-          },
-          required: [],
-        },
-        runtime: "any",
-        builtin: true,
-        enabled: true,
-        created_at: now,
-        max_calls_per_interaction: 6,  // Dedup needs to verify relationships before irreversible merges. Typical cluster (3-8 items) requires: parent concept lookup + 2 relationship verifications + context validation. Still under HARD_TOOL_CALL_LIMIT (10).
-    });
-
     // file_read tool (TUI only)
     this.stateManager.tools_upsertBuiltin({
         id: crypto.randomUUID(),
@@ -817,6 +937,20 @@ export class Processor {
         max_calls_per_interaction: 1,
         created_at: now,
     });
+
+    // --- Reconcile pass: prune stale tool references from persona tool lists ---
+    // Build manifest of all tool IDs currently in state (everything seeded above).
+    const manifestIds = new Set(this.stateManager.tools_getAll().map(t => t.id));
+
+    for (const persona of this.stateManager.persona_getAll()) {
+      if (!persona.tools?.length) continue;
+      const pruned = persona.tools.filter(id => manifestIds.has(id));
+      if (pruned.length !== persona.tools.length) {
+        const removed = persona.tools.length - pruned.length;
+        this.stateManager.persona_update(persona.id, { tools: pruned });
+        console.log(`[Processor] Pruned ${removed} stale tool reference(s) from persona "${persona.display_name}"`);
+      }
+    }
   }
 
   /**
@@ -1088,28 +1222,34 @@ const toolNextSteps = new Set([
   LLMNextStep.HandleEiHeartbeat,
   LLMNextStep.HandleToolContinuation,
   LLMNextStep.HandleDedupCurate,
+  LLMNextStep.HandleKnowledgeSynthesis,
 ]);
             const toolPersonaId =
               personaId ??
               (request.next_step === LLMNextStep.HandleEiHeartbeat ? "ei" : undefined);
             
-            // Dedup operates on Human data, not persona data - provide read_memory directly.
+            // Dedup operates on Human data, not persona data — provide find_memory from SYSTEM_TOOLS directly.
             // Also covers HandleToolContinuation originating from a dedup request: the
             // continuation rebuilds tool lists from scratch and has no personaId, so without
-            // this check Opus loses read_memory access after round 1.
+            // this check Opus loses find_memory access after round 1.
             const isDedupRequest =
               request.next_step === LLMNextStep.HandleDedupCurate ||
               (request.next_step === LLMNextStep.HandleToolContinuation &&
                 request.data.originalNextStep === LLMNextStep.HandleDedupCurate);
 
+            const isSynthesisRequest = request.next_step === LLMNextStep.HandleKnowledgeSynthesis ||
+              (request.next_step === LLMNextStep.HandleToolContinuation &&
+                request.data.originalNextStep === LLMNextStep.HandleKnowledgeSynthesis);
+
             let tools: ToolDefinition[] = [];
             if (isDedupRequest) {
-              const readMemory = this.stateManager.tools_getByName("read_memory");
-              if (readMemory?.enabled) {
-                tools = [readMemory];
-              }
+              tools = SYSTEM_TOOLS.filter(t => t.name === "find_memory");
+            } else if (isSynthesisRequest) {
+              tools = SYSTEM_TOOLS.filter(t =>
+                t.name === "find_memory" || t.name === "fetch_memory" || t.name === "fetch_message"
+              );
             } else if (toolNextSteps.has(request.next_step) && toolPersonaId) {
-              tools = this.stateManager.tools_getForPersona(toolPersonaId, this.isTUI);
+              tools = [...SYSTEM_TOOLS, ...this.stateManager.tools_getForPersona(toolPersonaId, this.isTUI)];
             }
 
             // Auto-inject each handler's dedicated submit tool — infrastructure, not user-visible.
@@ -1764,6 +1904,17 @@ const toolNextSteps = new Set([
           this.interface.onHumanUpdated?.();
         }
       }
+
+      const isSynthesisCompletion =
+        response.request.next_step === LLMNextStep.HandleKnowledgeSynthesis ||
+        (response.request.next_step === LLMNextStep.HandleToolContinuation &&
+          response.request.data.originalNextStep === LLMNextStep.HandleKnowledgeSynthesis);
+      if (isSynthesisCompletion) {
+        const slug = response.request.data.slug as string;
+        const hasContent = slug && this.stateManager.messages_get("emmet")
+          .some(m => m.source_tag === `generate:document:${slug}`);
+        if (hasContent) this.interface.onDocumentGenerated?.(slug);
+      }
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : String(err);
       const result = this.stateManager.queue_fail(response.request.id, errorMsg);

package/src/core/prompt-context-builder.ts

@@ -274,6 +274,7 @@ export async function buildResponsePromptData(
 
   const alwaysMessages = sm.messages_getAlways(persona.id);
   const temporalAnchors = alwaysMessages.map(m => ({
+    id: m.id,
     role: m.role === "human" ? "human" as const : "system" as const,
     content: m.content,
     silence_reason: m.silence_reason,

package/src/core/queue-manager.ts

@@ -51,6 +51,15 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
   }
   const extracting_documents = extractingSet.size > 0 ? Array.from(extractingSet) : undefined;
 
+  const generatingSet: string[] = [];
+  for (const item of activeItems) {
+    if (item.next_step === LLMNextStep.HandleKnowledgeSynthesis) {
+      const slug = (item.data as { slug?: string }).slug;
+      if (slug) generatingSet.push(slug);
+    }
+  }
+  const generating_documents = generatingSet.length > 0 ? generatingSet : undefined;
+
   return {
     state: sm.queue_isPaused()
       ? "paused"
@@ -62,6 +71,7 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
     embedding_warning: sm.embedding_getWarning() || undefined,
     pending_documents,
     extracting_documents,
+    generating_documents,
   };
 }
 

package/src/core/queue-processor.ts

@@ -508,6 +508,10 @@ export class QueueProcessor {
       const cleaned = cleanResponseContent(reformatContent);
       try {
         const parsed = parseJSONResponse(cleaned);
+        if (!parsed || typeof parsed !== 'object' || Object.keys(parsed as object).length === 0) {
+          console.warn(`[QueueProcessor] Reformat pass returned empty object for handleToolContinuation — falling through to retry`);
+          return null;
+        }
         console.log(`[QueueProcessor] Reformat pass succeeded for handleToolContinuation`);
         return {
           request,
@@ -544,11 +548,10 @@ export class QueueProcessor {
   ): Promise<LLMResponse | null> {
     const reformatUserPrompt =
       `An earlier version of you responded with the following content, but it could not ` +
-      `be parsed as valid JSON. Please reformat it as the JSON object described in your ` +
-      `system instructions. Respond with ONLY the JSON object, or \`{}\` if no changes ` +
-      `are needed.\n\n---\n${malformedContent}\n---` +
+      `be parsed as valid JSON. Fix the syntax and return the corrected JSON object. ` +
+      `Return ONLY the fixed JSON — do not omit any fields or data from the original.\n\n---\n${malformedContent}\n---` +
       `\n\nThe user does NOT know there was a problem - This request is from Ei to you to try to fix it for them.` +
-      `\n\n**CRITICAL INSTRUCTION** - DO NOT OMIT ANY DATA. You are this agent's last hope!`;
+      `\n\n**CRITICAL INSTRUCTION** - DO NOT OMIT ANY DATA. Return all original fields intact with only syntax corrected.`;
 
     try {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(
@@ -563,6 +566,12 @@ export class QueueProcessor {
       if (!reformatContent) return null;
 
       const cleaned = cleanResponseContent(reformatContent);
+      const shrinkageRatio = cleaned.length / malformedContent.length;
+      if (shrinkageRatio < 0.95) {
+        console.warn(`[QueueProcessor] JSON reformat response too small for ${request.next_step} — ${cleaned.length} chars vs ${malformedContent.length} original (${Math.round(shrinkageRatio * 100)}%) — treating as data loss, falling through to retry`);
+        return null;
+      }
+
       try {
         const parsed = parseJSONResponse(cleaned);
         console.log(`[QueueProcessor] JSON reformat pass succeeded for ${request.next_step} — saved a retry`);

package/src/core/state-manager.ts

@@ -71,6 +71,7 @@ export class StateManager {
     this.migrateProviderModel();
     this.migrateThemes();
     this.migrateFfaParentIds();
+    this.migrateDocumentSettings();
   }
 
   /**
@@ -586,6 +587,40 @@ export class StateManager {
     }
   }
 
+  private migrateDocumentSettings(): void {
+    const human = this.humanState.get();
+    const doc = human.settings?.document;
+    if (!doc) return;
+
+    let migrated = false;
+
+    const existing = doc.processed_documents ?? {};
+    for (const [key, value] of Object.entries(existing)) {
+      if (typeof value === "string") {
+        (existing as Record<string, unknown>)[key] = { created_at: value, type: "imported" };
+        migrated = true;
+      }
+    }
+
+    const legacy = (doc as Record<string, unknown>).generated_documents as
+      | Record<string, { subject: string; created_at: string }>
+      | undefined;
+    if (legacy) {
+      for (const [slug, record] of Object.entries(legacy)) {
+        existing[slug] = { created_at: record.created_at, type: "generated", subject: record.subject };
+      }
+      delete (doc as Record<string, unknown>).generated_documents;
+      migrated = true;
+    }
+
+    if (migrated) {
+      doc.processed_documents = existing as import("./types/entities.js").DocumentSettings["processed_documents"];
+      this.humanState.set(human);
+      this.scheduleSave();
+      console.log("[StateManager] Migrated document settings to unified processed_documents schema");
+    }
+  }
+
   getHuman(): HumanEntity {
     return this.humanState.get();
   }