ei-tui 0.9.4 → 1.0.1

package/src/core/orchestrators/ceremony.ts

@@ -1,4 +1,4 @@
-import { LLMRequestType, LLMPriority, LLMNextStep, RoomMode, ContextStatus, type CeremonyConfig, type PersonaTopic, type Topic, type DataItemBase } from "../types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep, RoomMode, ContextStatus, type CeremonyConfig, type PersonaTopic, type Topic } from "../types.js";
 import type { StateManager } from "../state-manager.js";
 import { normalizeRoomMessages } from "../handlers/utils.js";
 import { applyDecayToValue } from "../utils/index.js";
@@ -12,7 +12,9 @@ import {
 } from "./human-extraction.js";
 import { queuePersonaTopicRating, type PersonaTopicContext, type PersonaTopicOptions } from "./persona-topics.js";
 import { getRoomVisibleMessages, queueRoomHumanExtraction } from "./room-extraction.js";
-import { buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
+import { type RewriteItemType } from "../../prompts/ceremony/index.js";
+import { buildPersonRewriteScanPrompt } from "../../prompts/ceremony/people-rewrite.js";
+import { buildTopicRewriteScanPrompt } from "../../prompts/ceremony/topic-rewrite.js";
 import { buildReflectionCriticPrompt } from "../../prompts/reflection/index.js";
 import { getModelForPersona } from "../heartbeat-manager.js";
 
@@ -51,7 +53,7 @@ export function shouldStartCeremony(config: CeremonyConfig, state: StateManager,
  * Start the ceremony by queuing Exposure scans for all active personas with recent activity.
  * 
  * IMPORTANT: Sets last_ceremony FIRST to prevent re-triggering from the processor loop.
- * The actual Decay → Prune → Expire → Explore phases happen later via handleCeremonyProgress
+ * The actual Decay → Person Rewrite → Topic Rewrite phases happen later via handleCeremonyProgress
  * once all exposure scans have completed.
  */
 export function startCeremony(state: StateManager): void {
@@ -167,7 +169,7 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
  * AND at the end of startCeremony (for the zero-messages edge case).
  * 
  * If any ceremony_progress items remain in the queue, does nothing — more work pending.
- * Phase 1: Dedup → Phase 2: Expose → Phase 3: EventSummary → Decay → Expire
+ * Phase 1: Dedup → Phase 2: Expose → Phase 3: EventSummary → Decay → Phase 4: Person Rewrite → Topic Rewrite (fire-and-forget)
  */
 export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
   if (state.queue_hasPendingCeremonies()) {
@@ -236,6 +238,12 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
     return;
   }
 
+  if (lastPhase === 4) {
+    console.log("[ceremony:progress] Person Rewrite complete, starting Topic Rewrite");
+    queueTopicRewritePhase(state);
+    return;
+  }
+
   if (lastPhase === 2) {
     console.log("[ceremony:progress] Expose complete, starting EventSummary phase");
     const options: ExtractionOptions = { ceremony_progress: 3 };
@@ -249,7 +257,7 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
     return;
   }
   
-  // Phase 3 (EventSummary) complete → advance to Decay/Prune/Expire/Explore
+  // Phase 3 (EventSummary) complete → advance to Decay/Prune then Person Rewrite (phase 4)
   console.log("[ceremony:progress] EventSummary complete, advancing to Decay");
   
   const personas = state.persona_getAll();
@@ -276,8 +284,16 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
   // Human ceremony: decay topics + people
   runHumanCeremony(state);
 
-  // Rewrite phase: fire-and-forget scans for bloated human data items
-  queueRewritePhase(state);
+  // Person Rewrite phase (phase 4): scan bloated Person records, extract Topics from them.
+  // Gated via ceremony_progress so Topic Rewrite can run after — Topics created here
+  // need to be visible before Topic Rewrite snapshots the threshold.
+  queuePersonRewritePhase(state);
+
+  // Zero-work guard: if no person rewrites queued, advance to topic rewrite immediately
+  if (!state.queue_hasPendingCeremonies()) {
+    console.log("[ceremony:progress] No person rewrite work, advancing to Topic Rewrite");
+    handleCeremonyProgress(state, 4);
+  }
 
   // Reflection phase: fire-and-forget critic calls for persona person records above threshold
   queueReflectionPhase(state);
@@ -441,15 +457,6 @@ export function runHumanCeremony(state: StateManager): void {
 
 const REWRITE_DESCRIPTION_THRESHOLD = 750;
 
-/**
- * Queue Phase 1 "scan" for every human data item whose description exceeds the
- * threshold. Gated on rewrite_model being set in HumanSettings.
- * 
- * Fire-and-forget: no ceremony_progress, no blocking. Expire/Explore proceed
- * immediately since they only touch persona topics (zero overlap with human data).
- * Phase 2 items enqueue at Normal priority, naturally processing before more
- * Low-priority Phase 1 scans.
- */
 /**
  * Forces an unconditional, threshold-bypassing Person scan on Apply/Dismiss.
  * Cannot be replaced by checkAndQueueHumanExtraction — that function gates on
@@ -479,41 +486,77 @@ export function queueReflectionDrain(personaId: string, state: StateManager): vo
   console.log(`[reflection:drain] Queued Person scan for ${persona.display_name} (${unextractedPeople.length} messages) — clears on completion`);
 }
 
-export function queueRewritePhase(state: StateManager): void {
-  const human = state.getHuman();
-  const rewriteModel = human.settings?.rewrite_model;
+function getRewriteModel(state: StateManager): string | undefined {
+  return state.getHuman().settings?.rewrite_model;
+}
 
+export function queuePersonRewritePhase(state: StateManager): void {
+  const rewriteModel = getRewriteModel(state);
   if (!rewriteModel) {
-    console.log("[ceremony:rewrite] rewrite_model not set — skipping rewrite phase");
+    console.log("[ceremony:rewrite] rewrite_model not set — skipping person rewrite phase");
     return;
   }
 
-  const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
-
-  for (const topic of human.topics) {
-    if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !topic.rewrite_checked) {
-      itemsToScan.push({ item: topic, type: "topic" });
-    }
-  }
-  for (const person of human.people) {
+  const human = state.getHuman();
+  const personsToScan = human.people.filter(person => {
     const isPersonaLinked = (person.identifiers ?? []).some(
       i => i.type.toLowerCase() === 'ei persona'
     );
-    if (!isPersonaLinked && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD && !person.rewrite_checked) {
-      itemsToScan.push({ item: person, type: "person" });
-    }
+    return !isPersonaLinked
+      && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
+      && !person.rewrite_checked;
+  });
+
+  if (personsToScan.length === 0) {
+    console.log("[ceremony:rewrite] No persons above threshold — skipping person rewrite phase");
+    return;
+  }
+
+  console.log(`[ceremony:rewrite] Found ${personsToScan.length} person(s) above ${REWRITE_DESCRIPTION_THRESHOLD} chars — queueing person rewrite scans`);
+
+  for (const person of personsToScan) {
+    const prompt = buildPersonRewriteScanPrompt({ item: person, itemType: "person" });
+    state.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: prompt.system,
+      user: prompt.user,
+      next_step: LLMNextStep.HandleRewriteScan,
+      model: rewriteModel,
+      data: {
+        itemId: person.id,
+        itemType: "person" as RewriteItemType,
+        rewriteModel,
+        ceremony_progress: 4,
+      },
+    });
   }
 
-  if (itemsToScan.length === 0) {
-    console.log("[ceremony:rewrite] No items above threshold — nothing to rewrite");
+  console.log(`[ceremony:rewrite] Queued ${personsToScan.length} person rewrite scan(s)`);
+}
+
+export function queueTopicRewritePhase(state: StateManager): void {
+  const rewriteModel = getRewriteModel(state);
+  if (!rewriteModel) {
+    console.log("[ceremony:rewrite] rewrite_model not set — skipping topic rewrite phase");
     return;
   }
 
-  console.log(`[ceremony:rewrite] Found ${itemsToScan.length} item(s) above ${REWRITE_DESCRIPTION_THRESHOLD} chars — queueing Phase 1 scans`);
+  const human = state.getHuman();
+  const topicsToScan = human.topics.filter(topic =>
+    (topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
+    && !topic.rewrite_checked
+  );
+
+  if (topicsToScan.length === 0) {
+    console.log("[ceremony:rewrite] No topics above threshold — skipping topic rewrite phase");
+    return;
+  }
 
-  for (const { item, type } of itemsToScan) {
-    const prompt = buildRewriteScanPrompt({ item, itemType: type });
+  console.log(`[ceremony:rewrite] Found ${topicsToScan.length} topic(s) above ${REWRITE_DESCRIPTION_THRESHOLD} chars — queueing topic rewrite scans`);
 
+  for (const topic of topicsToScan) {
+    const prompt = buildTopicRewriteScanPrompt({ item: topic, itemType: "topic" });
     state.queue_enqueue({
       type: LLMRequestType.JSON,
       priority: LLMPriority.Low,
@@ -522,14 +565,14 @@ export function queueRewritePhase(state: StateManager): void {
       next_step: LLMNextStep.HandleRewriteScan,
       model: rewriteModel,
       data: {
-        itemId: item.id,
-        itemType: type,
-        rewriteModel,  // pass through so Phase 1 handler can queue Phase 2 with the same model
+        itemId: topic.id,
+        itemType: "topic" as RewriteItemType,
+        rewriteModel,
       },
     });
   }
 
-  console.log(`[ceremony:rewrite] Queued ${itemsToScan.length} Phase 1 scan(s) at Low priority`);
+  console.log(`[ceremony:rewrite] Queued ${topicsToScan.length} topic rewrite scan(s)`);
 }
 
 function queueEventSummaryForAll(state: StateManager, options?: ExtractionOptions): void {

package/src/core/orchestrators/human-extraction.ts

@@ -164,6 +164,7 @@ export function queueTopicScan(context: ExtractionContext, state: StateManager,
       messages_context: chunk.messages_context,
       messages_analyze: chunk.messages_analyze,
       participant_context: buildParticipantContext(context.personaId, state),
+      technical_context: (context.sources?.length ?? 0) > 0,
     });
 
     state.queue_enqueue({
@@ -275,6 +276,7 @@ export function queueDirectTopicUpdate(
       messages_analyze: chunk.messages_analyze,
       persona_name: chunk.channelDisplayName,
       participant_context: buildParticipantContext(context.personaId, state),
+      technical_context: (context.sources?.length ?? 0) > 0,
     });
 
     state.queue_enqueue({
@@ -291,6 +293,7 @@ export function queueDirectTopicUpdate(
         existingItemId: topic.id,
         analyze_from_timestamp: getAnalyzeFromTimestamp(chunk),
         extraction_model: extractionModel,
+        sources: context.sources,
       },
     });
   }
@@ -306,7 +309,7 @@ const EMBEDDING_MIN_SIMILARITY = 0.3;
  * Higher than EMBEDDING_MIN_SIMILARITY (0.3) because we need near-duplicates,
  * not just vague thematic overlap.
  */
-export const VALIDATE_MIN_SIMILARITY = 0.85;
+export const VALIDATE_MIN_SIMILARITY = 0.92;
 
 /**
  * Queue a topic match request using embedding-based similarity (topics only).
@@ -425,6 +428,7 @@ export function queueTopicUpdate(
       messages_analyze: chunk.messages_analyze,
       persona_name: chunk.channelDisplayName,
       participant_context: buildParticipantContext(primaryPersonaId, state),
+      technical_context: (context.sources?.length ?? 0) > 0,
     });
 
     state.queue_enqueue({

package/src/core/persona-manager.ts

@@ -1,6 +1,7 @@
 import {
   RESERVED_PERSONA_NAMES,
   isReservedPersonaName,
+  isReservedPersonaId,
   type PersonaSummary,
   type PersonaEntity,
   type PersonaCreationInput,
@@ -110,6 +111,9 @@ export async function deletePersona(
   personaId: string,
   _deleteHumanData: boolean
 ): Promise<boolean> {
+  if (isReservedPersonaId(personaId)) {
+    throw new Error(`Cannot delete reserved persona "${personaId}". Use archive instead.`);
+  }
   const persona = sm.persona_getById(personaId);
   if (!persona) return false;
   sm.persona_delete(personaId);

package/src/core/processor.ts

@@ -39,7 +39,9 @@ import { ContextStatus as ContextStatusEnum, RoomMode } from "./types.js";
 import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
 import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
+import { EMMETT_PERSONA_DEFINITION } from "../templates/emmett.js";
 import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueReflectionDrain, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction, queueTargetedPersonUpdate, queueTargetedTopicUpdate } from "./orchestrators/index.js";
+import { finishDocumentBatch } from "./handlers/document-segmentation.js";
 import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
 import { DEFAULT_SEED_TRAITS } from "./constants/seed-traits.js";
 
@@ -132,6 +134,8 @@ import {
   markAllRoomMessagesRead,
 } from "./room-manager.js";
 import type { RoomCreationInput, RoomEntity, RoomMessage, RoomSummary } from "./types.js";
+import { previewUnsource as _previewUnsource } from "../integrations/document/unsource.js";
+import type { UnsourcePreview, UnsourceResult } from "../integrations/document/unsource.js";
 
 const DEFAULT_LOOP_INTERVAL_MS = 100;
 const DEFAULT_OPENCODE_POLLING_MS = 60000;
@@ -282,6 +286,46 @@ export class Processor {
     this.interface.onMessageAdded?.(eiEntity.id);
   }
 
+  private bootstrapEmmett(): void {
+    const existing = this.stateManager.persona_getById("emmet");
+    if (existing) {
+      if (existing.is_archived) {
+        this.stateManager.persona_unarchive("emmet");
+      }
+      return;
+    }
+    const readMemoryTool = this.stateManager.tools_getByName("read_memory");
+    const emmettEntity: PersonaEntity = {
+      ...EMMETT_PERSONA_DEFINITION,
+      id: "emmet",
+      display_name: "Emmett",
+      last_updated: new Date().toISOString(),
+      tools: readMemoryTool ? [readMemoryTool.id] : [],
+    };
+    this.stateManager.persona_add(emmettEntity);
+    this.interface.onPersonaAdded?.();
+  }
+
+  async importDocument(content: string, filename: string): Promise<import("../integrations/document/types.js").DocumentImportResult> {
+    this.bootstrapEmmett();
+    const { importDocument } = await import("../integrations/document/importer.js");
+    return importDocument({
+      stateManager: this.stateManager,
+      interface: this.interface,
+      content,
+      filename,
+    });
+  }
+
+  getUnsourcePreview(sourceTag: string): UnsourcePreview {
+    return _previewUnsource(sourceTag, this.stateManager);
+  }
+
+  async executeUnsource(preview: UnsourcePreview): Promise<UnsourceResult> {
+    const { executeUnsource } = await import("../integrations/document/unsource.js");
+    return executeUnsource(preview, this.stateManager);
+  }
+
   /**
    * Seed built-in tool providers and tools if they don't exist yet.
    * Called on every startup (after state load/restore) — safe to call repeatedly.
@@ -1168,6 +1212,15 @@ const toolNextSteps = new Set([
       await this.checkAndSyncCursor(human, now);
     }
 
+    if (
+      this.isTUI &&
+      human.settings?.personaHistory?.integration &&
+      !human.settings.personaHistory.complete &&
+      this.stateManager.queue_length() === 0
+    ) {
+      await this.checkAndSyncPersonaHistory(human);
+    }
+
     if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
       if (human.settings?.sync && remoteSync.isConfigured()) {
         const state = this.stateManager.getStorageState();
@@ -1180,7 +1233,7 @@ const toolNextSteps = new Set([
     }
 
     for (const persona of this.stateManager.persona_getAll()) {
-      if (persona.is_paused || persona.is_archived) continue;
+      if (persona.is_paused || persona.is_archived || persona.is_static) continue;
 
       const defaultHeartbeatMs = this.stateManager.getHuman().settings?.default_heartbeat_ms ?? 1800000;
       const heartbeatDelay = persona.heartbeat_delay_ms ?? defaultHeartbeatMs;
@@ -1408,6 +1461,32 @@ const toolNextSteps = new Set([
       });
   }
 
+  private personaHistoryImportInProgress = false;
+
+  private async checkAndSyncPersonaHistory(_human: HumanEntity): Promise<void> {
+    if (this.personaHistoryImportInProgress) return;
+
+    this.personaHistoryImportInProgress = true;
+    import("../integrations/persona-history/importer.js")
+      .then(({ importPersonaHistory }) =>
+        importPersonaHistory({ stateManager: this.stateManager })
+      )
+      .then((result) => {
+        if (result.scansQueued > 0) {
+          console.log(
+            `[Processor] PersonaHistory: ${result.scansQueued} scans queued` +
+            (result.complete ? " — import complete" : "")
+          );
+        }
+      })
+      .catch((err) => {
+        console.warn(`[Processor] PersonaHistory sync failed:`, err);
+      })
+      .finally(() => {
+        this.personaHistoryImportInProgress = false;
+      });
+  }
+
   private augmentRoomRequest(request: LLMRequest): LLMRequest {
     if (request.next_step !== LLMNextStep.HandleRoomResponse) return request;
 
@@ -1675,6 +1754,16 @@ const toolNextSteps = new Set([
       if (typeof response.request.data.ceremony_progress === "number") {
         handleCeremonyProgress(this.stateManager, response.request.data.ceremony_progress);
       }
+
+      if (response.request.next_step === LLMNextStep.HandleDocumentSegmentation) {
+        const batchId = response.request.data.batchId as string;
+        const filename = response.request.data.filename as string;
+        if (batchId && !this.stateManager.queue_hasPendingDocumentSegments(batchId)) {
+          finishDocumentBatch(batchId, filename, this.stateManager);
+          this.interface.onMessageAdded?.("emmet");
+          this.interface.onHumanUpdated?.();
+        }
+      }
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : String(err);
       const result = this.stateManager.queue_fail(response.request.id, errorMsg);

package/src/core/queue-manager.ts

@@ -18,6 +18,39 @@ export async function resumeQueue(sm: StateManager): Promise<void> {
 }
 
 export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
+  const activeItems = sm.queue_getAllActiveItems();
+  const segmentationItems = activeItems.filter(
+    r => r.next_step === LLMNextStep.HandleDocumentSegmentation
+  );
+
+  const batchMap = new Map<string, { filename: string; count: number }>();
+  for (const item of segmentationItems) {
+    const { batchId, filename } = item.data as { batchId: string; filename: string };
+    if (!batchId || !filename) continue;
+    const existing = batchMap.get(batchId);
+    if (existing) {
+      existing.count++;
+    } else {
+      batchMap.set(batchId, { filename, count: 1 });
+    }
+  }
+
+  const pending_documents = batchMap.size > 0
+    ? Array.from(batchMap.entries()).map(([batchId, { filename, count }]) => ({ batchId, filename, count }))
+    : undefined;
+
+  const extractingSet = new Set<string>();
+  for (const item of activeItems) {
+    const sources = item.data.sources as string[] | undefined;
+    if (!Array.isArray(sources)) continue;
+    for (const s of sources) {
+      if (typeof s === "string" && s.startsWith("import:document:")) {
+        extractingSet.add(s.slice("import:document:".length));
+      }
+    }
+  }
+  const extracting_documents = extractingSet.size > 0 ? Array.from(extractingSet) : undefined;
+
   return {
     state: sm.queue_isPaused()
       ? "paused"
@@ -27,6 +60,8 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
     pending_count: sm.queue_length(),
     dlq_count: sm.queue_dlqLength(),
     embedding_warning: sm.embedding_getWarning() || undefined,
+    pending_documents,
+    extracting_documents,
   };
 }
 

package/src/core/queue-processor.ts

@@ -200,7 +200,7 @@ export class QueueProcessor {
         hydratedUser,
         messages,
         request.model,
-        { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.data.originalNextStep ?? request.next_step}+tool_continuation` },
         this.currentAccounts
       );
 
@@ -219,7 +219,7 @@ export class QueueProcessor {
             if (!args.should_respond && args.content) {
               args.should_respond = true;
             }
-            console.log(`[QueueProcessor] submit tool "${submitCall.name}" called — returning arguments as parsed response`);
+      console.debug(`[QueueProcessor] submit tool "${submitCall.name}" called — returning arguments as parsed response`);
             return {
               request,
               success: true,
@@ -297,9 +297,9 @@ export class QueueProcessor {
     const isHeartbeat = request.next_step === LLMNextStep.HandleHeartbeatCheck || request.next_step === LLMNextStep.HandleEiHeartbeat;
     if (isHeartbeat) {
       const personaName = request.data.personaDisplayName as string | undefined ?? 'Ei';
-      console.log(`[${personaName} Heartbeat] LLM call - tools offered: ${openAITools.length} (${activeTools.map(t => t.name).join(', ') || 'none'})`);
+      console.debug(`[${personaName} Heartbeat] LLM call - tools offered: ${openAITools.length} (${activeTools.map(t => t.name).join(', ') || 'none'})`);
     } else {
-      console.log(`[QueueProcessor] LLM call for ${request.next_step}, tools=${openAITools.length}`);
+      console.debug(`[QueueProcessor] LLM call for ${request.next_step}, tools=${openAITools.length}`);
     }
 
     const { content, finishReason, rawToolCalls, assistantMessage, thinking } = await callLLMRaw(
@@ -307,18 +307,18 @@ export class QueueProcessor {
       hydratedUser,
       messages,
       request.model,
-      { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate },
+      { signal: this.abortController?.signal, tools: openAITools, onUsageUpdate: this.currentOnUsageUpdate, nextStep: request.next_step },
       this.currentAccounts
     );
     if (thinking) {
-      console.log(`[QueueProcessor] Extended thinking on ${request.next_step} (${thinking.length} chars) — TODO(#13): stream to TUI`);
+      console.debug(`[QueueProcessor] Extended thinking on ${request.next_step} (${thinking.length} chars) — TODO(#13): stream to TUI`);
     }
 
     // =========================================================================
     // Tool call path: execute tools, enqueue HandleToolContinuation, done.
     // =========================================================================
     if (finishReason === "tool_calls" && rawToolCalls?.length) {
-      console.log(`[QueueProcessor] finish_reason=tool_calls — executing tools, will enqueue HandleToolContinuation`);
+      console.debug(`[QueueProcessor] finish_reason=tool_calls — executing tools, will enqueue HandleToolContinuation`);
 
       const toolCalls = parseToolCalls(rawToolCalls);
       if (toolCalls.length === 0) {
@@ -364,7 +364,7 @@ export class QueueProcessor {
         });
       }
 
-      console.log(`[QueueProcessor] Tool execution complete: ${results.length} result(s). Enqueueing HandleToolContinuation.`);
+      console.debug(`[QueueProcessor] Tool execution complete: ${results.length} result(s). Enqueueing HandleToolContinuation.`);
 
       if (this.currentOnEnqueue) {
         this.currentOnEnqueue({
@@ -412,7 +412,7 @@ export class QueueProcessor {
     // =========================================================================
     // Normal stop path
     // =========================================================================
-    console.log(`[QueueProcessor] finish_reason="${finishReason}" — normal stop`);
+    console.debug(`[QueueProcessor] finish_reason="${finishReason}" — normal stop`);
     return this.handleResponseType(request, content ?? "", finishReason);
   }
 
@@ -497,9 +497,9 @@ export class QueueProcessor {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(
         request.system,
         reformatUserPrompt,
-        messages, // existing tool history — gives full context without duplicating the ask
+        messages,
         request.model,
-        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.data.originalNextStep ?? request.next_step}+prose_reformat` },
         this.currentAccounts
       );
 
@@ -554,9 +554,9 @@ export class QueueProcessor {
       const { content: reformatContent, finishReason: reformatReason } = await callLLMRaw(
         request.system,
         reformatUserPrompt,
-        [], // no message history needed — schema is already in the system prompt
+        [],
         request.model,
-        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate },
+        { signal: this.abortController?.signal, onUsageUpdate: this.currentOnUsageUpdate, nextStep: `${request.next_step}+json_reformat` },
         this.currentAccounts
       );
 

package/src/core/state/queue.ts

@@ -1,5 +1,5 @@
 import type { LLMRequest, QueueFailResult } from "../types.js";
-import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS } from "../types.js";
+import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS, LLMNextStep } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -200,6 +200,14 @@ export class QueueState {
     return this.queue.some(r => r.state !== "dlq" && typeof r.data.ceremony_progress === "number" && r.data.ceremony_progress > 0);
   }
 
+  hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queue.some(r =>
+      r.state !== "dlq" &&
+      r.next_step === LLMNextStep.HandleDocumentSegmentation &&
+      r.data.batchId === batchId
+    );
+  }
+
   clear(): number {
     const count = this.queue.filter(r => r.state !== "dlq").length;
     this.queue = this.queue.filter(r => r.state === "dlq");

package/src/core/state-manager.ts

@@ -917,6 +917,10 @@ export class StateManager {
     return this.queueState.hasPendingCeremonies();
   }
 
+  queue_hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queueState.hasPendingDocumentSegments(batchId);
+  }
+
   queue_clear(): number {
     const result = this.queueState.clear();
     this.scheduleSave();
@@ -1050,7 +1054,7 @@ export class StateManager {
   tools_getForPersona(personaId: string, isTUI: boolean): ToolDefinition[] {
     const persona = this.personaState.getById(personaId);
     if (!persona?.tools?.length) {
-      console.log(`[Tools] tools_getForPersona(${personaId}): persona has no assigned tools`);
+      console.debug(`[Tools] tools_getForPersona(${personaId}): persona has no assigned tools`);
       return [];
     }
     const assignedIds = new Set(persona.tools);
@@ -1073,13 +1077,13 @@ export class StateManager {
     if (result.length < assignedIds.size) {
       for (const id of assignedIds) {
         const tool = this.tools.find(t => t.id === id);
-        if (!tool) { console.log(`[Tools] tools_getForPersona: assigned tool id=${id} not found in registry`); continue; }
-        if (!tool.enabled) { console.log(`[Tools] tools_getForPersona: tool "${tool.name}" is disabled`); continue; }
-        if (!enabledProviderIds.has(tool.provider_id)) { console.log(`[Tools] tools_getForPersona: tool "${tool.name}" provider is disabled`); continue; }
-        if (!(tool.runtime === "any" || (tool.runtime === "node" && isTUI))) { console.log(`[Tools] tools_getForPersona: tool "${tool.name}" runtime "${tool.runtime}" not available (isTUI=${isTUI})`); continue; }
+        if (!tool) { console.debug(`[Tools] tools_getForPersona: assigned tool id=${id} not found in registry`); continue; }
+        if (!tool.enabled) { console.debug(`[Tools] tools_getForPersona: tool "${tool.name}" is disabled`); continue; }
+        if (!enabledProviderIds.has(tool.provider_id)) { console.debug(`[Tools] tools_getForPersona: tool "${tool.name}" provider is disabled`); continue; }
+        if (!(tool.runtime === "any" || (tool.runtime === "node" && isTUI))) { console.debug(`[Tools] tools_getForPersona: tool "${tool.name}" runtime "${tool.runtime}" not available (isTUI=${isTUI})`); continue; }
       }
     }
-    console.log(`[Tools] tools_getForPersona(${personaId}): resolved ${result.length}/${assignedIds.size} tools: [${result.map(t => t.name).join(", ")}]`);
+    console.debug(`[Tools] tools_getForPersona(${personaId}): resolved ${result.length}/${assignedIds.size} tools: [${result.map(t => t.name).join(", ")}]`);
     return result;
   }
 

package/src/core/types/entities.ts

@@ -20,6 +20,11 @@ export interface OpenCodeSettings {
   processed_sessions?: Record<string, string>;  // sessionId → ISO timestamp of last import
 }
 
+export interface DocumentSettings {
+  extraction_model?: string;
+  processed_documents?: Record<string, string>;
+}
+
 export interface CeremonyConfig {
   time: string;  // "HH:MM" format (e.g., "09:00")
   last_ceremony?: string;  // ISO timestamp
@@ -117,8 +122,10 @@ export interface HumanSettings {
   backup?: BackupConfig;
   claudeCode?: import("../../integrations/claude-code/types.js").ClaudeCodeSettings;
   cursor?: import("../../integrations/cursor/types.js").CursorSettings;
+  document?: DocumentSettings;
   active_theme?: string;
   custom_themes?: ThemeDefinition[];
+  personaHistory?: import("../../integrations/persona-history/types.js").PersonaHistorySettings;
 }
 
 export interface HumanEntity {
@@ -202,3 +209,11 @@ export type ReservedPersonaName = typeof RESERVED_PERSONA_NAMES[number];
 export function isReservedPersonaName(name: string): boolean {
   return RESERVED_PERSONA_NAMES.includes(name.toLowerCase() as ReservedPersonaName);
 }
+
+// Reserved persona IDs (built-in system personas that cannot be deleted)
+export const RESERVED_PERSONA_IDS = ["ei", "emmet"] as const;
+export type ReservedPersonaId = typeof RESERVED_PERSONA_IDS[number];
+
+export function isReservedPersonaId(id: string): boolean {
+  return (RESERVED_PERSONA_IDS as readonly string[]).includes(id);
+}

package/src/core/types/enums.ts

@@ -52,6 +52,7 @@ export enum LLMNextStep {
   HandlePersonaPreview = "handlePersonaPreview",
   HandleTopicValidate = "handleTopicValidate",
   HandleReflectionCritic = "handleReflectionCritic",
+  HandleDocumentSegmentation = "handleDocumentSegmentation",
 }
 
 export enum ProviderType {

package/src/core/types/integrations.ts

@@ -75,6 +75,8 @@ export interface QueueStatus {
   current_operation?: string;
   /** True when the embedding service failed and topic/person matching fell back to recent items. */
   embedding_warning?: boolean;
+  pending_documents?: Array<{ batchId: string; filename: string; count: number }>;
+  extracting_documents?: string[];
 }
 
 export interface EiError {