ei-tui 0.9.4 → 1.0.0

package/src/core/persona-manager.ts

@@ -1,6 +1,7 @@
 import {
   RESERVED_PERSONA_NAMES,
   isReservedPersonaName,
+  isReservedPersonaId,
   type PersonaSummary,
   type PersonaEntity,
   type PersonaCreationInput,
@@ -110,6 +111,9 @@ export async function deletePersona(
   personaId: string,
   _deleteHumanData: boolean
 ): Promise<boolean> {
+  if (isReservedPersonaId(personaId)) {
+    throw new Error(`Cannot delete reserved persona "${personaId}". Use archive instead.`);
+  }
   const persona = sm.persona_getById(personaId);
   if (!persona) return false;
   sm.persona_delete(personaId);

package/src/core/processor.ts

@@ -39,7 +39,9 @@ import { ContextStatus as ContextStatusEnum, RoomMode } from "./types.js";
 import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
 import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
+import { EMMETT_PERSONA_DEFINITION } from "../templates/emmett.js";
 import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueReflectionDrain, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction, queueTargetedPersonUpdate, queueTargetedTopicUpdate } from "./orchestrators/index.js";
+import { finishDocumentBatch } from "./handlers/document-segmentation.js";
 import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
 import { DEFAULT_SEED_TRAITS } from "./constants/seed-traits.js";
 
@@ -132,6 +134,8 @@ import {
   markAllRoomMessagesRead,
 } from "./room-manager.js";
 import type { RoomCreationInput, RoomEntity, RoomMessage, RoomSummary } from "./types.js";
+import { previewUnsource as _previewUnsource } from "../integrations/document/unsource.js";
+import type { UnsourcePreview, UnsourceResult } from "../integrations/document/unsource.js";
 
 const DEFAULT_LOOP_INTERVAL_MS = 100;
 const DEFAULT_OPENCODE_POLLING_MS = 60000;
@@ -282,6 +286,46 @@ export class Processor {
     this.interface.onMessageAdded?.(eiEntity.id);
   }
 
+  private bootstrapEmmett(): void {
+    const existing = this.stateManager.persona_getById("emmet");
+    if (existing) {
+      if (existing.is_archived) {
+        this.stateManager.persona_unarchive("emmet");
+      }
+      return;
+    }
+    const readMemoryTool = this.stateManager.tools_getByName("read_memory");
+    const emmettEntity: PersonaEntity = {
+      ...EMMETT_PERSONA_DEFINITION,
+      id: "emmet",
+      display_name: "Emmett",
+      last_updated: new Date().toISOString(),
+      tools: readMemoryTool ? [readMemoryTool.id] : [],
+    };
+    this.stateManager.persona_add(emmettEntity);
+    this.interface.onPersonaAdded?.();
+  }
+
+  async importDocument(content: string, filename: string): Promise<import("../integrations/document/types.js").DocumentImportResult> {
+    this.bootstrapEmmett();
+    const { importDocument } = await import("../integrations/document/importer.js");
+    return importDocument({
+      stateManager: this.stateManager,
+      interface: this.interface,
+      content,
+      filename,
+    });
+  }
+
+  getUnsourcePreview(sourceTag: string): UnsourcePreview {
+    return _previewUnsource(sourceTag, this.stateManager);
+  }
+
+  async executeUnsource(preview: UnsourcePreview): Promise<UnsourceResult> {
+    const { executeUnsource } = await import("../integrations/document/unsource.js");
+    return executeUnsource(preview, this.stateManager);
+  }
+
   /**
    * Seed built-in tool providers and tools if they don't exist yet.
    * Called on every startup (after state load/restore) — safe to call repeatedly.
@@ -1168,6 +1212,15 @@ const toolNextSteps = new Set([
       await this.checkAndSyncCursor(human, now);
     }
 
+    if (
+      this.isTUI &&
+      human.settings?.personaHistory?.integration &&
+      !human.settings.personaHistory.complete &&
+      this.stateManager.queue_length() === 0
+    ) {
+      await this.checkAndSyncPersonaHistory(human);
+    }
+
     if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
       if (human.settings?.sync && remoteSync.isConfigured()) {
         const state = this.stateManager.getStorageState();
@@ -1180,7 +1233,7 @@ const toolNextSteps = new Set([
     }
 
     for (const persona of this.stateManager.persona_getAll()) {
-      if (persona.is_paused || persona.is_archived) continue;
+      if (persona.is_paused || persona.is_archived || persona.is_static) continue;
 
       const defaultHeartbeatMs = this.stateManager.getHuman().settings?.default_heartbeat_ms ?? 1800000;
       const heartbeatDelay = persona.heartbeat_delay_ms ?? defaultHeartbeatMs;
@@ -1408,6 +1461,32 @@ const toolNextSteps = new Set([
       });
   }
 
+  private personaHistoryImportInProgress = false;
+
+  private async checkAndSyncPersonaHistory(_human: HumanEntity): Promise<void> {
+    if (this.personaHistoryImportInProgress) return;
+
+    this.personaHistoryImportInProgress = true;
+    import("../integrations/persona-history/importer.js")
+      .then(({ importPersonaHistory }) =>
+        importPersonaHistory({ stateManager: this.stateManager })
+      )
+      .then((result) => {
+        if (result.scansQueued > 0) {
+          console.log(
+            `[Processor] PersonaHistory: ${result.scansQueued} scans queued` +
+            (result.complete ? " — import complete" : "")
+          );
+        }
+      })
+      .catch((err) => {
+        console.warn(`[Processor] PersonaHistory sync failed:`, err);
+      })
+      .finally(() => {
+        this.personaHistoryImportInProgress = false;
+      });
+  }
+
   private augmentRoomRequest(request: LLMRequest): LLMRequest {
     if (request.next_step !== LLMNextStep.HandleRoomResponse) return request;
 
@@ -1675,6 +1754,16 @@ const toolNextSteps = new Set([
       if (typeof response.request.data.ceremony_progress === "number") {
         handleCeremonyProgress(this.stateManager, response.request.data.ceremony_progress);
       }
+
+      if (response.request.next_step === LLMNextStep.HandleDocumentSegmentation) {
+        const batchId = response.request.data.batchId as string;
+        const filename = response.request.data.filename as string;
+        if (batchId && !this.stateManager.queue_hasPendingDocumentSegments(batchId)) {
+          finishDocumentBatch(batchId, filename, this.stateManager);
+          this.interface.onMessageAdded?.("emmet");
+          this.interface.onHumanUpdated?.();
+        }
+      }
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : String(err);
       const result = this.stateManager.queue_fail(response.request.id, errorMsg);

package/src/core/queue-manager.ts

@@ -18,6 +18,39 @@ export async function resumeQueue(sm: StateManager): Promise<void> {
 }
 
 export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
+  const activeItems = sm.queue_getAllActiveItems();
+  const segmentationItems = activeItems.filter(
+    r => r.next_step === LLMNextStep.HandleDocumentSegmentation
+  );
+
+  const batchMap = new Map<string, { filename: string; count: number }>();
+  for (const item of segmentationItems) {
+    const { batchId, filename } = item.data as { batchId: string; filename: string };
+    if (!batchId || !filename) continue;
+    const existing = batchMap.get(batchId);
+    if (existing) {
+      existing.count++;
+    } else {
+      batchMap.set(batchId, { filename, count: 1 });
+    }
+  }
+
+  const pending_documents = batchMap.size > 0
+    ? Array.from(batchMap.entries()).map(([batchId, { filename, count }]) => ({ batchId, filename, count }))
+    : undefined;
+
+  const extractingSet = new Set<string>();
+  for (const item of activeItems) {
+    const sources = item.data.sources as string[] | undefined;
+    if (!Array.isArray(sources)) continue;
+    for (const s of sources) {
+      if (typeof s === "string" && s.startsWith("import:document:")) {
+        extractingSet.add(s.slice("import:document:".length));
+      }
+    }
+  }
+  const extracting_documents = extractingSet.size > 0 ? Array.from(extractingSet) : undefined;
+
   return {
     state: sm.queue_isPaused()
       ? "paused"
@@ -27,6 +60,8 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
     pending_count: sm.queue_length(),
     dlq_count: sm.queue_dlqLength(),
     embedding_warning: sm.embedding_getWarning() || undefined,
+    pending_documents,
+    extracting_documents,
   };
 }
 

package/src/core/state/queue.ts

@@ -1,5 +1,5 @@
 import type { LLMRequest, QueueFailResult } from "../types.js";
-import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS } from "../types.js";
+import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS, LLMNextStep } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -200,6 +200,14 @@ export class QueueState {
     return this.queue.some(r => r.state !== "dlq" && typeof r.data.ceremony_progress === "number" && r.data.ceremony_progress > 0);
   }
 
+  hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queue.some(r =>
+      r.state !== "dlq" &&
+      r.next_step === LLMNextStep.HandleDocumentSegmentation &&
+      r.data.batchId === batchId
+    );
+  }
+
   clear(): number {
     const count = this.queue.filter(r => r.state !== "dlq").length;
     this.queue = this.queue.filter(r => r.state === "dlq");

package/src/core/state-manager.ts

@@ -917,6 +917,10 @@ export class StateManager {
     return this.queueState.hasPendingCeremonies();
   }
 
+  queue_hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queueState.hasPendingDocumentSegments(batchId);
+  }
+
   queue_clear(): number {
     const result = this.queueState.clear();
     this.scheduleSave();

package/src/core/types/entities.ts

@@ -20,6 +20,11 @@ export interface OpenCodeSettings {
   processed_sessions?: Record<string, string>;  // sessionId → ISO timestamp of last import
 }
 
+export interface DocumentSettings {
+  extraction_model?: string;
+  processed_documents?: Record<string, string>;
+}
+
 export interface CeremonyConfig {
   time: string;  // "HH:MM" format (e.g., "09:00")
   last_ceremony?: string;  // ISO timestamp
@@ -117,8 +122,10 @@ export interface HumanSettings {
   backup?: BackupConfig;
   claudeCode?: import("../../integrations/claude-code/types.js").ClaudeCodeSettings;
   cursor?: import("../../integrations/cursor/types.js").CursorSettings;
+  document?: DocumentSettings;
   active_theme?: string;
   custom_themes?: ThemeDefinition[];
+  personaHistory?: import("../../integrations/persona-history/types.js").PersonaHistorySettings;
 }
 
 export interface HumanEntity {
@@ -202,3 +209,11 @@ export type ReservedPersonaName = typeof RESERVED_PERSONA_NAMES[number];
 export function isReservedPersonaName(name: string): boolean {
   return RESERVED_PERSONA_NAMES.includes(name.toLowerCase() as ReservedPersonaName);
 }
+
+// Reserved persona IDs (built-in system personas that cannot be deleted)
+export const RESERVED_PERSONA_IDS = ["ei", "emmet"] as const;
+export type ReservedPersonaId = typeof RESERVED_PERSONA_IDS[number];
+
+export function isReservedPersonaId(id: string): boolean {
+  return (RESERVED_PERSONA_IDS as readonly string[]).includes(id);
+}

package/src/core/types/enums.ts

@@ -52,6 +52,7 @@ export enum LLMNextStep {
   HandlePersonaPreview = "handlePersonaPreview",
   HandleTopicValidate = "handleTopicValidate",
   HandleReflectionCritic = "handleReflectionCritic",
+  HandleDocumentSegmentation = "handleDocumentSegmentation",
 }
 
 export enum ProviderType {

package/src/core/types/integrations.ts

@@ -75,6 +75,8 @@ export interface QueueStatus {
   current_operation?: string;
   /** True when the embedding service failed and topic/person matching fell back to recent items. */
   embedding_warning?: boolean;
+  pending_documents?: Array<{ batchId: string; filename: string; count: number }>;
+  extracting_documents?: string[];
 }
 
 export interface EiError {

package/src/core/types/llm.ts

@@ -27,6 +27,15 @@ export interface Message {
 
   external?: boolean;          // Set by integration importers (OpenCode, Cursor, Claude Code); invisible to LLM context
 
+  /**
+   * Integration source tag. Set ONLY on external: true messages by importers (document, Slack, etc.)
+   * to identify which external source this synthetic message came from.
+   * Format: "import:document:filename" | "slack:channelId" | etc.
+   * Enables quote provenance tracing: quote.message_id → message.source_tag → original source.
+   * Never set on conversational messages.
+   */
+  source_tag?: string;
+
 }
 
 export interface ChatMessage {

package/src/integrations/document/chunker.ts

@@ -0,0 +1,88 @@
+import { expandToWordBoundaries } from "../../core/handlers/human-matching.js";
+
+const DEFAULT_CHUNK_CHARS = 6000;
+const DEFAULT_OVERLAP_CHARS = 300;
+
+const MARKDOWN_SEPARATORS = ["\n## ", "\n### ", "\n#### ", "\n\n", "\n", ". ", " ", ""];
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+
+function splitOnSeparator(text: string, separator: string): string[] {
+  if (separator === "") {
+    return text.split("");
+  }
+  return text.split(separator);
+}
+
+function mergeChunks(pieces: string[], separator: string, chunkSize: number): string[] {
+  const merged: string[] = [];
+  let current = "";
+
+  for (const piece of pieces) {
+    const candidate = current ? current + separator + piece : piece;
+    if (candidate.length <= chunkSize) {
+      current = candidate;
+    } else {
+      if (current) merged.push(current);
+      current = piece.length <= chunkSize ? piece : piece;
+    }
+  }
+  if (current) merged.push(current);
+  return merged;
+}
+
+function recursiveSplit(
+  text: string,
+  separators: string[],
+  chunkSize: number
+): string[] {
+  if (text.length <= chunkSize) {
+    return [text];
+  }
+
+  const [separator, ...remainingSeparators] = separators;
+
+  if (separator === undefined) {
+    return [text];
+  }
+
+  const pieces = splitOnSeparator(text, separator);
+  const result: string[] = [];
+
+  for (const piece of pieces) {
+    if (piece.length <= chunkSize) {
+      result.push(piece);
+    } else if (remainingSeparators.length > 0) {
+      result.push(...recursiveSplit(piece, remainingSeparators, chunkSize));
+    } else {
+      result.push(piece);
+    }
+  }
+
+  return mergeChunks(result, separator, chunkSize);
+}
+
+function applyOverlap(chunks: string[], overlapChars: number): string[] {
+  if (overlapChars <= 0 || chunks.length <= 1) return chunks;
+
+  return chunks.map((chunk, i) => {
+    if (i === 0) return chunk;
+    const prev = chunks[i - 1];
+    const rawStart = Math.max(0, prev.length - overlapChars);
+    const { start } = expandToWordBoundaries(prev, rawStart, rawStart);
+    const prefix = prev.slice(start);
+    return prefix + chunk;
+  });
+}
+
+export function recursiveCharacterSplit(
+  text: string,
+  options?: { chunkSize?: number; overlap?: number; isMarkdown?: boolean }
+): string[] {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_CHARS;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP_CHARS;
+  const separators = options?.isMarkdown ? MARKDOWN_SEPARATORS : DEFAULT_SEPARATORS;
+
+  const rawChunks = recursiveSplit(text, separators, chunkSize);
+  const nonEmpty = rawChunks.filter(c => c.trim().length > 0);
+  return applyOverlap(nonEmpty, overlap);
+}

package/src/integrations/document/importer.ts

@@ -0,0 +1,82 @@
+import type { PersonaEntity } from "../../core/types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep } from "../../core/types.js";
+import { EMMETT_PERSONA_DEFINITION } from "../../templates/emmett.js";
+import { recursiveCharacterSplit } from "./chunker.js";
+import type { DocumentImportOptions, DocumentImportResult } from "./types.js";
+
+const SEGMENTATION_SYSTEM_PROMPT = `You are a document segmentation assistant. Your job is to identify natural conceptual boundaries in document content and split it into coherent segments suitable for knowledge extraction. Each segment should be a self-contained unit of information.`;
+
+const SEGMENTATION_USER_TEMPLATE = `Split the following document content into conceptual segments. Return a JSON array of strings, where each string is one segment. Preserve all original text — do not summarize or paraphrase. Identify boundaries at topic shifts, section changes, or logical breaks.
+
+---
+
+{content}`;
+
+export async function importDocument(options: DocumentImportOptions): Promise<DocumentImportResult> {
+  const { stateManager, interface: eiInterface, content: rawContent, filename, signal } = options;
+
+  const isMarkdown = filename.toLowerCase().endsWith(".md") || filename.toLowerCase().endsWith(".markdown");
+
+  const result: DocumentImportResult = {
+    chunksQueued: 0,
+    documentName: filename,
+  };
+
+  let emmett = stateManager.persona_getById("emmet");
+  if (emmett?.is_archived) {
+    stateManager.persona_unarchive("emmet");
+    emmett = stateManager.persona_getById("emmet")!;
+  }
+  if (!emmett) {
+    const emmettEntity: PersonaEntity = {
+      ...EMMETT_PERSONA_DEFINITION,
+      id: "emmet",
+      display_name: "Emmett",
+      last_updated: new Date().toISOString(),
+    };
+    stateManager.persona_add(emmettEntity);
+    eiInterface.onPersonaAdded?.();
+    emmett = stateManager.persona_getById("emmet")!;
+  }
+
+  const sourceTag = `import:document:${filename}`;
+  const existingMsgs = stateManager.messages_get("emmet");
+  const staleIds = existingMsgs
+    .filter(m => m.external === true && m.source_tag === sourceTag)
+    .map(m => m.id);
+  if (staleIds.length > 0) {
+    stateManager.messages_remove("emmet", staleIds);
+  }
+
+  if (signal?.aborted) return result;
+
+  const preChunks = recursiveCharacterSplit(rawContent, { isMarkdown });
+
+  if (preChunks.length === 0) return result;
+
+  const batchId = crypto.randomUUID();
+  const docSettings = stateManager.getHuman().settings?.document;
+  const model = docSettings?.extraction_model ?? stateManager.getHuman().settings?.default_model;
+
+  for (let i = 0; i < preChunks.length; i++) {
+    const chunk = preChunks[i];
+    stateManager.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: SEGMENTATION_SYSTEM_PROMPT,
+      user: SEGMENTATION_USER_TEMPLATE.replace("{content}", chunk),
+      next_step: LLMNextStep.HandleDocumentSegmentation,
+      model,
+      data: {
+        batchId,
+        filename,
+        chunkIndex: i,
+        originalContent: chunk,
+      },
+    });
+  }
+
+  result.chunksQueued = preChunks.length;
+  result.batchId = batchId;
+  return result;
+}

package/src/integrations/document/index.ts

@@ -0,0 +1,2 @@
+export { importDocument } from "./importer.js";
+export type { DocumentImportOptions, DocumentImportResult } from "./types.js";

package/src/integrations/document/invoice.ts

@@ -0,0 +1,63 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { UnsourcePreview, UnsourceResult } from "./unsource.js";
+
+/**
+ * Pure function — no filesystem access. Safe to call in any environment.
+ * Returns the invoice as a markdown string.
+ */
+export function generateInvoiceMarkdown(preview: UnsourcePreview, result: UnsourceResult): string {
+  const timestamp = new Date().toISOString();
+
+  const deletedCount = result.deleted.facts + result.deleted.topics + result.deleted.people;
+  const strippedCount = result.stripped.facts + result.stripped.topics + result.stripped.people;
+
+  const lines: string[] = [
+    `# Unsource: ${preview.sourceTag}`,
+    `Run at: ${timestamp}`,
+    "",
+    `## Deleted (${deletedCount} items, ${result.deleted.quotes} quotes)`,
+  ];
+
+  for (const f of preview.toDelete.facts) lines.push(`- [Fact] ${f.name}`);
+  for (const t of preview.toDelete.topics) lines.push(`- [Topic] ${t.name}`);
+  for (const p of preview.toDelete.people) lines.push(`- [Person] ${p.name}`);
+  for (const q of preview.toDelete.quotes) {
+    const excerpt = q.text.length > 80 ? `${q.text.slice(0, 80)}...` : q.text;
+    lines.push(`- [Quote] "${excerpt}"`);
+  }
+
+  if (
+    preview.toStrip.facts.length > 0 ||
+    preview.toStrip.topics.length > 0 ||
+    preview.toStrip.people.length > 0
+  ) {
+    lines.push("");
+    lines.push(`## Retained — shared with other sources (${strippedCount} items)`);
+    lines.push(`Source removed from these items. They had additional sources or non-Emmett personas.`);
+    lines.push("");
+    for (const f of preview.toStrip.facts) lines.push(`- [Fact] ${f.name}`);
+    for (const t of preview.toStrip.topics) lines.push(`- [Topic] ${t.name}`);
+    for (const p of preview.toStrip.people) lines.push(`- [Person] ${p.name}`);
+    lines.push("");
+    lines.push(`Run \`/me topics\` or \`/me people\` to review or delete retained items manually.`);
+  }
+
+  return lines.join("\n") + "\n";
+}
+
+export async function writeUnsourceInvoice(
+  preview: UnsourcePreview,
+  result: UnsourceResult,
+  dataPath: string
+): Promise<string> {
+  const timestamp = new Date().toISOString();
+  const sanitizedTag = preview.sourceTag.replace(/[^a-zA-Z0-9._-]/g, "_");
+  const filename = `${timestamp.replace(/[:.]/g, "-")}-${sanitizedTag}.md`;
+  const dir = join(dataPath, "unsourced");
+  const filePath = join(dir, filename);
+
+  await mkdir(dir, { recursive: true });
+  await writeFile(filePath, generateInvoiceMarkdown(preview, result), "utf8");
+  return filePath;
+}

package/src/integrations/document/types.ts

@@ -0,0 +1,16 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { Ei_Interface } from "../../core/types.js";
+
+export interface DocumentImportOptions {
+  stateManager: StateManager;
+  interface: Ei_Interface;
+  content: string;
+  filename: string;
+  signal?: AbortSignal;
+}
+
+export interface DocumentImportResult {
+  chunksQueued: number;
+  documentName: string;
+  batchId?: string;
+}