ei-tui 0.9.4 → 1.0.1

package/src/core/types/llm.ts

@@ -27,6 +27,15 @@ export interface Message {
 
   external?: boolean;          // Set by integration importers (OpenCode, Cursor, Claude Code); invisible to LLM context
 
+  /**
+   * Integration source tag. Set ONLY on external: true messages by importers (document, Slack, etc.)
+   * to identify which external source this synthetic message came from.
+   * Format: "import:document:filename" | "slack:channelId" | etc.
+   * Enables quote provenance tracing: quote.message_id → message.source_tag → original source.
+   * Never set on conversational messages.
+   */
+  source_tag?: string;
+
 }
 
 export interface ChatMessage {

package/src/integrations/document/chunker.ts

@@ -0,0 +1,88 @@
+import { expandToWordBoundaries } from "../../core/handlers/human-matching.js";
+
+const DEFAULT_CHUNK_CHARS = 6000;
+const DEFAULT_OVERLAP_CHARS = 300;
+
+const MARKDOWN_SEPARATORS = ["\n## ", "\n### ", "\n#### ", "\n\n", "\n", ". ", " ", ""];
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+
+function splitOnSeparator(text: string, separator: string): string[] {
+  if (separator === "") {
+    return text.split("");
+  }
+  return text.split(separator);
+}
+
+function mergeChunks(pieces: string[], separator: string, chunkSize: number): string[] {
+  const merged: string[] = [];
+  let current = "";
+
+  for (const piece of pieces) {
+    const candidate = current ? current + separator + piece : piece;
+    if (candidate.length <= chunkSize) {
+      current = candidate;
+    } else {
+      if (current) merged.push(current);
+      current = piece.length <= chunkSize ? piece : piece;
+    }
+  }
+  if (current) merged.push(current);
+  return merged;
+}
+
+function recursiveSplit(
+  text: string,
+  separators: string[],
+  chunkSize: number
+): string[] {
+  if (text.length <= chunkSize) {
+    return [text];
+  }
+
+  const [separator, ...remainingSeparators] = separators;
+
+  if (separator === undefined) {
+    return [text];
+  }
+
+  const pieces = splitOnSeparator(text, separator);
+  const result: string[] = [];
+
+  for (const piece of pieces) {
+    if (piece.length <= chunkSize) {
+      result.push(piece);
+    } else if (remainingSeparators.length > 0) {
+      result.push(...recursiveSplit(piece, remainingSeparators, chunkSize));
+    } else {
+      result.push(piece);
+    }
+  }
+
+  return mergeChunks(result, separator, chunkSize);
+}
+
+function applyOverlap(chunks: string[], overlapChars: number): string[] {
+  if (overlapChars <= 0 || chunks.length <= 1) return chunks;
+
+  return chunks.map((chunk, i) => {
+    if (i === 0) return chunk;
+    const prev = chunks[i - 1];
+    const rawStart = Math.max(0, prev.length - overlapChars);
+    const { start } = expandToWordBoundaries(prev, rawStart, rawStart);
+    const prefix = prev.slice(start);
+    return prefix + chunk;
+  });
+}
+
+export function recursiveCharacterSplit(
+  text: string,
+  options?: { chunkSize?: number; overlap?: number; isMarkdown?: boolean }
+): string[] {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_CHARS;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP_CHARS;
+  const separators = options?.isMarkdown ? MARKDOWN_SEPARATORS : DEFAULT_SEPARATORS;
+
+  const rawChunks = recursiveSplit(text, separators, chunkSize);
+  const nonEmpty = rawChunks.filter(c => c.trim().length > 0);
+  return applyOverlap(nonEmpty, overlap);
+}

package/src/integrations/document/importer.ts

@@ -0,0 +1,82 @@
+import type { PersonaEntity } from "../../core/types.js";
+import { LLMRequestType, LLMPriority, LLMNextStep } from "../../core/types.js";
+import { EMMETT_PERSONA_DEFINITION } from "../../templates/emmett.js";
+import { recursiveCharacterSplit } from "./chunker.js";
+import type { DocumentImportOptions, DocumentImportResult } from "./types.js";
+
+const SEGMENTATION_SYSTEM_PROMPT = `You are a document segmentation assistant. Your job is to identify natural conceptual boundaries in document content and split it into coherent segments suitable for knowledge extraction. Each segment should be a self-contained unit of information.`;
+
+const SEGMENTATION_USER_TEMPLATE = `Split the following document content into conceptual segments. Return a JSON array of strings, where each string is one segment. Preserve all original text — do not summarize or paraphrase. Identify boundaries at topic shifts, section changes, or logical breaks.
+
+---
+
+{content}`;
+
+export async function importDocument(options: DocumentImportOptions): Promise<DocumentImportResult> {
+  const { stateManager, interface: eiInterface, content: rawContent, filename, signal } = options;
+
+  const isMarkdown = filename.toLowerCase().endsWith(".md") || filename.toLowerCase().endsWith(".markdown");
+
+  const result: DocumentImportResult = {
+    chunksQueued: 0,
+    documentName: filename,
+  };
+
+  let emmett = stateManager.persona_getById("emmet");
+  if (emmett?.is_archived) {
+    stateManager.persona_unarchive("emmet");
+    emmett = stateManager.persona_getById("emmet")!;
+  }
+  if (!emmett) {
+    const emmettEntity: PersonaEntity = {
+      ...EMMETT_PERSONA_DEFINITION,
+      id: "emmet",
+      display_name: "Emmett",
+      last_updated: new Date().toISOString(),
+    };
+    stateManager.persona_add(emmettEntity);
+    eiInterface.onPersonaAdded?.();
+    emmett = stateManager.persona_getById("emmet")!;
+  }
+
+  const sourceTag = `import:document:${filename}`;
+  const existingMsgs = stateManager.messages_get("emmet");
+  const staleIds = existingMsgs
+    .filter(m => m.external === true && m.source_tag === sourceTag)
+    .map(m => m.id);
+  if (staleIds.length > 0) {
+    stateManager.messages_remove("emmet", staleIds);
+  }
+
+  if (signal?.aborted) return result;
+
+  const preChunks = recursiveCharacterSplit(rawContent, { isMarkdown });
+
+  if (preChunks.length === 0) return result;
+
+  const batchId = crypto.randomUUID();
+  const docSettings = stateManager.getHuman().settings?.document;
+  const model = docSettings?.extraction_model ?? stateManager.getHuman().settings?.default_model;
+
+  for (let i = 0; i < preChunks.length; i++) {
+    const chunk = preChunks[i];
+    stateManager.queue_enqueue({
+      type: LLMRequestType.JSON,
+      priority: LLMPriority.Low,
+      system: SEGMENTATION_SYSTEM_PROMPT,
+      user: SEGMENTATION_USER_TEMPLATE.replace("{content}", chunk),
+      next_step: LLMNextStep.HandleDocumentSegmentation,
+      model,
+      data: {
+        batchId,
+        filename,
+        chunkIndex: i,
+        originalContent: chunk,
+      },
+    });
+  }
+
+  result.chunksQueued = preChunks.length;
+  result.batchId = batchId;
+  return result;
+}

package/src/integrations/document/index.ts

@@ -0,0 +1,2 @@
+export { importDocument } from "./importer.js";
+export type { DocumentImportOptions, DocumentImportResult } from "./types.js";

package/src/integrations/document/invoice.ts

@@ -0,0 +1,63 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { UnsourcePreview, UnsourceResult } from "./unsource.js";
+
+/**
+ * Pure function — no filesystem access. Safe to call in any environment.
+ * Returns the invoice as a markdown string.
+ */
+export function generateInvoiceMarkdown(preview: UnsourcePreview, result: UnsourceResult): string {
+  const timestamp = new Date().toISOString();
+
+  const deletedCount = result.deleted.facts + result.deleted.topics + result.deleted.people;
+  const strippedCount = result.stripped.facts + result.stripped.topics + result.stripped.people;
+
+  const lines: string[] = [
+    `# Unsource: ${preview.sourceTag}`,
+    `Run at: ${timestamp}`,
+    "",
+    `## Deleted (${deletedCount} items, ${result.deleted.quotes} quotes)`,
+  ];
+
+  for (const f of preview.toDelete.facts) lines.push(`- [Fact] ${f.name}`);
+  for (const t of preview.toDelete.topics) lines.push(`- [Topic] ${t.name}`);
+  for (const p of preview.toDelete.people) lines.push(`- [Person] ${p.name}`);
+  for (const q of preview.toDelete.quotes) {
+    const excerpt = q.text.length > 80 ? `${q.text.slice(0, 80)}...` : q.text;
+    lines.push(`- [Quote] "${excerpt}"`);
+  }
+
+  if (
+    preview.toStrip.facts.length > 0 ||
+    preview.toStrip.topics.length > 0 ||
+    preview.toStrip.people.length > 0
+  ) {
+    lines.push("");
+    lines.push(`## Retained — shared with other sources (${strippedCount} items)`);
+    lines.push(`Source removed from these items. They had additional sources or non-Emmett personas.`);
+    lines.push("");
+    for (const f of preview.toStrip.facts) lines.push(`- [Fact] ${f.name}`);
+    for (const t of preview.toStrip.topics) lines.push(`- [Topic] ${t.name}`);
+    for (const p of preview.toStrip.people) lines.push(`- [Person] ${p.name}`);
+    lines.push("");
+    lines.push(`Run \`/me topics\` or \`/me people\` to review or delete retained items manually.`);
+  }
+
+  return lines.join("\n") + "\n";
+}
+
+export async function writeUnsourceInvoice(
+  preview: UnsourcePreview,
+  result: UnsourceResult,
+  dataPath: string
+): Promise<string> {
+  const timestamp = new Date().toISOString();
+  const sanitizedTag = preview.sourceTag.replace(/[^a-zA-Z0-9._-]/g, "_");
+  const filename = `${timestamp.replace(/[:.]/g, "-")}-${sanitizedTag}.md`;
+  const dir = join(dataPath, "unsourced");
+  const filePath = join(dir, filename);
+
+  await mkdir(dir, { recursive: true });
+  await writeFile(filePath, generateInvoiceMarkdown(preview, result), "utf8");
+  return filePath;
+}

package/src/integrations/document/types.ts

@@ -0,0 +1,16 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { Ei_Interface } from "../../core/types.js";
+
+export interface DocumentImportOptions {
+  stateManager: StateManager;
+  interface: Ei_Interface;
+  content: string;
+  filename: string;
+  signal?: AbortSignal;
+}
+
+export interface DocumentImportResult {
+  chunksQueued: number;
+  documentName: string;
+  batchId?: string;
+}

package/src/integrations/document/unsource.ts

@@ -0,0 +1,164 @@
+import type { StateManager } from "../../core/state-manager.js";
+
+export interface UnsourcePreview {
+  sourceTag: string;
+  toDelete: {
+    facts: Array<{ id: string; name: string }>;
+    topics: Array<{ id: string; name: string }>;
+    people: Array<{ id: string; name: string }>;
+    quotes: Array<{ id: string; text: string }>;
+  };
+  toStrip: {
+    facts: Array<{ id: string; name: string }>;
+    topics: Array<{ id: string; name: string }>;
+    people: Array<{ id: string; name: string }>;
+  };
+}
+
+export interface UnsourceResult {
+  sourceTag: string;
+  deleted: { facts: number; topics: number; people: number; quotes: number };
+  stripped: { facts: number; topics: number; people: number };
+}
+
+export function previewUnsource(sourceTag: string, stateManager: StateManager): UnsourcePreview {
+  const human = stateManager.getHuman();
+
+  const preview: UnsourcePreview = {
+    sourceTag,
+    toDelete: { facts: [], topics: [], people: [], quotes: [] },
+    toStrip: { facts: [], topics: [], people: [] },
+  };
+
+  for (const fact of human.facts) {
+    if (!fact.sources?.includes(sourceTag)) continue;
+    const remainingSources = fact.sources.filter(s => s !== sourceTag);
+    const nonEmmettPersonas = (fact.interested_personas ?? []).filter(id => id !== "emmet");
+    if (remainingSources.length === 0 && nonEmmettPersonas.length === 0) {
+      preview.toDelete.facts.push({ id: fact.id, name: fact.name });
+    } else {
+      preview.toStrip.facts.push({ id: fact.id, name: fact.name });
+    }
+  }
+
+  for (const topic of human.topics) {
+    if (!topic.sources?.includes(sourceTag)) continue;
+    const remainingSources = topic.sources.filter(s => s !== sourceTag);
+    const nonEmmettPersonas = (topic.interested_personas ?? []).filter(id => id !== "emmet");
+    if (remainingSources.length === 0 && nonEmmettPersonas.length === 0) {
+      preview.toDelete.topics.push({ id: topic.id, name: topic.name });
+    } else {
+      preview.toStrip.topics.push({ id: topic.id, name: topic.name });
+    }
+  }
+
+  for (const person of human.people) {
+    if (!person.sources?.includes(sourceTag)) continue;
+    const remainingSources = person.sources.filter(s => s !== sourceTag);
+    const nonEmmettPersonas = (person.interested_personas ?? []).filter(id => id !== "emmet");
+    if (remainingSources.length === 0 && nonEmmettPersonas.length === 0) {
+      preview.toDelete.people.push({ id: person.id, name: person.name });
+    } else {
+      preview.toStrip.people.push({ id: person.id, name: person.name });
+    }
+  }
+
+  const emmettMessages = stateManager.messages_get("emmet");
+  const sourceMessageIds = new Set(
+    emmettMessages
+      .filter(m => m.source_tag === sourceTag)
+      .map(m => m.id)
+  );
+
+  for (const quote of human.quotes) {
+    if (quote.message_id && sourceMessageIds.has(quote.message_id)) {
+      preview.toDelete.quotes.push({ id: quote.id, text: quote.text });
+    }
+  }
+
+  return preview;
+}
+
+export async function executeUnsource(
+  preview: UnsourcePreview,
+  stateManager: StateManager
+): Promise<UnsourceResult> {
+  const result: UnsourceResult = {
+    sourceTag: preview.sourceTag,
+    deleted: { facts: 0, topics: 0, people: 0, quotes: 0 },
+    stripped: { facts: 0, topics: 0, people: 0 },
+  };
+
+  for (const q of preview.toDelete.quotes) {
+    stateManager.human_quote_remove(q.id);
+    result.deleted.quotes++;
+  }
+
+  for (const f of preview.toDelete.facts) {
+    stateManager.human_fact_remove(f.id);
+    result.deleted.facts++;
+  }
+
+  for (const t of preview.toDelete.topics) {
+    stateManager.human_topic_remove(t.id);
+    result.deleted.topics++;
+  }
+
+  for (const p of preview.toDelete.people) {
+    stateManager.human_person_remove(p.id);
+    result.deleted.people++;
+  }
+
+  if (
+    preview.toStrip.facts.length > 0 ||
+    preview.toStrip.topics.length > 0 ||
+    preview.toStrip.people.length > 0
+  ) {
+    const human = stateManager.getHuman();
+    const stripIds = new Set([
+      ...preview.toStrip.facts.map(f => f.id),
+      ...preview.toStrip.topics.map(t => t.id),
+      ...preview.toStrip.people.map(p => p.id),
+    ]);
+
+    for (const fact of human.facts) {
+      if (stripIds.has(fact.id) && fact.sources) {
+        fact.sources = fact.sources.filter(s => s !== preview.sourceTag);
+        result.stripped.facts++;
+      }
+    }
+    for (const topic of human.topics) {
+      if (stripIds.has(topic.id) && topic.sources) {
+        topic.sources = topic.sources.filter(s => s !== preview.sourceTag);
+        result.stripped.topics++;
+      }
+    }
+    for (const person of human.people) {
+      if (stripIds.has(person.id) && person.sources) {
+        person.sources = person.sources.filter(s => s !== preview.sourceTag);
+        result.stripped.people++;
+      }
+    }
+
+    stateManager.setHuman(human);
+  }
+
+  const sourceMessageIds = stateManager.messages_get("emmet")
+    .filter(m => m.source_tag === preview.sourceTag)
+    .map(m => m.id);
+  if (sourceMessageIds.length > 0) {
+    stateManager.messages_remove("emmet", sourceMessageIds);
+  }
+
+  const filename = preview.sourceTag.startsWith("import:document:")
+    ? preview.sourceTag.slice("import:document:".length)
+    : preview.sourceTag;
+
+  const human = stateManager.getHuman();
+  if (human.settings?.document?.processed_documents) {
+    delete human.settings.document.processed_documents[filename];
+    stateManager.setHuman(human);
+  }
+
+  return result;
+}

package/src/integrations/persona-history/importer.ts

@@ -0,0 +1,197 @@
+import type { StateManager } from "../../core/state-manager.js";
+import type { Message } from "../../core/types.js";
+import {
+  queueTopicScan,
+  queuePersonScan,
+  type ExtractionContext,
+} from "../../core/orchestrators/human-extraction.js";
+
+export interface PersonaHistoryImportResult {
+  daysQueued: number;
+  personasProcessed: number;
+  scansQueued: number;
+  complete: boolean;
+}
+
+export interface PersonaHistoryImporterOptions {
+  stateManager: StateManager;
+}
+
+function dayBounds(dateStr: string): { start: number; end: number } {
+  const start = new Date(dateStr + "T00:00:00.000Z").getTime();
+  const end = new Date(dateStr + "T23:59:59.999Z").getTime();
+  return { start, end };
+}
+
+function nextDay(dateStr: string): string {
+  const d = new Date(dateStr + "T12:00:00.000Z");
+  d.setUTCDate(d.getUTCDate() + 1);
+  return d.toISOString().slice(0, 10);
+}
+
+function todayUTC(): string {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function messagesForDay(messages: Message[], start: number, end: number): Message[] {
+  return messages.filter(m => {
+    const ts = new Date(m.timestamp).getTime();
+    return ts >= start && ts <= end;
+  });
+}
+
+function hasUnprocessed(messages: Message[]): boolean {
+  return messages.some(m => !m.t || !m.p);
+}
+
+export async function importPersonaHistory(
+  options: PersonaHistoryImporterOptions
+): Promise<PersonaHistoryImportResult> {
+  const { stateManager } = options;
+  const human = stateManager.getHuman();
+  const settings = human.settings?.personaHistory;
+
+  const result: PersonaHistoryImportResult = {
+    daysQueued: 0,
+    personasProcessed: 0,
+    scansQueued: 0,
+    complete: false,
+  };
+
+  if (settings?.complete) {
+    result.complete = true;
+    return result;
+  }
+
+  const personas = stateManager.persona_getAll().filter(p => !p.is_archived);
+  const today = todayUTC();
+
+  let currentDate = settings?.last_queued_date
+    ? nextDay(settings.last_queued_date)
+    : settings?.start_date ?? findEarliestMessageDate(stateManager) ?? today;
+
+  if (currentDate > today) {
+    markComplete(stateManager);
+    result.complete = true;
+    return result;
+  }
+
+  console.log(`[PersonaHistory] Queuing day: ${currentDate}`);
+
+  const { start, end } = dayBounds(currentDate);
+
+  for (const persona of personas) {
+    const allMessages = stateManager.messages_get(persona.id);
+    const dayMessages = messagesForDay(allMessages, start, end);
+
+    if (dayMessages.length === 0) continue;
+    if (!hasUnprocessed(dayMessages)) continue;
+
+    const firstDayIdx = allMessages.findIndex(m => {
+      const ts = new Date(m.timestamp).getTime();
+      return ts >= start;
+    });
+    const contextMsgs = firstDayIdx > 0 ? allMessages.slice(Math.max(0, firstDayIdx - 20), firstDayIdx) : [];
+
+    const context: ExtractionContext = {
+      personaId: persona.id,
+      channelDisplayName: persona.display_name,
+      messages_context: contextMsgs,
+      messages_analyze: dayMessages,
+    };
+
+    const extractionModel = settings?.extraction_model;
+    queueTopicScan(context, stateManager, { extraction_model: extractionModel });
+    queuePersonScan(context, stateManager, { extraction_model: extractionModel });
+
+    result.personasProcessed++;
+    result.scansQueued += 2;
+  }
+
+  for (const room of Object.values((stateManager.getStorageState() as any).rooms ?? {})) {
+    const r = room as { id: string; display_name: string; messages?: Message[] };
+    if (!r.messages || r.messages.length === 0) continue;
+
+    const dayMessages = messagesForDay(r.messages, start, end);
+    if (dayMessages.length === 0) continue;
+    if (!hasUnprocessed(dayMessages)) continue;
+
+    const firstDayIdx = r.messages.findIndex((m: Message) => {
+      const ts = new Date(m.timestamp).getTime();
+      return ts >= start;
+    });
+    const contextMsgs = firstDayIdx > 0 ? r.messages.slice(Math.max(0, firstDayIdx - 20), firstDayIdx) : [];
+
+    const context: ExtractionContext = {
+      personaId: r.id,
+      channelDisplayName: r.display_name,
+      messages_context: contextMsgs,
+      messages_analyze: dayMessages,
+      roomId: r.id,
+    };
+
+    const extractionModel = settings?.extraction_model;
+    queueTopicScan(context, stateManager, { extraction_model: extractionModel });
+    queuePersonScan(context, stateManager, { extraction_model: extractionModel });
+
+    result.scansQueued += 2;
+  }
+
+  result.daysQueued = 1;
+
+  const isLastDay = currentDate >= today;
+  advanceProgress(stateManager, currentDate, isLastDay);
+
+  if (isLastDay) {
+    result.complete = true;
+    console.log(`[PersonaHistory] All days queued — marking complete`);
+  } else {
+    console.log(`[PersonaHistory] Day ${currentDate} queued (${result.scansQueued} scans), next: ${nextDay(currentDate)}`);
+  }
+
+  return result;
+}
+
+function findEarliestMessageDate(stateManager: StateManager): string | null {
+  const personas = stateManager.persona_getAll();
+  let earliest: number | null = null;
+
+  for (const persona of personas) {
+    const msgs = stateManager.messages_get(persona.id);
+    for (const m of msgs) {
+      const ts = new Date(m.timestamp).getTime();
+      if (earliest === null || ts < earliest) earliest = ts;
+    }
+  }
+
+  return earliest !== null ? new Date(earliest).toISOString().slice(0, 10) : null;
+}
+
+function advanceProgress(stateManager: StateManager, date: string, complete: boolean): void {
+  const human = stateManager.getHuman();
+  stateManager.setHuman({
+    ...human,
+    settings: {
+      ...human.settings,
+      personaHistory: {
+        ...human.settings?.personaHistory,
+        last_queued_date: date,
+        ...(complete && { complete: true }),
+      },
+    },
+  });
+}
+
+function markComplete(stateManager: StateManager): void {
+  const human = stateManager.getHuman();
+  stateManager.setHuman({
+    ...human,
+    settings: {
+      ...human.settings,
+      personaHistory: {
+        ...human.settings?.personaHistory,
+        complete: true,
+      },
+    },
+  });
+}

package/src/integrations/persona-history/index.ts

@@ -0,0 +1,3 @@
+export { importPersonaHistory } from "./importer.js";
+export type { PersonaHistoryImportResult, PersonaHistoryImporterOptions } from "./importer.js";
+export type { PersonaHistorySettings } from "./types.js";

package/src/integrations/persona-history/types.ts

@@ -0,0 +1,7 @@
+export interface PersonaHistorySettings {
+  integration?: boolean;
+  extraction_model?: string;
+  start_date?: string;      // ISO date string "YYYY-MM-DD", defaults to earliest message found
+  last_queued_date?: string; // ISO date of last day fully queued — resume point if interrupted
+  complete?: boolean;       // Set true when all days have been queued; prevents re-runs
+}