ei-tui 0.9.3 → 1.0.0

package/src/core/processor.ts

@@ -39,7 +39,9 @@ import { ContextStatus as ContextStatusEnum, RoomMode } from "./types.js";
 import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/index.js";
 import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
 import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
+import { EMMETT_PERSONA_DEFINITION } from "../templates/emmett.js";
 import { shouldStartCeremony, startCeremony, handleCeremonyProgress, queueReflectionDrain, queueUserDedupRequest, queueRoomCapture, queuePersonaCapture, checkAndQueueRoomExtraction, queueTargetedPersonUpdate, queueTargetedTopicUpdate } from "./orchestrators/index.js";
+import { finishDocumentBatch } from "./handlers/document-segmentation.js";
 import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
 import { DEFAULT_SEED_TRAITS } from "./constants/seed-traits.js";
 
@@ -132,6 +134,8 @@ import {
   markAllRoomMessagesRead,
 } from "./room-manager.js";
 import type { RoomCreationInput, RoomEntity, RoomMessage, RoomSummary } from "./types.js";
+import { previewUnsource as _previewUnsource } from "../integrations/document/unsource.js";
+import type { UnsourcePreview, UnsourceResult } from "../integrations/document/unsource.js";
 
 const DEFAULT_LOOP_INTERVAL_MS = 100;
 const DEFAULT_OPENCODE_POLLING_MS = 60000;
@@ -282,6 +286,46 @@ export class Processor {
     this.interface.onMessageAdded?.(eiEntity.id);
   }
 
+  private bootstrapEmmett(): void {
+    const existing = this.stateManager.persona_getById("emmet");
+    if (existing) {
+      if (existing.is_archived) {
+        this.stateManager.persona_unarchive("emmet");
+      }
+      return;
+    }
+    const readMemoryTool = this.stateManager.tools_getByName("read_memory");
+    const emmettEntity: PersonaEntity = {
+      ...EMMETT_PERSONA_DEFINITION,
+      id: "emmet",
+      display_name: "Emmett",
+      last_updated: new Date().toISOString(),
+      tools: readMemoryTool ? [readMemoryTool.id] : [],
+    };
+    this.stateManager.persona_add(emmettEntity);
+    this.interface.onPersonaAdded?.();
+  }
+
+  async importDocument(content: string, filename: string): Promise<import("../integrations/document/types.js").DocumentImportResult> {
+    this.bootstrapEmmett();
+    const { importDocument } = await import("../integrations/document/importer.js");
+    return importDocument({
+      stateManager: this.stateManager,
+      interface: this.interface,
+      content,
+      filename,
+    });
+  }
+
+  getUnsourcePreview(sourceTag: string): UnsourcePreview {
+    return _previewUnsource(sourceTag, this.stateManager);
+  }
+
+  async executeUnsource(preview: UnsourcePreview): Promise<UnsourceResult> {
+    const { executeUnsource } = await import("../integrations/document/unsource.js");
+    return executeUnsource(preview, this.stateManager);
+  }
+
   /**
    * Seed built-in tool providers and tools if they don't exist yet.
    * Called on every startup (after state load/restore) — safe to call repeatedly.
@@ -877,8 +921,8 @@ export class Processor {
       modified = true;
     }
 
-    if (human.settings.default_context_window_hours == null) {
-      human.settings.default_context_window_hours = 8;
+    if (human.settings.default_context_window_ms == null) {
+      human.settings.default_context_window_ms = 28800000;
       modified = true;
     }
 
@@ -1044,7 +1088,6 @@ const toolNextSteps = new Set([
   LLMNextStep.HandleEiHeartbeat,
   LLMNextStep.HandleToolContinuation,
   LLMNextStep.HandleDedupCurate,
-  LLMNextStep.HandlePersonIdentifierMigration,
 ]);
             const toolPersonaId =
               personaId ??
@@ -1059,13 +1102,8 @@ const toolNextSteps = new Set([
               (request.next_step === LLMNextStep.HandleToolContinuation &&
                 request.data.originalNextStep === LLMNextStep.HandleDedupCurate);
 
-            const isPersonMigrationRequest =
-              request.next_step === LLMNextStep.HandlePersonIdentifierMigration ||
-              (request.next_step === LLMNextStep.HandleToolContinuation &&
-                request.data.originalNextStep === LLMNextStep.HandlePersonIdentifierMigration);
-
             let tools: ToolDefinition[] = [];
-            if (isDedupRequest || isPersonMigrationRequest) {
+            if (isDedupRequest) {
               const readMemory = this.stateManager.tools_getByName("read_memory");
               if (readMemory?.enabled) {
                 tools = [readMemory];
@@ -1174,6 +1212,15 @@ const toolNextSteps = new Set([
       await this.checkAndSyncCursor(human, now);
     }
 
+    if (
+      this.isTUI &&
+      human.settings?.personaHistory?.integration &&
+      !human.settings.personaHistory.complete &&
+      this.stateManager.queue_length() === 0
+    ) {
+      await this.checkAndSyncPersonaHistory(human);
+    }
+
     if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
       if (human.settings?.sync && remoteSync.isConfigured()) {
         const state = this.stateManager.getStorageState();
@@ -1186,7 +1233,7 @@ const toolNextSteps = new Set([
     }
 
     for (const persona of this.stateManager.persona_getAll()) {
-      if (persona.is_paused || persona.is_archived) continue;
+      if (persona.is_paused || persona.is_archived || persona.is_static) continue;
 
       const defaultHeartbeatMs = this.stateManager.getHuman().settings?.default_heartbeat_ms ?? 1800000;
       const heartbeatDelay = persona.heartbeat_delay_ms ?? defaultHeartbeatMs;
@@ -1201,14 +1248,14 @@ const toolNextSteps = new Set([
 
         if (timeSinceHeartbeat >= heartbeatDelay) {
            const history = this.stateManager.messages_get(persona.id);
-           const contextWindowHours =
-             persona.context_window_hours 
-             ?? this.stateManager.getHuman().settings?.default_context_window_hours 
-             ?? 8;
+           const contextWindowMs =
+             persona.context_window_ms 
+             ?? this.stateManager.getHuman().settings?.default_context_window_ms 
+             ?? 28800000;
           const contextHistory = filterMessagesForContext(
             history,
             persona.context_boundary,
-            contextWindowHours
+            contextWindowMs
           );
           const trailing = countTrailingPersonaMessages(contextHistory);
           if (trailing < 3) {
@@ -1414,6 +1461,32 @@ const toolNextSteps = new Set([
       });
   }
 
+  private personaHistoryImportInProgress = false;
+
+  private async checkAndSyncPersonaHistory(_human: HumanEntity): Promise<void> {
+    if (this.personaHistoryImportInProgress) return;
+
+    this.personaHistoryImportInProgress = true;
+    import("../integrations/persona-history/importer.js")
+      .then(({ importPersonaHistory }) =>
+        importPersonaHistory({ stateManager: this.stateManager })
+      )
+      .then((result) => {
+        if (result.scansQueued > 0) {
+          console.log(
+            `[Processor] PersonaHistory: ${result.scansQueued} scans queued` +
+            (result.complete ? " — import complete" : "")
+          );
+        }
+      })
+      .catch((err) => {
+        console.warn(`[Processor] PersonaHistory sync failed:`, err);
+      })
+      .finally(() => {
+        this.personaHistoryImportInProgress = false;
+      });
+  }
+
   private augmentRoomRequest(request: LLMRequest): LLMRequest {
     if (request.next_step !== LLMNextStep.HandleRoomResponse) return request;
 
@@ -1621,13 +1694,6 @@ const toolNextSteps = new Set([
         }
       }
 
-      if (response.request.next_step === LLMNextStep.HandlePersonaDescriptions) {
-        const personaId = response.request.data.personaId as string;
-        if (personaId) {
-          this.interface.onPersonaUpdated?.(personaId);
-        }
-      }
-
       if (
         response.request.next_step === LLMNextStep.HandlePersonaTraitExtraction ||
         response.request.next_step === LLMNextStep.HandlePersonaTopicRating
@@ -1688,6 +1754,16 @@ const toolNextSteps = new Set([
       if (typeof response.request.data.ceremony_progress === "number") {
         handleCeremonyProgress(this.stateManager, response.request.data.ceremony_progress);
       }
+
+      if (response.request.next_step === LLMNextStep.HandleDocumentSegmentation) {
+        const batchId = response.request.data.batchId as string;
+        const filename = response.request.data.filename as string;
+        if (batchId && !this.stateManager.queue_hasPendingDocumentSegments(batchId)) {
+          finishDocumentBatch(batchId, filename, this.stateManager);
+          this.interface.onMessageAdded?.("emmet");
+          this.interface.onHumanUpdated?.();
+        }
+      }
     } catch (err) {
       const errorMsg = err instanceof Error ? err.message : String(err);
       const result = this.stateManager.queue_fail(response.request.id, errorMsg);

package/src/core/prompt-context-builder.ts

@@ -291,6 +291,7 @@ export async function buildResponsePromptData(
       topics: persona.topics,
       interested_topics: persona.topics.filter(t => t.exposure_desired - t.exposure_current > 0.2),
       include_message_timestamps: persona.include_message_timestamps,
+      pending_update: persona.pending_update,
     },
     human: filteredHuman,
     visible_personas: visiblePersonas,
@@ -318,10 +319,10 @@ export async function buildRoomResponsePromptData(
 
   let sourceMessages: RoomMessage[];
   if (room.mode === RoomMode.FreeForAll) {
-    const contextWindowHours = room.context_window_hours
-      ?? human.settings?.default_context_window_hours
-      ?? 8;
-    const windowCutoff = new Date(Date.now() - contextWindowHours * 60 * 60 * 1000).toISOString();
+    const contextWindowMs = room.context_window_ms
+      ?? human.settings?.default_context_window_ms
+      ?? 28800000;
+    const windowCutoff = new Date(Date.now() - contextWindowMs).toISOString();
     const boundaryMs = room.context_boundary ? new Date(room.context_boundary).getTime() : 0;
     sourceMessages = allSourceMessages.filter(m => {
       const msgMs = new Date(m.timestamp).getTime();
@@ -334,8 +335,8 @@ export async function buildRoomResponsePromptData(
     const byCount = allSourceMessages.slice(-MIN_ROOM_MESSAGES);
     if (byCount.length > sourceMessages.length) sourceMessages = byCount;
   } else {
-    const contextWindowHours = human.settings?.default_context_window_hours ?? 8;
-    const windowCutoff = new Date(Date.now() - contextWindowHours * 60 * 60 * 1000).toISOString();
+    const contextWindowMs = human.settings?.default_context_window_ms ?? 28800000;
+    const windowCutoff = new Date(Date.now() - contextWindowMs).toISOString();
     const byTime = allSourceMessages.filter(m => m.timestamp >= windowCutoff);
     const byCount = allSourceMessages.slice(-MIN_ROOM_MESSAGES);
     sourceMessages = byTime.length >= byCount.length ? byTime : byCount;

package/src/core/queue-manager.ts

@@ -18,6 +18,39 @@ export async function resumeQueue(sm: StateManager): Promise<void> {
 }
 
 export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
+  const activeItems = sm.queue_getAllActiveItems();
+  const segmentationItems = activeItems.filter(
+    r => r.next_step === LLMNextStep.HandleDocumentSegmentation
+  );
+
+  const batchMap = new Map<string, { filename: string; count: number }>();
+  for (const item of segmentationItems) {
+    const { batchId, filename } = item.data as { batchId: string; filename: string };
+    if (!batchId || !filename) continue;
+    const existing = batchMap.get(batchId);
+    if (existing) {
+      existing.count++;
+    } else {
+      batchMap.set(batchId, { filename, count: 1 });
+    }
+  }
+
+  const pending_documents = batchMap.size > 0
+    ? Array.from(batchMap.entries()).map(([batchId, { filename, count }]) => ({ batchId, filename, count }))
+    : undefined;
+
+  const extractingSet = new Set<string>();
+  for (const item of activeItems) {
+    const sources = item.data.sources as string[] | undefined;
+    if (!Array.isArray(sources)) continue;
+    for (const s of sources) {
+      if (typeof s === "string" && s.startsWith("import:document:")) {
+        extractingSet.add(s.slice("import:document:".length));
+      }
+    }
+  }
+  const extracting_documents = extractingSet.size > 0 ? Array.from(extractingSet) : undefined;
+
   return {
     state: sm.queue_isPaused()
       ? "paused"
@@ -27,6 +60,8 @@ export async function getQueueStatus(sm: StateManager): Promise<QueueStatus> {
     pending_count: sm.queue_length(),
     dlq_count: sm.queue_dlqLength(),
     embedding_warning: sm.embedding_getWarning() || undefined,
+    pending_documents,
+    extracting_documents,
   };
 }
 

package/src/core/state/personas.ts

@@ -1,21 +1,5 @@
 import type { PersonaEntity, Message, ContextStatus } from "../types.js";
 
-// TODO(v1.0.0): Remove LegacyMessage migration — verbal_response/action_response no longer written
-type LegacyMessage = Message & { verbal_response?: string; action_response?: string };
-
-function migrateMessage(msg: Message): Message {
-  if (msg.content) return msg;
-  if (msg.silence_reason) return msg;
-  const legacy = msg as LegacyMessage;
-  const hasLegacy = 'verbal_response' in legacy || 'action_response' in legacy;
-  if (!hasLegacy) return msg;
-  const parts: string[] = [];
-  if (legacy.action_response) parts.push(`_${legacy.action_response}_`);
-  if (legacy.verbal_response) parts.push(legacy.verbal_response);
-  const { verbal_response: _vr, action_response: _ar, ...rest } = legacy;
-  return parts.length > 0 ? { ...rest, content: parts.join('\n\n') } : rest as Message;
-}
-
 export interface PersonaData {
   entity: PersonaEntity;
   messages: Message[];
@@ -29,7 +13,7 @@ export class PersonaState {
     this.personas = new Map(
       Object.entries(personas).map(([id, data]) => [
         id,
-        { entity: data.entity, messages: data.messages.map(migrateMessage) },
+        { entity: data.entity, messages: data.messages },
       ])
     );
   }

package/src/core/state/queue.ts

@@ -1,5 +1,5 @@
 import type { LLMRequest, QueueFailResult } from "../types.js";
-import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS } from "../types.js";
+import { DLQ_MAX_COUNT, DLQ_MAX_AGE_DAYS, LLMNextStep } from "../types.js";
 
 const BASE_BACKOFF_MS = 2_000;
 const MAX_BACKOFF_MS = 30_000;
@@ -200,6 +200,14 @@ export class QueueState {
     return this.queue.some(r => r.state !== "dlq" && typeof r.data.ceremony_progress === "number" && r.data.ceremony_progress > 0);
   }
 
+  hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queue.some(r =>
+      r.state !== "dlq" &&
+      r.next_step === LLMNextStep.HandleDocumentSegmentation &&
+      r.data.batchId === batchId
+    );
+  }
+
   clear(): number {
     const count = this.queue.filter(r => r.state !== "dlq").length;
     this.queue = this.queue.filter(r => r.state === "dlq");

package/src/core/state-manager.ts

@@ -69,76 +69,10 @@ export class StateManager {
     this.migrateMessageFlags();
     this.migrateInterestedPersonas();
     this.migrateProviderModel();
-    this.migratePersonaMessageContent();
-    this.migrateRoomMessageContent();
     this.migrateThemes();
     this.migrateFfaParentIds();
   }
 
-  private migratePersonaMessageContent(): void {
-    // TODO(v1.0.0): Remove legacy persona message migration — verbal_response/action_response no longer written
-    const rawPersonas = (this.personaState as unknown as { personas: Map<string, { messages: Message[] }> }).personas;
-    let migratedCount = 0;
-    for (const [, data] of rawPersonas) {
-      const messages = data.messages;
-      for (const msg of messages) {
-        const legacy = msg as Message & { verbal_response?: string; action_response?: string };
-        if (!('verbal_response' in legacy || 'action_response' in legacy)) continue;
-        if (msg.content) {
-          delete (legacy as any).verbal_response;
-          delete (legacy as any).action_response;
-          migratedCount++;
-          continue;
-        }
-        if (msg.silence_reason) {
-          delete (legacy as any).verbal_response;
-          delete (legacy as any).action_response;
-          migratedCount++;
-          continue;
-        }
-        const parts: string[] = [];
-        if (legacy.action_response) parts.push(`_${legacy.action_response}_`);
-        if (legacy.verbal_response) parts.push(legacy.verbal_response);
-        if (parts.length > 0) (msg as any).content = parts.join('\n\n');
-        delete (legacy as any).verbal_response;
-        delete (legacy as any).action_response;
-        migratedCount++;
-      }
-    }
-    if (migratedCount > 0) {
-      this.scheduleSave();
-      console.log(`[StateManager] Migrated ${migratedCount} persona messages to unified content field`);
-    }
-  }
-
-  private migrateRoomMessageContent(): void {
-    const rooms = this.roomState.getAll(true);
-    let migratedCount = 0;
-
-    for (const room of rooms) {
-      for (const msg of room.messages) {
-        if (msg.content) continue;
-        if (msg.silence_reason) continue;
-        // TODO(v1.0.0): Remove legacy room message migration — verbal_response/action_response no longer written
-        const legacy = msg as RoomMessage & { verbal_response?: string; action_response?: string };
-        const hasLegacy = 'verbal_response' in legacy || 'action_response' in legacy;
-        if (!hasLegacy) continue;
-        const parts: string[] = [];
-        if (legacy.action_response) parts.push(`_${legacy.action_response}_`);
-        if (legacy.verbal_response) parts.push(legacy.verbal_response);
-        if (parts.length > 0) msg.content = parts.join('\n\n');
-        delete (msg as any).verbal_response;
-        delete (msg as any).action_response;
-        migratedCount++;
-      }
-    }
-
-    if (migratedCount > 0) {
-      this.scheduleSave();
-      console.log(`[StateManager] Migrated ${migratedCount} room messages to unified content field`);
-    }
-  }
-
   /**
    * Migration: learned_by used to store display names; now stores persona IDs.
    * On load, attempt to resolve display names -> IDs using current persona map.
@@ -983,6 +917,10 @@ export class StateManager {
     return this.queueState.hasPendingCeremonies();
   }
 
+  queue_hasPendingDocumentSegments(batchId: string): boolean {
+    return this.queueState.hasPendingDocumentSegments(batchId);
+  }
+
   queue_clear(): number {
     const result = this.queueState.clear();
     this.scheduleSave();

package/src/core/types/entities.ts

@@ -20,6 +20,11 @@ export interface OpenCodeSettings {
   processed_sessions?: Record<string, string>;  // sessionId → ISO timestamp of last import
 }
 
+export interface DocumentSettings {
+  extraction_model?: string;
+  processed_documents?: Record<string, string>;
+}
+
 export interface CeremonyConfig {
   time: string;  // "HH:MM" format (e.g., "09:00")
   last_ceremony?: string;  // ISO timestamp
@@ -103,12 +108,11 @@ export interface HumanSettings {
   default_model?: string;           // Will store ModelConfig.id GUID post-migration
   oneshot_model?: string;           // Model for AI-assist (wand) requests; falls back to default_model. Will store ModelConfig.id GUID post-migration.
   rewrite_model?: string;           // Model for rewrite ceremony step; must be capable (Sonnet/Opus class). Unset = rewrite disabled. Will store ModelConfig.id GUID post-migration.
-  people_migration_complete?: boolean; // Set to true when all Person records have identifiers. Ceremony migration step short-circuits when true.
   queue_paused?: boolean;
   skip_quote_delete_confirm?: boolean;
   name_display?: string;
   default_heartbeat_ms?: number;
-  default_context_window_hours?: number;
+  default_context_window_ms?: number;
   message_min_count?: number;
   message_max_age_days?: number;
   accounts?: ProviderAccount[];
@@ -118,8 +122,10 @@ export interface HumanSettings {
   backup?: BackupConfig;
   claudeCode?: import("../../integrations/claude-code/types.js").ClaudeCodeSettings;
   cursor?: import("../../integrations/cursor/types.js").CursorSettings;
+  document?: DocumentSettings;
   active_theme?: string;
   custom_themes?: ThemeDefinition[];
+  personaHistory?: import("../../integrations/persona-history/types.js").PersonaHistorySettings;
 }
 
 export interface HumanEntity {
@@ -150,7 +156,7 @@ export interface PersonaEntity {
   archived_at?: string;
   is_static: boolean;
   heartbeat_delay_ms?: number;
-  context_window_hours?: number;
+  context_window_ms?: number;
   include_message_timestamps?: boolean; // Prepend ISO timestamp to each message sent to the LLM
   context_boundary?: string;  // ISO timestamp - messages before this excluded from LLM context
   last_updated: string;
@@ -203,3 +209,11 @@ export type ReservedPersonaName = typeof RESERVED_PERSONA_NAMES[number];
 export function isReservedPersonaName(name: string): boolean {
   return RESERVED_PERSONA_NAMES.includes(name.toLowerCase() as ReservedPersonaName);
 }
+
+// Reserved persona IDs (built-in system personas that cannot be deleted)
+export const RESERVED_PERSONA_IDS = ["ei", "emmet"] as const;
+export type ReservedPersonaId = typeof RESERVED_PERSONA_IDS[number];
+
+export function isReservedPersonaId(id: string): boolean {
+  return (RESERVED_PERSONA_IDS as readonly string[]).includes(id);
+}

package/src/core/types/enums.ts

@@ -26,7 +26,6 @@ export enum LLMPriority {
 export enum LLMNextStep {
   HandlePersonaResponse = "handlePersonaResponse",
   HandlePersonaGeneration = "handlePersonaGeneration",
-  HandlePersonaDescriptions = "handlePersonaDescriptions",
   HandleFactFind = "handleFactFind",
   HandleHumanTopicScan = "handleHumanTopicScan",
   HandleHumanPersonScan = "handleHumanPersonScan",
@@ -51,9 +50,9 @@ export enum LLMNextStep {
   HandleRoomResponse = "handleRoomResponse",
   HandleRoomJudge = "handleRoomJudge",
   HandlePersonaPreview = "handlePersonaPreview",
-  HandlePersonIdentifierMigration = "handlePersonIdentifierMigration",
   HandleTopicValidate = "handleTopicValidate",
   HandleReflectionCritic = "handleReflectionCritic",
+  HandleDocumentSegmentation = "handleDocumentSegmentation",
 }
 
 export enum ProviderType {

package/src/core/types/integrations.ts

@@ -75,6 +75,8 @@ export interface QueueStatus {
   current_operation?: string;
   /** True when the embedding service failed and topic/person matching fell back to recent items. */
   embedding_warning?: boolean;
+  pending_documents?: Array<{ batchId: string; filename: string; count: number }>;
+  extracting_documents?: string[];
 }
 
 export interface EiError {

package/src/core/types/llm.ts

@@ -27,6 +27,15 @@ export interface Message {
 
   external?: boolean;          // Set by integration importers (OpenCode, Cursor, Claude Code); invisible to LLM context
 
+  /**
+   * Integration source tag. Set ONLY on external: true messages by importers (document, Slack, etc.)
+   * to identify which external source this synthetic message came from.
+   * Format: "import:document:filename" | "slack:channelId" | etc.
+   * Enables quote provenance tracing: quote.message_id → message.source_tag → original source.
+   * Never set on conversational messages.
+   */
+  source_tag?: string;
+
 }
 
 export interface ChatMessage {

package/src/core/types/rooms.ts

@@ -35,7 +35,7 @@ export interface RoomEntity {
   created_at: string;
   last_updated: string;
   capture_used?: boolean;
-  context_window_hours?: number;  // FFA only; falls back to human.settings.default_context_window_hours
+  context_window_ms?: number;  // FFA only; falls back to human.settings.default_context_window_ms
   context_boundary?: string;      // FFA only; ISO timestamp; same semantics as persona context_boundary
   messages: RoomMessage[];
 }

package/src/integrations/claude-code/importer.ts

@@ -262,7 +262,7 @@ export async function importClaudeCodeSessions(
 
     const context: ExtractionContext = {
       personaId: persona.id,
-      personaDisplayName: persona.display_name,
+      channelDisplayName: persona.display_name,
       messages_context: contextMsgs,
       messages_analyze: toAnalyze,
       sources: [`claudecode:${getMachineId()}:${targetSession.id}`],

package/src/integrations/cursor/importer.ts

@@ -221,7 +221,7 @@ export async function importCursorSessions(
 
     const context: ExtractionContext = {
       personaId: persona.id,
-      personaDisplayName: persona.display_name,
+      channelDisplayName: persona.display_name,
       messages_context: contextMsgs,
       messages_analyze: toAnalyze,
       sources: [`cursor:${getMachineId()}:${targetSession.id}`],

package/src/integrations/document/chunker.ts

@@ -0,0 +1,88 @@
+import { expandToWordBoundaries } from "../../core/handlers/human-matching.js";
+
+const DEFAULT_CHUNK_CHARS = 6000;
+const DEFAULT_OVERLAP_CHARS = 300;
+
+const MARKDOWN_SEPARATORS = ["\n## ", "\n### ", "\n#### ", "\n\n", "\n", ". ", " ", ""];
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+
+function splitOnSeparator(text: string, separator: string): string[] {
+  if (separator === "") {
+    return text.split("");
+  }
+  return text.split(separator);
+}
+
+function mergeChunks(pieces: string[], separator: string, chunkSize: number): string[] {
+  const merged: string[] = [];
+  let current = "";
+
+  for (const piece of pieces) {
+    const candidate = current ? current + separator + piece : piece;
+    if (candidate.length <= chunkSize) {
+      current = candidate;
+    } else {
+      if (current) merged.push(current);
+      current = piece.length <= chunkSize ? piece : piece;
+    }
+  }
+  if (current) merged.push(current);
+  return merged;
+}
+
+function recursiveSplit(
+  text: string,
+  separators: string[],
+  chunkSize: number
+): string[] {
+  if (text.length <= chunkSize) {
+    return [text];
+  }
+
+  const [separator, ...remainingSeparators] = separators;
+
+  if (separator === undefined) {
+    return [text];
+  }
+
+  const pieces = splitOnSeparator(text, separator);
+  const result: string[] = [];
+
+  for (const piece of pieces) {
+    if (piece.length <= chunkSize) {
+      result.push(piece);
+    } else if (remainingSeparators.length > 0) {
+      result.push(...recursiveSplit(piece, remainingSeparators, chunkSize));
+    } else {
+      result.push(piece);
+    }
+  }
+
+  return mergeChunks(result, separator, chunkSize);
+}
+
+function applyOverlap(chunks: string[], overlapChars: number): string[] {
+  if (overlapChars <= 0 || chunks.length <= 1) return chunks;
+
+  return chunks.map((chunk, i) => {
+    if (i === 0) return chunk;
+    const prev = chunks[i - 1];
+    const rawStart = Math.max(0, prev.length - overlapChars);
+    const { start } = expandToWordBoundaries(prev, rawStart, rawStart);
+    const prefix = prev.slice(start);
+    return prefix + chunk;
+  });
+}
+
+export function recursiveCharacterSplit(
+  text: string,
+  options?: { chunkSize?: number; overlap?: number; isMarkdown?: boolean }
+): string[] {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_CHARS;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP_CHARS;
+  const separators = options?.isMarkdown ? MARKDOWN_SEPARATORS : DEFAULT_SEPARATORS;
+
+  const rawChunks = recursiveSplit(text, separators, chunkSize);
+  const nonEmpty = rawChunks.filter(c => c.trim().length > 0);
+  return applyOverlap(nonEmpty, overlap);
+}