ei-tui 0.2.0 → 0.3.1

package/src/core/handlers/rewrite.ts

@@ -3,7 +3,6 @@ import {
   LLMPriority,
   LLMNextStep,
   type LLMResponse,
-  type Fact,
   type Topic,
   type Person,
   type DataItemBase,
@@ -46,7 +45,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
   // Re-read the item from current state (it may have changed since scan was queued)
   const human = state.getHuman();
   const allItems: DataItemBase[] = [
-    ...human.facts, ...human.topics, ...human.people,
+    ...human.topics, ...human.people,
   ];
   const currentItem = allItems.find(i => i.id === itemId);
   if (!currentItem) {
@@ -59,7 +58,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
   for (const searchTerm of subjects) {
     try {
       const results = await searchHumanData(state, searchTerm, {
-        types: ["fact", "topic", "person"],
+        types: ["topic", "person"],
         limit: 4,  // fetch 4 so we can exclude original and still have 3
       });
       const allMatches: DataItemBase[] = [
@@ -120,14 +119,13 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
 
   // Look up the original item to inherit persona_groups
   const allItems: DataItemBase[] = [
-    ...human.facts, ...human.topics, ...human.people,
+    ...human.topics, ...human.people,
   ];
   const originalItem = allItems.find(i => i.id === itemId);
   const inheritedGroups = originalItem?.persona_groups;
 
   // Helper: resolve actual type from existing records (don't trust LLM's type field)
   const resolveExistingType = (id: string): RewriteItemType | null => {
-    if (human.facts.find(f => f.id === id)) return "fact";
     if (human.topics.find(t => t.id === id)) return "topic";
     if (human.people.find(p => p.id === id)) return "person";
     return null;
@@ -160,18 +158,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
     }
 
     switch (resolvedType) {
-      case "fact": {
-        const existing = human.facts.find(f => f.id === item.id)!;
-        state.human_fact_upsert({
-          ...existing,
-          name: item.name,
-          description: item.description,
-          sentiment: item.sentiment ?? existing.sentiment,
-          last_updated: now,
-          embedding,
-        });
-        break;
-      }
       case "topic": {
         const existing = human.topics.find(t => t.id === item.id)!;
         state.human_topic_upsert({
@@ -228,14 +214,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
     };
 
     switch (item.type) {
-      case "fact": {
-        const fact: Fact = {
-          ...baseFields,
-          validated_date: now,
-        };
-        state.human_fact_upsert(fact);
-        break;
-      }
       case "topic": {
         if (!item.category) {
           console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);

package/src/core/human-data-manager.ts

@@ -145,14 +145,14 @@ export async function getQuotesForMessage(sm: StateManager, messageId: string):
 export async function searchHumanData(
   sm: StateManager,
   query: string,
-  options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number } = {}
+  options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
 ): Promise<{
   facts: Fact[];
   topics: Topic[];
   people: Person[];
   quotes: Quote[];
 }> {
-  const { types = ["fact", "topic", "person", "quote"], limit = 10 } = options;
+  const { types = ["fact", "topic", "person", "quote"], limit = 10, recent } = options;
   const human = sm.getHuman();
   const SIMILARITY_THRESHOLD = 0.3;
 
@@ -163,30 +163,54 @@ export async function searchHumanData(
     quotes: [] as Quote[],
   };
 
+  const recentSort = <T extends { last_updated?: string; last_mentioned?: string }>(items: T[]): T[] =>
+    [...items].sort((a, b) => {
+      const aDate = a.last_mentioned ?? a.last_updated ?? "";
+      const bDate = b.last_mentioned ?? b.last_updated ?? "";
+      return bDate.localeCompare(aDate);
+    });
+
   let queryVector: number[] | null = null;
-  try {
-    const embeddingService = getEmbeddingService();
-    queryVector = await embeddingService.embed(query);
-  } catch (err) {
-    console.warn("[searchHumanData] Failed to generate query embedding:", err);
+  if (query) {
+    try {
+      const embeddingService = getEmbeddingService();
+      queryVector = await embeddingService.embed(query);
+    } catch (err) {
+      console.warn("[searchHumanData] Failed to generate query embedding:", err);
+    }
   }
 
-  const searchItems = <T extends { id: string; embedding?: number[] }>(
+  const searchItems = <T extends { id: string; embedding?: number[]; last_updated?: string; last_mentioned?: string }>(
     items: T[],
     textExtractor: (item: T) => string
   ): T[] => {
+    if (recent && !query) {
+      return recentSort(items).slice(0, limit);
+    }
+
     const withEmbeddings = items.filter((i) => i.embedding?.length);
 
     if (queryVector && withEmbeddings.length > 0) {
-      return findTopK(queryVector, withEmbeddings, limit)
+      const topK = recent ? Math.max(limit * 5, 50) : limit;
+      const found = findTopK(queryVector, withEmbeddings, topK)
         .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
         .map(({ item }) => item);
+      if (recent) {
+        return recentSort(found).slice(0, limit);
+      }
+      return found;
     }
 
+    if (!query) return [];
+
     const lowerQuery = query.toLowerCase();
-    return items
+    const found = items
       .filter((i) => textExtractor(i).toLowerCase().includes(lowerQuery))
-      .slice(0, limit);
+      .slice(0, recent ? Math.max(limit * 5, 50) : limit);
+    if (recent) {
+      return recentSort(found).slice(0, limit);
+    }
+    return found;
   };
 
   if (types.includes("fact")) {

package/src/core/orchestrators/ceremony.ts

@@ -616,12 +616,6 @@ export function queueRewritePhase(state: StateManager): void {
 
   const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
 
-  for (const fact of human.facts) {
-    if ((fact.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
-      itemsToScan.push({ item: fact, type: "fact" });
-    }
-  }
-
   for (const topic of human.topics) {
     if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
       itemsToScan.push({ item: topic, type: "topic" });

package/src/core/orchestrators/dedup-phase.ts

@@ -20,7 +20,7 @@ interface Cluster {
 // DEDUP CANDIDATE FINDING (copied from ceremony.ts)
 // =============================================================================
 
-const DEDUP_DEFAULT_THRESHOLD = 0.85;  // Lowered from 0.95 based on experimental analysis: 0.95 only catches 3.9% of duplicate name groups, 0.85 catches 46.7%
+const DEDUP_DEFAULT_THRESHOLD = 0.95;  // Raised from 0.90: Ei's topic corpus is a single dense project domain — mega-cluster persists all the way to 0.92. At 0.95, max cluster drops to 7 items.
 
 function findDedupCandidates<T extends DedupableItem>(
   items: T[],
@@ -139,7 +139,6 @@ export function queueDedupPhase(state: StateManager): void {
   console.log(`[Dedup] Starting deduplication phase (threshold: ${threshold})`);
   
   const entityTypes: Array<{ type: DataItemType; items: DedupableItem[] }> = [
-    { type: "fact", items: human.facts },
     { type: "topic", items: human.topics },
     { type: "person", items: human.people },
   ];
@@ -178,7 +177,7 @@ export function queueDedupPhase(state: StateManager): void {
       // Build prompt
       const prompt = buildDedupPrompt({
         cluster: clusterEntities,
-        itemType: type,
+        itemType: type as "topic" | "person",
         similarityRange: { min: cluster.minSim, max: cluster.maxSim }
       });
       

package/src/core/processor.ts

@@ -108,6 +108,7 @@ const DEFAULT_LOOP_INTERVAL_MS = 100;
 const DEFAULT_CONTEXT_WINDOW_HOURS = 8;
 const DEFAULT_OPENCODE_POLLING_MS = 1800000;
 const DEFAULT_CLAUDE_CODE_POLLING_MS = 1800000;
+const DEFAULT_CURSOR_POLLING_MS = 1800000;
 
 let processorInstanceCount = 0;
 
@@ -128,6 +129,8 @@ export class Processor {
   private openCodeImportInProgress = false;
   private lastClaudeCodeSync = 0;
   private claudeCodeImportInProgress = false;
+  private lastCursorSync = 0;
+  private cursorImportInProgress = false;
   private pendingConflict: StateConflictData | null = null;
   private storage: Storage | null = null;
   private importAbortController = new AbortController();
@@ -278,19 +281,20 @@ export class Processor {
         name: "read_memory",
         display_name: "Read Memory",
         description:
-          "Search your personal memory for relevant facts, traits, topics, people, or quotes. Use this when you need information about the user that may not be in the current conversation.",
+          "Search your personal memory for relevant facts, topics, people, or quotes. Use this when you need information about the user that may not be in the current conversation. Use `recent: true` to retrieve what's been discussed recently.",
         input_schema: {
           type: "object",
           properties: {
             query: { type: "string", description: "What to search for in memory" },
             types: {
               type: "array",
-              items: { type: "string", enum: ["fact", "trait", "topic", "person", "quote"] },
+              items: { type: "string", enum: ["fact", "topic", "person", "quote"] },
               description: "Limit search to specific memory types (default: all types)",
             },
             limit: { type: "number", description: "Max results to return (default: 10, max: 20)" },
+            recent: { type: "boolean", description: "If true, return recently-mentioned results sorted by last_mentioned date instead of relevance. Combine with a query to filter recent results by topic." },
           },
-          required: ["query"],
+          required: [],
         },
         runtime: "any",
         builtin: true,
@@ -876,6 +880,14 @@ const toolNextSteps = new Set([
       await this.checkAndSyncClaudeCode(human, now);
     }
 
+    if (
+      this.isTUI &&
+      human.settings?.cursor?.integration &&
+      this.stateManager.queue_length() === 0
+    ) {
+      await this.checkAndSyncCursor(human, now);
+    }
+
     if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
       if (human.settings?.sync && remoteSync.isConfigured()) {
         const state = this.stateManager.getStorageState();
@@ -1062,6 +1074,59 @@ const toolNextSteps = new Set([
       });
   }
 
+  private async checkAndSyncCursor(human: HumanEntity, now: number): Promise<void> {
+    if (this.cursorImportInProgress) {
+      return;
+    }
+
+    const cursor = human.settings?.cursor;
+    const pollingInterval = cursor?.polling_interval_ms ?? DEFAULT_CURSOR_POLLING_MS;
+    const lastSync = cursor?.last_sync ? new Date(cursor.last_sync).getTime() : 0;
+    const timeSinceSync = now - lastSync;
+
+    if (timeSinceSync < pollingInterval && this.lastCursorSync > 0) {
+      return;
+    }
+
+    this.lastCursorSync = now;
+    const syncTimestamp = new Date().toISOString();
+    this.stateManager.setHuman({
+      ...this.stateManager.getHuman(),
+      settings: {
+        ...this.stateManager.getHuman().settings,
+        cursor: {
+          ...cursor,
+          last_sync: syncTimestamp,
+        },
+      },
+    });
+
+    this.cursorImportInProgress = true;
+    import("../integrations/cursor/importer.js")
+      .then(({ importCursorSessions }) =>
+        importCursorSessions({
+          stateManager: this.stateManager,
+          interface: this.interface,
+          signal: this.importAbortController.signal,
+        })
+      )
+      .then((result) => {
+        if (result.sessionsProcessed > 0) {
+          console.log(
+            `[Processor] Cursor sync complete: ${result.sessionsProcessed} sessions, ` +
+              `${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
+              `${result.extractionScansQueued} extraction scans queued`
+          );
+        }
+      })
+      .catch((err) => {
+        console.warn(`[Processor] Cursor sync failed:`, err);
+      })
+      .finally(() => {
+        this.cursorImportInProgress = false;
+      });
+  }
+
   private classifyLLMError(error: string): string {
     const match = error.match(/\((\d{3})\)/);
     if (match) {
@@ -1396,7 +1461,7 @@ const toolNextSteps = new Set([
 
   async searchHumanData(
     query: string,
-    options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number } = {}
+    options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
   ): Promise<{
     facts: Fact[];
     topics: Topic[];

package/src/core/tools/builtin/read-memory.ts

@@ -1,15 +1,9 @@
-/**
- * read_memory builtin tool
- *
- * Delegates to Processor.searchHumanData() — no external call, runtime: "any".
- * The searchHumanData function is injected at construction to avoid circular deps.
- */
 import type { ToolExecutor } from "../types.js";
 import type { Fact, Topic, Person, Quote } from "../../types.js";
 
 type SearchHumanData = (
   query: string,
-  options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number }
+  options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean }
 ) => Promise<{ facts: Fact[]; topics: Topic[]; people: Person[]; quotes: Quote[] }>;
 
 export function createReadMemoryExecutor(searchHumanData: SearchHumanData): ToolExecutor {
@@ -18,10 +12,12 @@ export function createReadMemoryExecutor(searchHumanData: SearchHumanData): Tool
 
     async execute(args: Record<string, unknown>): Promise<string> {
       const query = typeof args.query === "string" ? args.query.trim() : "";
-      console.log(`[read_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}`);
-      if (!query) {
+      const recent = args.recent === true;
+      console.log(`[read_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}, recent=${recent}`);
+
+      if (!query && !recent) {
         console.warn("[read_memory] missing query argument");
-        return JSON.stringify({ error: "Missing required argument: query" });
+        return JSON.stringify({ error: "Missing required argument: query (or use recent: true)" });
       }
 
       const types = Array.isArray(args.types)
@@ -32,7 +28,7 @@ export function createReadMemoryExecutor(searchHumanData: SearchHumanData): Tool
 
       const limit = typeof args.limit === "number" && args.limit > 0 ? Math.min(args.limit, 20) : 10;
 
-      const results = await searchHumanData(query, { types, limit });
+      const results = await searchHumanData(query, { types, limit, recent });
 
       const total = results.facts.length + results.topics.length + results.people.length + results.quotes.length;
       console.log(`[read_memory] query="${query}" => ${total} results (facts=${results.facts.length}, topics=${results.topics.length}, people=${results.people.length}, quotes=${results.quotes.length})`);

package/src/core/types/data-items.ts

@@ -10,6 +10,7 @@ export interface DataItemBase {
   description: string;
   sentiment: number;
   last_updated: string;
+  last_mentioned?: string;        // Set by extraction only, never ceremony. Used for --recent sorting.
   learned_by?: string;           // Persona ID that originally learned this item (stable UUID)
   last_changed_by?: string;      // Persona ID that most recently updated this item (stable UUID)
   persona_groups?: string[];

package/src/core/types/entities.ts

@@ -86,6 +86,7 @@ export interface HumanSettings {
   ceremony?: CeremonyConfig;
   backup?: BackupConfig;
   claudeCode?: import("../../integrations/claude-code/types.js").ClaudeCodeSettings;
+  cursor?: import("../../integrations/cursor/types.js").CursorSettings;
 }
 
 export interface HumanEntity {

package/src/integrations/claude-code/importer.ts

@@ -158,6 +158,10 @@ function updateProcessedState(
       ...human.settings,
       claudeCode: {
         ...human.settings?.claudeCode,
+        // extraction_point is a progress indicator for user visibility only —
+        // it does NOT gate imports. processed_sessions is the sole source of
+        // truth for which sessions have been seen and when.
+        extraction_point: session.lastMessageAt,
         processed_sessions: processedSessions,
       },
     },
@@ -212,7 +216,8 @@ export async function importClaudeCodeSessions(
 
   // ─── Step 2: Find next unprocessed session ────────────────────────────
   const human = stateManager.getHuman();
-  const processedSessions = human.settings?.claudeCode?.processed_sessions ?? {};
+  const settings = human.settings?.claudeCode;
+  const processedSessions = settings?.processed_sessions ?? {};
   const now = Date.now();
 
   let targetSession: ClaudeCodeSession | null = null;

package/src/integrations/claude-code/types.ts

@@ -161,5 +161,6 @@ export interface ClaudeCodeSettings {
   extraction_model?: string;         // "Provider:model" for extraction. Unset = uses default_model.
   extraction_token_limit?: number;   // Token budget for extraction chunking. Unset = resolved from model.
   last_sync?: string;            // ISO timestamp
+  extraction_point?: string;     // ISO timestamp - floor cursor for processed-session skip
   processed_sessions?: Record<string, string>; // sessionId → ISO timestamp of last import
 }