ei-tui 0.2.0 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/cli/README.md +83 -2
- package/src/cli/commands/facts.ts +2 -2
- package/src/cli/commands/people.ts +2 -2
- package/src/cli/commands/quotes.ts +2 -2
- package/src/cli/commands/topics.ts +2 -2
- package/src/cli/mcp.ts +94 -0
- package/src/cli/retrieval.ts +64 -6
- package/src/cli.ts +61 -16
- package/src/core/handlers/dedup.ts +2 -8
- package/src/core/handlers/human-extraction.ts +1 -0
- package/src/core/handlers/human-matching.ts +2 -0
- package/src/core/handlers/rewrite.ts +3 -25
- package/src/core/human-data-manager.ts +35 -11
- package/src/core/orchestrators/ceremony.ts +0 -6
- package/src/core/orchestrators/dedup-phase.ts +2 -3
- package/src/core/orchestrators/human-extraction.ts +20 -6
- package/src/core/processor.ts +93 -4
- package/src/core/queue-manager.ts +1 -0
- package/src/core/state-manager.ts +9 -0
- package/src/core/tools/builtin/read-memory.ts +7 -11
- package/src/core/tools/builtin/web-fetch.ts +73 -0
- package/src/core/tools/index.ts +2 -0
- package/src/core/types/data-items.ts +1 -0
- package/src/core/types/entities.ts +1 -0
- package/src/core/types/integrations.ts +2 -0
- package/src/integrations/claude-code/importer.ts +6 -1
- package/src/integrations/claude-code/types.ts +1 -0
- package/src/integrations/cursor/importer.ts +282 -0
- package/src/integrations/cursor/index.ts +10 -0
- package/src/integrations/cursor/reader.ts +209 -0
- package/src/integrations/cursor/types.ts +140 -0
- package/src/integrations/opencode/importer.ts +7 -1
- package/src/prompts/ceremony/dedup.ts +0 -33
- package/src/prompts/ceremony/rewrite.ts +4 -20
- package/src/prompts/ceremony/types.ts +1 -1
- package/src/prompts/response/index.ts +17 -9
- package/tui/src/context/keyboard.tsx +4 -1
- package/tui/src/util/yaml-serializers.ts +28 -0
|
@@ -3,7 +3,6 @@ import {
|
|
|
3
3
|
LLMPriority,
|
|
4
4
|
LLMNextStep,
|
|
5
5
|
type LLMResponse,
|
|
6
|
-
type Fact,
|
|
7
6
|
type Topic,
|
|
8
7
|
type Person,
|
|
9
8
|
type DataItemBase,
|
|
@@ -46,7 +45,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
46
45
|
// Re-read the item from current state (it may have changed since scan was queued)
|
|
47
46
|
const human = state.getHuman();
|
|
48
47
|
const allItems: DataItemBase[] = [
|
|
49
|
-
...human.
|
|
48
|
+
...human.topics, ...human.people,
|
|
50
49
|
];
|
|
51
50
|
const currentItem = allItems.find(i => i.id === itemId);
|
|
52
51
|
if (!currentItem) {
|
|
@@ -59,7 +58,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
59
58
|
for (const searchTerm of subjects) {
|
|
60
59
|
try {
|
|
61
60
|
const results = await searchHumanData(state, searchTerm, {
|
|
62
|
-
types: ["
|
|
61
|
+
types: ["topic", "person"],
|
|
63
62
|
limit: 4, // fetch 4 so we can exclude original and still have 3
|
|
64
63
|
});
|
|
65
64
|
const allMatches: DataItemBase[] = [
|
|
@@ -120,14 +119,13 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
120
119
|
|
|
121
120
|
// Look up the original item to inherit persona_groups
|
|
122
121
|
const allItems: DataItemBase[] = [
|
|
123
|
-
...human.
|
|
122
|
+
...human.topics, ...human.people,
|
|
124
123
|
];
|
|
125
124
|
const originalItem = allItems.find(i => i.id === itemId);
|
|
126
125
|
const inheritedGroups = originalItem?.persona_groups;
|
|
127
126
|
|
|
128
127
|
// Helper: resolve actual type from existing records (don't trust LLM's type field)
|
|
129
128
|
const resolveExistingType = (id: string): RewriteItemType | null => {
|
|
130
|
-
if (human.facts.find(f => f.id === id)) return "fact";
|
|
131
129
|
if (human.topics.find(t => t.id === id)) return "topic";
|
|
132
130
|
if (human.people.find(p => p.id === id)) return "person";
|
|
133
131
|
return null;
|
|
@@ -160,18 +158,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
160
158
|
}
|
|
161
159
|
|
|
162
160
|
switch (resolvedType) {
|
|
163
|
-
case "fact": {
|
|
164
|
-
const existing = human.facts.find(f => f.id === item.id)!;
|
|
165
|
-
state.human_fact_upsert({
|
|
166
|
-
...existing,
|
|
167
|
-
name: item.name,
|
|
168
|
-
description: item.description,
|
|
169
|
-
sentiment: item.sentiment ?? existing.sentiment,
|
|
170
|
-
last_updated: now,
|
|
171
|
-
embedding,
|
|
172
|
-
});
|
|
173
|
-
break;
|
|
174
|
-
}
|
|
175
161
|
case "topic": {
|
|
176
162
|
const existing = human.topics.find(t => t.id === item.id)!;
|
|
177
163
|
state.human_topic_upsert({
|
|
@@ -228,14 +214,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
228
214
|
};
|
|
229
215
|
|
|
230
216
|
switch (item.type) {
|
|
231
|
-
case "fact": {
|
|
232
|
-
const fact: Fact = {
|
|
233
|
-
...baseFields,
|
|
234
|
-
validated_date: now,
|
|
235
|
-
};
|
|
236
|
-
state.human_fact_upsert(fact);
|
|
237
|
-
break;
|
|
238
|
-
}
|
|
239
217
|
case "topic": {
|
|
240
218
|
if (!item.category) {
|
|
241
219
|
console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);
|
|
@@ -145,14 +145,14 @@ export async function getQuotesForMessage(sm: StateManager, messageId: string):
|
|
|
145
145
|
export async function searchHumanData(
|
|
146
146
|
sm: StateManager,
|
|
147
147
|
query: string,
|
|
148
|
-
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number } = {}
|
|
148
|
+
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
|
|
149
149
|
): Promise<{
|
|
150
150
|
facts: Fact[];
|
|
151
151
|
topics: Topic[];
|
|
152
152
|
people: Person[];
|
|
153
153
|
quotes: Quote[];
|
|
154
154
|
}> {
|
|
155
|
-
const { types = ["fact", "topic", "person", "quote"], limit = 10 } = options;
|
|
155
|
+
const { types = ["fact", "topic", "person", "quote"], limit = 10, recent } = options;
|
|
156
156
|
const human = sm.getHuman();
|
|
157
157
|
const SIMILARITY_THRESHOLD = 0.3;
|
|
158
158
|
|
|
@@ -163,30 +163,54 @@ export async function searchHumanData(
|
|
|
163
163
|
quotes: [] as Quote[],
|
|
164
164
|
};
|
|
165
165
|
|
|
166
|
+
const recentSort = <T extends { last_updated?: string; last_mentioned?: string }>(items: T[]): T[] =>
|
|
167
|
+
[...items].sort((a, b) => {
|
|
168
|
+
const aDate = a.last_mentioned ?? a.last_updated ?? "";
|
|
169
|
+
const bDate = b.last_mentioned ?? b.last_updated ?? "";
|
|
170
|
+
return bDate.localeCompare(aDate);
|
|
171
|
+
});
|
|
172
|
+
|
|
166
173
|
let queryVector: number[] | null = null;
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
174
|
+
if (query) {
|
|
175
|
+
try {
|
|
176
|
+
const embeddingService = getEmbeddingService();
|
|
177
|
+
queryVector = await embeddingService.embed(query);
|
|
178
|
+
} catch (err) {
|
|
179
|
+
console.warn("[searchHumanData] Failed to generate query embedding:", err);
|
|
180
|
+
}
|
|
172
181
|
}
|
|
173
182
|
|
|
174
|
-
const searchItems = <T extends { id: string; embedding?: number[] }>(
|
|
183
|
+
const searchItems = <T extends { id: string; embedding?: number[]; last_updated?: string; last_mentioned?: string }>(
|
|
175
184
|
items: T[],
|
|
176
185
|
textExtractor: (item: T) => string
|
|
177
186
|
): T[] => {
|
|
187
|
+
if (recent && !query) {
|
|
188
|
+
return recentSort(items).slice(0, limit);
|
|
189
|
+
}
|
|
190
|
+
|
|
178
191
|
const withEmbeddings = items.filter((i) => i.embedding?.length);
|
|
179
192
|
|
|
180
193
|
if (queryVector && withEmbeddings.length > 0) {
|
|
181
|
-
|
|
194
|
+
const topK = recent ? Math.max(limit * 5, 50) : limit;
|
|
195
|
+
const found = findTopK(queryVector, withEmbeddings, topK)
|
|
182
196
|
.filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
|
|
183
197
|
.map(({ item }) => item);
|
|
198
|
+
if (recent) {
|
|
199
|
+
return recentSort(found).slice(0, limit);
|
|
200
|
+
}
|
|
201
|
+
return found;
|
|
184
202
|
}
|
|
185
203
|
|
|
204
|
+
if (!query) return [];
|
|
205
|
+
|
|
186
206
|
const lowerQuery = query.toLowerCase();
|
|
187
|
-
|
|
207
|
+
const found = items
|
|
188
208
|
.filter((i) => textExtractor(i).toLowerCase().includes(lowerQuery))
|
|
189
|
-
.slice(0, limit);
|
|
209
|
+
.slice(0, recent ? Math.max(limit * 5, 50) : limit);
|
|
210
|
+
if (recent) {
|
|
211
|
+
return recentSort(found).slice(0, limit);
|
|
212
|
+
}
|
|
213
|
+
return found;
|
|
190
214
|
};
|
|
191
215
|
|
|
192
216
|
if (types.includes("fact")) {
|
|
@@ -616,12 +616,6 @@ export function queueRewritePhase(state: StateManager): void {
|
|
|
616
616
|
|
|
617
617
|
const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
|
|
618
618
|
|
|
619
|
-
for (const fact of human.facts) {
|
|
620
|
-
if ((fact.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
621
|
-
itemsToScan.push({ item: fact, type: "fact" });
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
619
|
for (const topic of human.topics) {
|
|
626
620
|
if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
627
621
|
itemsToScan.push({ item: topic, type: "topic" });
|
|
@@ -20,7 +20,7 @@ interface Cluster {
|
|
|
20
20
|
// DEDUP CANDIDATE FINDING (copied from ceremony.ts)
|
|
21
21
|
// =============================================================================
|
|
22
22
|
|
|
23
|
-
const DEDUP_DEFAULT_THRESHOLD = 0.
|
|
23
|
+
const DEDUP_DEFAULT_THRESHOLD = 0.95; // Raised from 0.90: Ei's topic corpus is a single dense project domain — mega-cluster persists all the way to 0.92. At 0.95, max cluster drops to 7 items.
|
|
24
24
|
|
|
25
25
|
function findDedupCandidates<T extends DedupableItem>(
|
|
26
26
|
items: T[],
|
|
@@ -139,7 +139,6 @@ export function queueDedupPhase(state: StateManager): void {
|
|
|
139
139
|
console.log(`[Dedup] Starting deduplication phase (threshold: ${threshold})`);
|
|
140
140
|
|
|
141
141
|
const entityTypes: Array<{ type: DataItemType; items: DedupableItem[] }> = [
|
|
142
|
-
{ type: "fact", items: human.facts },
|
|
143
142
|
{ type: "topic", items: human.topics },
|
|
144
143
|
{ type: "person", items: human.people },
|
|
145
144
|
];
|
|
@@ -178,7 +177,7 @@ export function queueDedupPhase(state: StateManager): void {
|
|
|
178
177
|
// Build prompt
|
|
179
178
|
const prompt = buildDedupPrompt({
|
|
180
179
|
cluster: clusterEntities,
|
|
181
|
-
itemType: type,
|
|
180
|
+
itemType: type as "topic" | "person",
|
|
182
181
|
similarityRange: { min: cluster.minSim, max: cluster.maxSim }
|
|
183
182
|
});
|
|
184
183
|
|
|
@@ -313,19 +313,26 @@ export async function queueTopicMatch(
|
|
|
313
313
|
}));
|
|
314
314
|
|
|
315
315
|
console.log(`[queueTopicMatch] Embedding search: ${topicsWithEmbeddings.length} topics → ${topKItems.length} candidates`);
|
|
316
|
+
if (topKItems.length > 0) state.embedding_setWarning(false);
|
|
316
317
|
} catch (err) {
|
|
317
|
-
console.error(`[queueTopicMatch] Embedding search failed, falling back to
|
|
318
|
+
console.error(`[queueTopicMatch] Embedding search failed, falling back to recent topics:`, err);
|
|
319
|
+
state.embedding_setWarning(true);
|
|
318
320
|
}
|
|
319
321
|
}
|
|
320
322
|
|
|
321
323
|
if (topKItems.length === 0) {
|
|
322
|
-
|
|
323
|
-
|
|
324
|
+
const sorted = [...human.topics].sort((a, b) => {
|
|
325
|
+
const aDate = a.last_mentioned ?? a.last_updated;
|
|
326
|
+
const bDate = b.last_mentioned ?? b.last_updated;
|
|
327
|
+
return bDate.localeCompare(aDate);
|
|
328
|
+
});
|
|
329
|
+
topKItems = sorted.slice(0, EMBEDDING_TOP_K).map(t => ({
|
|
324
330
|
id: t.id,
|
|
325
331
|
name: t.name,
|
|
326
332
|
description: t.description,
|
|
327
333
|
category: t.category,
|
|
328
334
|
}));
|
|
335
|
+
console.log(`[queueTopicMatch] No embedding matches, using ${topKItems.length} most-recent topics`);
|
|
329
336
|
}
|
|
330
337
|
|
|
331
338
|
const prompt = buildTopicMatchPrompt({
|
|
@@ -388,19 +395,26 @@ export async function queuePersonMatch(
|
|
|
388
395
|
}));
|
|
389
396
|
|
|
390
397
|
console.log(`[queuePersonMatch] Embedding search: ${peopleWithEmbeddings.length} people → ${topKItems.length} candidates`);
|
|
398
|
+
if (topKItems.length > 0) state.embedding_setWarning(false);
|
|
391
399
|
} catch (err) {
|
|
392
|
-
console.error(`[queuePersonMatch] Embedding search failed, falling back to
|
|
400
|
+
console.error(`[queuePersonMatch] Embedding search failed, falling back to recent people:`, err);
|
|
401
|
+
state.embedding_setWarning(true);
|
|
393
402
|
}
|
|
394
403
|
}
|
|
395
404
|
|
|
396
405
|
if (topKItems.length === 0) {
|
|
397
|
-
|
|
398
|
-
|
|
406
|
+
const sorted = [...human.people].sort((a, b) => {
|
|
407
|
+
const aDate = a.last_mentioned ?? a.last_updated;
|
|
408
|
+
const bDate = b.last_mentioned ?? b.last_updated;
|
|
409
|
+
return bDate.localeCompare(aDate);
|
|
410
|
+
});
|
|
411
|
+
topKItems = sorted.slice(0, EMBEDDING_TOP_K).map(p => ({
|
|
399
412
|
id: p.id,
|
|
400
413
|
name: p.name,
|
|
401
414
|
description: p.description,
|
|
402
415
|
relationship: p.relationship,
|
|
403
416
|
}));
|
|
417
|
+
console.log(`[queuePersonMatch] No embedding matches, using ${topKItems.length} most-recent people`);
|
|
404
418
|
}
|
|
405
419
|
|
|
406
420
|
const prompt = buildPersonMatchPrompt({
|
package/src/core/processor.ts
CHANGED
|
@@ -108,6 +108,7 @@ const DEFAULT_LOOP_INTERVAL_MS = 100;
|
|
|
108
108
|
const DEFAULT_CONTEXT_WINDOW_HOURS = 8;
|
|
109
109
|
const DEFAULT_OPENCODE_POLLING_MS = 1800000;
|
|
110
110
|
const DEFAULT_CLAUDE_CODE_POLLING_MS = 1800000;
|
|
111
|
+
const DEFAULT_CURSOR_POLLING_MS = 1800000;
|
|
111
112
|
|
|
112
113
|
let processorInstanceCount = 0;
|
|
113
114
|
|
|
@@ -128,6 +129,8 @@ export class Processor {
|
|
|
128
129
|
private openCodeImportInProgress = false;
|
|
129
130
|
private lastClaudeCodeSync = 0;
|
|
130
131
|
private claudeCodeImportInProgress = false;
|
|
132
|
+
private lastCursorSync = 0;
|
|
133
|
+
private cursorImportInProgress = false;
|
|
131
134
|
private pendingConflict: StateConflictData | null = null;
|
|
132
135
|
private storage: Storage | null = null;
|
|
133
136
|
private importAbortController = new AbortController();
|
|
@@ -278,19 +281,20 @@ export class Processor {
|
|
|
278
281
|
name: "read_memory",
|
|
279
282
|
display_name: "Read Memory",
|
|
280
283
|
description:
|
|
281
|
-
"Search your personal memory for relevant facts,
|
|
284
|
+
"Search your personal memory for relevant facts, topics, people, or quotes. Use this when you need information about the user that may not be in the current conversation. Use `recent: true` to retrieve what's been discussed recently.",
|
|
282
285
|
input_schema: {
|
|
283
286
|
type: "object",
|
|
284
287
|
properties: {
|
|
285
288
|
query: { type: "string", description: "What to search for in memory" },
|
|
286
289
|
types: {
|
|
287
290
|
type: "array",
|
|
288
|
-
items: { type: "string", enum: ["fact", "
|
|
291
|
+
items: { type: "string", enum: ["fact", "topic", "person", "quote"] },
|
|
289
292
|
description: "Limit search to specific memory types (default: all types)",
|
|
290
293
|
},
|
|
291
294
|
limit: { type: "number", description: "Max results to return (default: 10, max: 20)" },
|
|
295
|
+
recent: { type: "boolean", description: "If true, return recently-mentioned results sorted by last_mentioned date instead of relevance. Combine with a query to filter recent results by topic." },
|
|
292
296
|
},
|
|
293
|
-
required: [
|
|
297
|
+
required: [],
|
|
294
298
|
},
|
|
295
299
|
runtime: "any",
|
|
296
300
|
builtin: true,
|
|
@@ -449,6 +453,30 @@ export class Processor {
|
|
|
449
453
|
});
|
|
450
454
|
}
|
|
451
455
|
|
|
456
|
+
// web_fetch tool
|
|
457
|
+
if (!this.stateManager.tools_getByName("web_fetch")) {
|
|
458
|
+
this.stateManager.tools_add({
|
|
459
|
+
id: crypto.randomUUID(),
|
|
460
|
+
provider_id: "ei",
|
|
461
|
+
name: "web_fetch",
|
|
462
|
+
display_name: "Web Fetch",
|
|
463
|
+
description:
|
|
464
|
+
"Fetch content from a URL and return the text. Useful for reading web pages, documentation, or public APIs. HTML is stripped to plain text. Only available in the TUI.",
|
|
465
|
+
input_schema: {
|
|
466
|
+
type: "object",
|
|
467
|
+
properties: {
|
|
468
|
+
url: { type: "string", description: "The URL to fetch (http or https only)" },
|
|
469
|
+
},
|
|
470
|
+
required: ["url"],
|
|
471
|
+
},
|
|
472
|
+
runtime: "node",
|
|
473
|
+
builtin: true,
|
|
474
|
+
enabled: true,
|
|
475
|
+
created_at: now,
|
|
476
|
+
max_calls_per_interaction: 3,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
|
|
452
480
|
// --- Tavily Search provider ---
|
|
453
481
|
if (!this.stateManager.tools_getProviderById("tavily")) {
|
|
454
482
|
const tavilyProvider: ToolProvider = {
|
|
@@ -876,6 +904,14 @@ const toolNextSteps = new Set([
|
|
|
876
904
|
await this.checkAndSyncClaudeCode(human, now);
|
|
877
905
|
}
|
|
878
906
|
|
|
907
|
+
if (
|
|
908
|
+
this.isTUI &&
|
|
909
|
+
human.settings?.cursor?.integration &&
|
|
910
|
+
this.stateManager.queue_length() === 0
|
|
911
|
+
) {
|
|
912
|
+
await this.checkAndSyncCursor(human, now);
|
|
913
|
+
}
|
|
914
|
+
|
|
879
915
|
if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
|
|
880
916
|
if (human.settings?.sync && remoteSync.isConfigured()) {
|
|
881
917
|
const state = this.stateManager.getStorageState();
|
|
@@ -1062,6 +1098,59 @@ const toolNextSteps = new Set([
|
|
|
1062
1098
|
});
|
|
1063
1099
|
}
|
|
1064
1100
|
|
|
1101
|
+
private async checkAndSyncCursor(human: HumanEntity, now: number): Promise<void> {
|
|
1102
|
+
if (this.cursorImportInProgress) {
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
const cursor = human.settings?.cursor;
|
|
1107
|
+
const pollingInterval = cursor?.polling_interval_ms ?? DEFAULT_CURSOR_POLLING_MS;
|
|
1108
|
+
const lastSync = cursor?.last_sync ? new Date(cursor.last_sync).getTime() : 0;
|
|
1109
|
+
const timeSinceSync = now - lastSync;
|
|
1110
|
+
|
|
1111
|
+
if (timeSinceSync < pollingInterval && this.lastCursorSync > 0) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
this.lastCursorSync = now;
|
|
1116
|
+
const syncTimestamp = new Date().toISOString();
|
|
1117
|
+
this.stateManager.setHuman({
|
|
1118
|
+
...this.stateManager.getHuman(),
|
|
1119
|
+
settings: {
|
|
1120
|
+
...this.stateManager.getHuman().settings,
|
|
1121
|
+
cursor: {
|
|
1122
|
+
...cursor,
|
|
1123
|
+
last_sync: syncTimestamp,
|
|
1124
|
+
},
|
|
1125
|
+
},
|
|
1126
|
+
});
|
|
1127
|
+
|
|
1128
|
+
this.cursorImportInProgress = true;
|
|
1129
|
+
import("../integrations/cursor/importer.js")
|
|
1130
|
+
.then(({ importCursorSessions }) =>
|
|
1131
|
+
importCursorSessions({
|
|
1132
|
+
stateManager: this.stateManager,
|
|
1133
|
+
interface: this.interface,
|
|
1134
|
+
signal: this.importAbortController.signal,
|
|
1135
|
+
})
|
|
1136
|
+
)
|
|
1137
|
+
.then((result) => {
|
|
1138
|
+
if (result.sessionsProcessed > 0) {
|
|
1139
|
+
console.log(
|
|
1140
|
+
`[Processor] Cursor sync complete: ${result.sessionsProcessed} sessions, ` +
|
|
1141
|
+
`${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
|
|
1142
|
+
`${result.extractionScansQueued} extraction scans queued`
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
})
|
|
1146
|
+
.catch((err) => {
|
|
1147
|
+
console.warn(`[Processor] Cursor sync failed:`, err);
|
|
1148
|
+
})
|
|
1149
|
+
.finally(() => {
|
|
1150
|
+
this.cursorImportInProgress = false;
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1065
1154
|
private classifyLLMError(error: string): string {
|
|
1066
1155
|
const match = error.match(/\((\d{3})\)/);
|
|
1067
1156
|
if (match) {
|
|
@@ -1396,7 +1485,7 @@ const toolNextSteps = new Set([
|
|
|
1396
1485
|
|
|
1397
1486
|
async searchHumanData(
|
|
1398
1487
|
query: string,
|
|
1399
|
-
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number } = {}
|
|
1488
|
+
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
|
|
1400
1489
|
): Promise<{
|
|
1401
1490
|
facts: Fact[];
|
|
1402
1491
|
topics: Topic[];
|
|
@@ -30,6 +30,7 @@ export class StateManager {
|
|
|
30
30
|
private persistenceState = new PersistenceState();
|
|
31
31
|
private providers: ToolProvider[] = [];
|
|
32
32
|
private tools: ToolDefinition[] = [];
|
|
33
|
+
private embeddingWarning = false;
|
|
33
34
|
|
|
34
35
|
async initialize(storage: Storage): Promise<void> {
|
|
35
36
|
this.persistenceState.setStorage(storage);
|
|
@@ -479,6 +480,14 @@ export class StateManager {
|
|
|
479
480
|
return this.queueState.dlqLength();
|
|
480
481
|
}
|
|
481
482
|
|
|
483
|
+
embedding_setWarning(warned: boolean): void {
|
|
484
|
+
this.embeddingWarning = warned;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
embedding_getWarning(): boolean {
|
|
488
|
+
return this.embeddingWarning;
|
|
489
|
+
}
|
|
490
|
+
|
|
482
491
|
queue_getDLQItems(): LLMRequest[] {
|
|
483
492
|
return this.queueState.getDLQItems();
|
|
484
493
|
}
|
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* read_memory builtin tool
|
|
3
|
-
*
|
|
4
|
-
* Delegates to Processor.searchHumanData() — no external call, runtime: "any".
|
|
5
|
-
* The searchHumanData function is injected at construction to avoid circular deps.
|
|
6
|
-
*/
|
|
7
1
|
import type { ToolExecutor } from "../types.js";
|
|
8
2
|
import type { Fact, Topic, Person, Quote } from "../../types.js";
|
|
9
3
|
|
|
10
4
|
type SearchHumanData = (
|
|
11
5
|
query: string,
|
|
12
|
-
options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number }
|
|
6
|
+
options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean }
|
|
13
7
|
) => Promise<{ facts: Fact[]; topics: Topic[]; people: Person[]; quotes: Quote[] }>;
|
|
14
8
|
|
|
15
9
|
export function createReadMemoryExecutor(searchHumanData: SearchHumanData): ToolExecutor {
|
|
@@ -18,10 +12,12 @@ export function createReadMemoryExecutor(searchHumanData: SearchHumanData): Tool
|
|
|
18
12
|
|
|
19
13
|
async execute(args: Record<string, unknown>): Promise<string> {
|
|
20
14
|
const query = typeof args.query === "string" ? args.query.trim() : "";
|
|
21
|
-
|
|
22
|
-
|
|
15
|
+
const recent = args.recent === true;
|
|
16
|
+
console.log(`[read_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}, recent=${recent}`);
|
|
17
|
+
|
|
18
|
+
if (!query && !recent) {
|
|
23
19
|
console.warn("[read_memory] missing query argument");
|
|
24
|
-
return JSON.stringify({ error: "Missing required argument: query" });
|
|
20
|
+
return JSON.stringify({ error: "Missing required argument: query (or use recent: true)" });
|
|
25
21
|
}
|
|
26
22
|
|
|
27
23
|
const types = Array.isArray(args.types)
|
|
@@ -32,7 +28,7 @@ export function createReadMemoryExecutor(searchHumanData: SearchHumanData): Tool
|
|
|
32
28
|
|
|
33
29
|
const limit = typeof args.limit === "number" && args.limit > 0 ? Math.min(args.limit, 20) : 10;
|
|
34
30
|
|
|
35
|
-
const results = await searchHumanData(query, { types, limit });
|
|
31
|
+
const results = await searchHumanData(query, { types, limit, recent });
|
|
36
32
|
|
|
37
33
|
const total = results.facts.length + results.topics.length + results.people.length + results.quotes.length;
|
|
38
34
|
console.log(`[read_memory] query="${query}" => ${total} results (facts=${results.facts.length}, topics=${results.topics.length}, people=${results.people.length}, quotes=${results.quotes.length})`);
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { ToolExecutor } from "../types.js";
|
|
2
|
+
|
|
3
|
+
const MAX_CHARS = 20000;
|
|
4
|
+
|
|
5
|
+
export const webFetchExecutor: ToolExecutor = {
|
|
6
|
+
name: "web_fetch",
|
|
7
|
+
|
|
8
|
+
async execute(args: Record<string, unknown>): Promise<string> {
|
|
9
|
+
const url = typeof args.url === "string" ? args.url.trim() : "";
|
|
10
|
+
if (!url) {
|
|
11
|
+
return JSON.stringify({ error: "Missing required argument: url" });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
let parsedUrl: URL;
|
|
15
|
+
try {
|
|
16
|
+
parsedUrl = new URL(url);
|
|
17
|
+
} catch {
|
|
18
|
+
return JSON.stringify({ error: `Invalid URL: ${url}` });
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
if (!["http:", "https:"].includes(parsedUrl.protocol)) {
|
|
22
|
+
return JSON.stringify({ error: "Only http and https URLs are supported" });
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
try {
|
|
26
|
+
console.log(`[web_fetch] fetching ${url}`);
|
|
27
|
+
const response = await fetch(url, {
|
|
28
|
+
headers: {
|
|
29
|
+
"User-Agent": "Ei/1.0 (AI companion; +https://github.com/Flare576/ei)",
|
|
30
|
+
"Accept": "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8",
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
if (!response.ok) {
|
|
35
|
+
return JSON.stringify({ error: `HTTP ${response.status}: ${response.statusText}`, url });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
39
|
+
let text = await response.text();
|
|
40
|
+
|
|
41
|
+
// Strip HTML noise for readability
|
|
42
|
+
if (contentType.includes("text/html")) {
|
|
43
|
+
text = text
|
|
44
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
45
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
46
|
+
.replace(/<[^>]+>/g, " ")
|
|
47
|
+
.replace(/ /g, " ")
|
|
48
|
+
.replace(/&/g, "&")
|
|
49
|
+
.replace(/</g, "<")
|
|
50
|
+
.replace(/>/g, ">")
|
|
51
|
+
.replace(/"/g, '"')
|
|
52
|
+
.replace(/\s+/g, " ")
|
|
53
|
+
.trim();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const truncated = text.length > MAX_CHARS;
|
|
57
|
+
if (truncated) text = text.slice(0, MAX_CHARS);
|
|
58
|
+
|
|
59
|
+
console.log(`[web_fetch] ${url} => ${text.length} chars${truncated ? " (truncated)" : ""}`);
|
|
60
|
+
|
|
61
|
+
return JSON.stringify({
|
|
62
|
+
url,
|
|
63
|
+
content_type: contentType,
|
|
64
|
+
content: text,
|
|
65
|
+
...(truncated ? { truncated: true, note: `Content truncated to ${MAX_CHARS} characters` } : {}),
|
|
66
|
+
});
|
|
67
|
+
} catch (err) {
|
|
68
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
69
|
+
console.warn(`[web_fetch] failed for ${url}: ${msg}`);
|
|
70
|
+
return JSON.stringify({ error: `Fetch failed: ${msg}`, url });
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
};
|
package/src/core/tools/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { ToolCall, ToolResult, ToolExecutor } from "./types.js";
|
|
|
11
11
|
import { tavilyWebSearchExecutor, tavilyNewsSearchExecutor } from "./builtin/web-search.js";
|
|
12
12
|
import { currentlyPlayingExecutor } from "./builtin/currently-playing.js";
|
|
13
13
|
import { likedSongsExecutor } from "./builtin/spotify-liked-songs.js";
|
|
14
|
+
import { webFetchExecutor } from "./builtin/web-fetch.js";
|
|
14
15
|
// file-read and list-directory are Node-only — imported lazily via registerFileReadExecutor() to avoid
|
|
15
16
|
// file-read and list-directory are Node-only — imported lazily via registerFileReadExecutor() to avoid
|
|
16
17
|
|
|
@@ -37,6 +38,7 @@ registerExecutor(tavilyWebSearchExecutor);
|
|
|
37
38
|
registerExecutor(tavilyNewsSearchExecutor);
|
|
38
39
|
registerExecutor(currentlyPlayingExecutor);
|
|
39
40
|
registerExecutor(likedSongsExecutor);
|
|
41
|
+
registerExecutor(webFetchExecutor);
|
|
40
42
|
// file_read and list_directory are registered lazily via registerFileReadExecutor() — Node/TUI only.
|
|
41
43
|
|
|
42
44
|
/**
|
|
@@ -10,6 +10,7 @@ export interface DataItemBase {
|
|
|
10
10
|
description: string;
|
|
11
11
|
sentiment: number;
|
|
12
12
|
last_updated: string;
|
|
13
|
+
last_mentioned?: string; // Set by extraction only, never ceremony. Used for --recent sorting.
|
|
13
14
|
learned_by?: string; // Persona ID that originally learned this item (stable UUID)
|
|
14
15
|
last_changed_by?: string; // Persona ID that most recently updated this item (stable UUID)
|
|
15
16
|
persona_groups?: string[];
|
|
@@ -86,6 +86,7 @@ export interface HumanSettings {
|
|
|
86
86
|
ceremony?: CeremonyConfig;
|
|
87
87
|
backup?: BackupConfig;
|
|
88
88
|
claudeCode?: import("../../integrations/claude-code/types.js").ClaudeCodeSettings;
|
|
89
|
+
cursor?: import("../../integrations/cursor/types.js").CursorSettings;
|
|
89
90
|
}
|
|
90
91
|
|
|
91
92
|
export interface HumanEntity {
|
|
@@ -69,6 +69,8 @@ export interface QueueStatus {
|
|
|
69
69
|
pending_count: number;
|
|
70
70
|
dlq_count: number;
|
|
71
71
|
current_operation?: string;
|
|
72
|
+
/** True when the embedding service failed and topic/person matching fell back to recent items. */
|
|
73
|
+
embedding_warning?: boolean;
|
|
72
74
|
}
|
|
73
75
|
|
|
74
76
|
export interface EiError {
|
|
@@ -158,6 +158,10 @@ function updateProcessedState(
|
|
|
158
158
|
...human.settings,
|
|
159
159
|
claudeCode: {
|
|
160
160
|
...human.settings?.claudeCode,
|
|
161
|
+
// extraction_point is a progress indicator for user visibility only —
|
|
162
|
+
// it does NOT gate imports. processed_sessions is the sole source of
|
|
163
|
+
// truth for which sessions have been seen and when.
|
|
164
|
+
extraction_point: session.lastMessageAt,
|
|
161
165
|
processed_sessions: processedSessions,
|
|
162
166
|
},
|
|
163
167
|
},
|
|
@@ -212,7 +216,8 @@ export async function importClaudeCodeSessions(
|
|
|
212
216
|
|
|
213
217
|
// ─── Step 2: Find next unprocessed session ────────────────────────────
|
|
214
218
|
const human = stateManager.getHuman();
|
|
215
|
-
const
|
|
219
|
+
const settings = human.settings?.claudeCode;
|
|
220
|
+
const processedSessions = settings?.processed_sessions ?? {};
|
|
216
221
|
const now = Date.now();
|
|
217
222
|
|
|
218
223
|
let targetSession: ClaudeCodeSession | null = null;
|
|
@@ -161,5 +161,6 @@ export interface ClaudeCodeSettings {
|
|
|
161
161
|
extraction_model?: string; // "Provider:model" for extraction. Unset = uses default_model.
|
|
162
162
|
extraction_token_limit?: number; // Token budget for extraction chunking. Unset = resolved from model.
|
|
163
163
|
last_sync?: string; // ISO timestamp
|
|
164
|
+
extraction_point?: string; // ISO timestamp - floor cursor for processed-session skip
|
|
164
165
|
processed_sessions?: Record<string, string>; // sessionId → ISO timestamp of last import
|
|
165
166
|
}
|