ei-tui 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +3 -1
  2. package/package.json +2 -21
  3. package/src/cli/README.md +42 -14
  4. package/src/cli/mcp.ts +237 -0
  5. package/src/cli.ts +17 -51
  6. package/src/core/handlers/dedup.ts +4 -15
  7. package/src/core/handlers/document-segmentation.ts +2 -3
  8. package/src/core/handlers/heartbeat.ts +5 -10
  9. package/src/core/handlers/human-extraction.ts +6 -0
  10. package/src/core/handlers/human-matching.ts +53 -10
  11. package/src/core/handlers/index.ts +2 -0
  12. package/src/core/handlers/knowledge-synthesis.ts +50 -0
  13. package/src/core/handlers/persona-generation.ts +4 -8
  14. package/src/core/handlers/persona-response.ts +3 -4
  15. package/src/core/handlers/persona-topics.ts +2 -4
  16. package/src/core/handlers/rewrite.ts +26 -9
  17. package/src/core/handlers/rooms.ts +6 -12
  18. package/src/core/llm-client.ts +53 -7
  19. package/src/core/message-manager.ts +2 -4
  20. package/src/core/orchestrators/ceremony.ts +44 -13
  21. package/src/core/orchestrators/human-extraction.ts +38 -1
  22. package/src/core/orchestrators/index.ts +1 -0
  23. package/src/core/processor.ts +192 -41
  24. package/src/core/prompt-context-builder.ts +1 -0
  25. package/src/core/queue-manager.ts +10 -0
  26. package/src/core/queue-processor.ts +13 -4
  27. package/src/core/state-manager.ts +35 -0
  28. package/src/core/tools/builtin/fetch-memory.ts +92 -0
  29. package/src/core/tools/builtin/fetch-message.ts +123 -0
  30. package/src/core/tools/builtin/find-memory.ts +99 -0
  31. package/src/core/tools/index.ts +88 -5
  32. package/src/core/tools/types.ts +1 -1
  33. package/src/core/types/data-items.ts +1 -1
  34. package/src/core/types/entities.ts +7 -1
  35. package/src/core/types/enums.ts +1 -0
  36. package/src/core/types/integrations.ts +3 -1
  37. package/src/integrations/claude-code/importer.ts +6 -0
  38. package/src/integrations/cursor/importer.ts +6 -0
  39. package/src/integrations/document/unsource.ts +5 -3
  40. package/src/integrations/opencode/importer.ts +13 -1
  41. package/src/integrations/persona-history/importer.ts +12 -1
  42. package/src/prompts/ceremony/dedup.ts +3 -3
  43. package/src/prompts/ceremony/people-rewrite.ts +2 -2
  44. package/src/prompts/ceremony/topic-rewrite.ts +2 -2
  45. package/src/prompts/ceremony/types.ts +1 -1
  46. package/src/prompts/human/person-scan.ts +17 -0
  47. package/src/prompts/human/types.ts +4 -0
  48. package/src/prompts/index.ts +3 -0
  49. package/src/prompts/response/sections.ts +14 -7
  50. package/src/prompts/response/types.ts +1 -0
  51. package/src/prompts/synthesis/index.ts +101 -0
  52. package/src/prompts/synthesis/types.ts +26 -0
  53. package/tui/src/commands/generate.tsx +98 -0
  54. package/tui/src/commands/unsource.tsx +17 -10
  55. package/tui/src/components/GeneratedDocsOverlay.tsx +136 -0
  56. package/tui/src/components/PromptInput.tsx +2 -0
  57. package/tui/src/context/ei.tsx +49 -2
  58. package/tui/src/util/logger.ts +22 -2
  59. package/tui/src/util/provider-detection.ts +5 -2
  60. package/tui/src/util/yaml-provider.ts +2 -8
  61. package/src/core/tools/builtin/read-memory.ts +0 -70
@@ -0,0 +1,92 @@
1
+ import type { ToolExecutor } from "../types.js";
2
+ import type { Fact, Topic, Person, Quote, HumanEntity } from "../../types.js";
3
+
4
+ type GetHuman = () => HumanEntity;
5
+
6
+ function cleanFact(f: Fact): Record<string, unknown> {
7
+ const { embedding, persona_groups, ...rest } = f;
8
+ void embedding; void persona_groups;
9
+ return rest;
10
+ }
11
+
12
+ function cleanTopic(t: Topic): Record<string, unknown> {
13
+ const { embedding, rewrite_length_floor, persona_groups, last_ei_asked, ...rest } = t;
14
+ void embedding; void rewrite_length_floor; void persona_groups; void last_ei_asked;
15
+ return rest;
16
+ }
17
+
18
+ function cleanPerson(p: Person): Record<string, unknown> {
19
+ const { embedding, rewrite_length_floor, persona_groups, last_ei_asked, ...rest } = p;
20
+ void embedding; void rewrite_length_floor; void persona_groups; void last_ei_asked;
21
+ return rest;
22
+ }
23
+
24
+ function cleanQuote(
25
+ q: Quote,
26
+ facts: Fact[],
27
+ topics: Topic[],
28
+ people: Person[]
29
+ ): Record<string, unknown> {
30
+ const { embedding, persona_groups, data_item_ids, ...rest } = q;
31
+ void embedding; void persona_groups;
32
+
33
+ const linked_items: Array<{ id: string; name: string; type: string }> = [];
34
+ for (const id of data_item_ids) {
35
+ const fact = facts.find(f => f.id === id);
36
+ if (fact) { linked_items.push({ id: fact.id, name: fact.name, type: "fact" }); continue; }
37
+ const topic = topics.find(t => t.id === id);
38
+ if (topic) { linked_items.push({ id: topic.id, name: topic.name, type: "topic" }); continue; }
39
+ const person = people.find(p => p.id === id);
40
+ if (person) { linked_items.push({ id: person.id, name: person.name, type: "person" }); continue; }
41
+ }
42
+
43
+ return { ...rest, linked_items };
44
+ }
45
+
46
+ export function createFetchMemoryExecutor(getHuman: GetHuman): ToolExecutor {
47
+ return {
48
+ name: "fetch_memory",
49
+
50
+ async execute(args: Record<string, unknown>): Promise<string> {
51
+ const id = typeof args.id === "string" ? args.id.trim() : "";
52
+ console.log(`[fetch_memory] called with id="${id}"`);
53
+
54
+ if (!id) {
55
+ console.warn("[fetch_memory] missing id argument");
56
+ return JSON.stringify({ error: "Missing required argument: id" });
57
+ }
58
+
59
+ const human = getHuman();
60
+
61
+ const fact = human.facts.find(f => f.id === id);
62
+ if (fact) {
63
+ console.log(`[fetch_memory] found fact id="${id}" name="${fact.name}"`);
64
+ return JSON.stringify({ type: "fact", ...cleanFact(fact) });
65
+ }
66
+
67
+ const topic = human.topics.find(t => t.id === id);
68
+ if (topic) {
69
+ console.log(`[fetch_memory] found topic id="${id}" name="${topic.name}"`);
70
+ return JSON.stringify({ type: "topic", ...cleanTopic(topic) });
71
+ }
72
+
73
+ const person = human.people.find(p => p.id === id);
74
+ if (person) {
75
+ console.log(`[fetch_memory] found person id="${id}" name="${person.name}"`);
76
+ return JSON.stringify({ type: "person", ...cleanPerson(person) });
77
+ }
78
+
79
+ const quote = human.quotes.find(q => q.id === id);
80
+ if (quote) {
81
+ console.log(`[fetch_memory] found quote id="${id}"`);
82
+ return JSON.stringify({
83
+ type: "quote",
84
+ ...cleanQuote(quote, human.facts, human.topics, human.people),
85
+ });
86
+ }
87
+
88
+ console.log(`[fetch_memory] no entity found for id="${id}"`);
89
+ return JSON.stringify({ error: "No accessible record found for this ID" });
90
+ },
91
+ };
92
+ }
@@ -0,0 +1,123 @@
1
+ import type { ToolExecutor } from "../types.js";
2
+ import type { Message } from "../../types.js";
3
+ import type { RoomMessage, RoomSummary } from "../../types/rooms.js";
4
+ import type { PersonaEntity } from "../../types/entities.js";
5
+
6
+ interface CleanMessage {
7
+ id: string;
8
+ role: string;
9
+ content?: string;
10
+ silence_reason?: string;
11
+ timestamp: string;
12
+ speaker_name?: string;
13
+ }
14
+
15
+ type GetAllPersonas = () => PersonaEntity[];
16
+ type GetPersonaMessages = (personaId: string) => Message[];
17
+ type GetRoomList = () => RoomSummary[];
18
+ type GetRoomMessages = (roomId: string) => RoomMessage[];
19
+ type GetRoomDisplayName = (roomId: string) => string | null;
20
+
21
+ function stripMessage(m: Message): CleanMessage {
22
+ return {
23
+ id: m.id,
24
+ role: m.role,
25
+ ...(m.content !== undefined ? { content: m.content } : {}),
26
+ ...(m.silence_reason !== undefined ? { silence_reason: m.silence_reason } : {}),
27
+ timestamp: m.timestamp,
28
+ ...(m.speaker_name !== undefined ? { speaker_name: m.speaker_name } : {}),
29
+ };
30
+ }
31
+
32
+ function stripRoomMessage(m: RoomMessage, personaDisplayName?: string): CleanMessage {
33
+ return {
34
+ id: m.id,
35
+ role: m.role,
36
+ ...(m.content !== undefined ? { content: m.content } : {}),
37
+ ...(m.silence_reason !== undefined ? { silence_reason: m.silence_reason } : {}),
38
+ timestamp: m.timestamp,
39
+ ...(personaDisplayName !== undefined ? { speaker_name: personaDisplayName } : {}),
40
+ };
41
+ }
42
+
43
+ export function createFetchMessageExecutor(
44
+ getAllPersonas: GetAllPersonas,
45
+ getPersonaMessages: GetPersonaMessages,
46
+ getRoomList: GetRoomList,
47
+ getRoomMessages: GetRoomMessages,
48
+ getRoomDisplayName: GetRoomDisplayName
49
+ ): ToolExecutor {
50
+ return {
51
+ name: "fetch_message",
52
+
53
+ async execute(args: Record<string, unknown>): Promise<string> {
54
+ const id = typeof args.id === "string" ? args.id.trim() : "";
55
+ const before = typeof args.before === "number" && args.before > 0 ? Math.floor(args.before) : 0;
56
+ const after = typeof args.after === "number" && args.after > 0 ? Math.floor(args.after) : 0;
57
+
58
+ console.log(`[fetch_message] called with id="${id}", before=${before}, after=${after}`);
59
+
60
+ if (!id) {
61
+ console.warn("[fetch_message] missing id argument");
62
+ return JSON.stringify({ error: "Missing required argument: id" });
63
+ }
64
+
65
+ const personas = getAllPersonas();
66
+
67
+ // TODO: add persona access gate when calling context is available —
68
+ // the execute() signature has no requestingPersonaId, so we search all personas.
69
+ for (const persona of personas) {
70
+ const messages = getPersonaMessages(persona.id);
71
+ const idx = messages.findIndex(m => m.id === id);
72
+ if (idx === -1) continue;
73
+
74
+ const msg = messages[idx];
75
+ const beforeMsgs = messages.slice(Math.max(0, idx - before), idx).map(stripMessage);
76
+ const afterMsgs = messages.slice(idx + 1, idx + 1 + after).map(stripMessage);
77
+
78
+ console.log(`[fetch_message] found in persona "${persona.display_name}" at idx=${idx}`);
79
+ return JSON.stringify({
80
+ message: stripMessage(msg),
81
+ before: beforeMsgs,
82
+ after: afterMsgs,
83
+ persona: persona.display_name,
84
+ });
85
+ }
86
+
87
+ // TODO: add persona access gate when calling context is available.
88
+ const rooms = getRoomList();
89
+ for (const roomSummary of rooms) {
90
+ const messages = getRoomMessages(roomSummary.id);
91
+ const idx = messages.findIndex(m => m.id === id);
92
+ if (idx === -1) continue;
93
+
94
+ const msg = messages[idx];
95
+ const roomDisplayName = getRoomDisplayName(roomSummary.id) ?? roomSummary.display_name;
96
+
97
+ const resolvePersonaName = (m: RoomMessage): string | undefined => {
98
+ if (m.role !== "persona" || !m.persona_id) return undefined;
99
+ const p = personas.find(pe => pe.id === m.persona_id);
100
+ return p?.display_name;
101
+ };
102
+
103
+ const beforeMsgs = messages
104
+ .slice(Math.max(0, idx - before), idx)
105
+ .map(m => stripRoomMessage(m, resolvePersonaName(m)));
106
+ const afterMsgs = messages
107
+ .slice(idx + 1, idx + 1 + after)
108
+ .map(m => stripRoomMessage(m, resolvePersonaName(m)));
109
+
110
+ console.log(`[fetch_message] found in room "${roomDisplayName}" at idx=${idx}`);
111
+ return JSON.stringify({
112
+ message: stripRoomMessage(msg, resolvePersonaName(msg)),
113
+ before: beforeMsgs,
114
+ after: afterMsgs,
115
+ persona: roomDisplayName,
116
+ });
117
+ }
118
+
119
+ console.log(`[fetch_message] message not found for id="${id}"`);
120
+ return JSON.stringify({ error: "Message not found" });
121
+ },
122
+ };
123
+ }
@@ -0,0 +1,99 @@
1
+ import type { ToolExecutor } from "../types.js";
2
+ import type { Fact, Topic, Person, Quote, HumanEntity } from "../../types.js";
3
+
4
+ interface PersonaSummary {
5
+ id: string;
6
+ display_name: string;
7
+ }
8
+
9
+ type SearchHumanData = (
10
+ query: string,
11
+ options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean; persona_filter?: string }
12
+ ) => Promise<{ facts: Fact[]; topics: Topic[]; people: Person[]; quotes: Quote[] }>;
13
+
14
+ type GetPersonaList = () => Promise<PersonaSummary[]>;
15
+
16
+ type GetHuman = () => HumanEntity;
17
+
18
+ export function createFindMemoryExecutor(searchHumanData: SearchHumanData, getPersonaList?: GetPersonaList, getHuman?: GetHuman): ToolExecutor {
19
+ return {
20
+ name: "find_memory",
21
+
22
+ async execute(args: Record<string, unknown>): Promise<string> {
23
+ const query = typeof args.query === "string" ? args.query.trim() : "";
24
+ const recent = args.recent === true;
25
+ const personaArg = typeof args.persona === "string" ? args.persona.trim() : "";
26
+ console.log(`[find_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}, recent=${recent}, persona="${personaArg}"`);
27
+
28
+ if (!query && !recent) {
29
+ console.warn("[find_memory] missing query argument");
30
+ return JSON.stringify({ error: "Missing required argument: query (or use recent: true)" });
31
+ }
32
+
33
+ const typeMap: Record<string, "fact" | "topic" | "person" | "quote"> = {
34
+ fact: "fact", facts: "fact",
35
+ topic: "topic", topics: "topic",
36
+ person: "person", people: "person",
37
+ quote: "quote", quotes: "quote",
38
+ };
39
+
40
+ const types = Array.isArray(args.types)
41
+ ? (args.types
42
+ .filter((t): t is string => typeof t === "string")
43
+ .map(t => typeMap[t])
44
+ .filter((t): t is "fact" | "topic" | "person" | "quote" => t !== undefined)
45
+ .filter((v, i, a) => a.indexOf(v) === i)
46
+ )
47
+ : undefined;
48
+
49
+ const limit = typeof args.limit === "number" && args.limit > 0 ? Math.min(args.limit, 20) : 10;
50
+
51
+ // Resolve persona display_name to ID
52
+ let persona_filter: string | undefined;
53
+ if (personaArg && getPersonaList) {
54
+ const personas = await getPersonaList();
55
+ const match = personas.find(p => p.display_name.toLowerCase() === personaArg.toLowerCase());
56
+ if (match) {
57
+ persona_filter = match.id;
58
+ console.log(`[find_memory] resolved persona "${personaArg}" to ID "${persona_filter}"`);
59
+ } else {
60
+ console.warn(`[find_memory] persona "${personaArg}" not found, proceeding without filter`);
61
+ }
62
+ }
63
+
64
+ const results = await searchHumanData(query, { types, limit, recent, persona_filter });
65
+
66
+ const total = results.facts.length + results.topics.length + results.people.length + results.quotes.length;
67
+ console.log(`[find_memory] query="${query}" => ${total} results (facts=${results.facts.length}, topics=${results.topics.length}, people=${results.people.length}, quotes=${results.quotes.length})`);
68
+
69
+ const output: Record<string, unknown[]> = {};
70
+ if (results.facts.length > 0) output.facts = results.facts.map(f => ({ id: f.id, name: f.name, description: f.description }));
71
+ if (results.topics.length > 0) output.topics = results.topics.map(t => ({ id: t.id, name: t.name, description: t.description }));
72
+ if (results.people.length > 0) output.people = results.people.map(p => ({ id: p.id, name: p.name, relationship: p.relationship, description: p.description, identifiers: p.identifiers ?? [] }));
73
+
74
+ if (results.quotes.length > 0) {
75
+ const human = getHuman ? getHuman() : null;
76
+ output.quotes = results.quotes.map(q => {
77
+ const linked_items: Array<{ id: string; name: string; type: string }> = [];
78
+ if (human && q.data_item_ids.length > 0) {
79
+ for (const itemId of q.data_item_ids) {
80
+ const fact = human.facts.find(f => f.id === itemId);
81
+ if (fact) { linked_items.push({ id: fact.id, name: fact.name, type: "fact" }); continue; }
82
+ const topic = human.topics.find(t => t.id === itemId);
83
+ if (topic) { linked_items.push({ id: topic.id, name: topic.name, type: "topic" }); continue; }
84
+ const person = human.people.find(p => p.id === itemId);
85
+ if (person) { linked_items.push({ id: person.id, name: person.name, type: "person" }); }
86
+ }
87
+ }
88
+ return { id: q.id, text: q.text, speaker: q.speaker, linked_items };
89
+ });
90
+ }
91
+
92
+ if (Object.keys(output).length === 0) {
93
+ return JSON.stringify({ result: "No relevant memories found for this query." });
94
+ }
95
+
96
+ return JSON.stringify(output);
97
+ },
98
+ };
99
+ }
@@ -18,6 +18,77 @@ import { webFetchExecutor } from "./builtin/web-fetch.js";
18
18
  /** Hard upper limit on total tool calls per interaction, regardless of individual limits. */
19
19
  export const HARD_TOOL_CALL_LIMIT = 10;
20
20
 
21
+ /**
22
+ * System tools — injected unconditionally into every LLM call that uses tools.
23
+ * NOT stored in state.json. NOT user-configurable. Do NOT count against HARD_TOOL_CALL_LIMIT.
24
+ * Enforce their own per-tool limits via max_calls_per_interaction.
25
+ */
26
+ export const SYSTEM_TOOLS: ToolDefinition[] = [
27
+ {
28
+ id: "builtin-find-memory",
29
+ provider_id: "ei",
30
+ name: "find_memory",
31
+ display_name: "Find Memory",
32
+ description: "Semantic search of your personal memory — facts, topics, people, and quotes learned across ALL conversations over time, not just this one. Use when the human references something from the past, mentions a person, or asks about a topic you might have learned about. Supports optional filters: types (array of 'facts', 'topics', 'people', 'quotes'), limit (1-20, default 10), recent (true = sort by recency), persona (filter to what a specific persona has learned — use display name).",
33
+ input_schema: {
34
+ type: "object",
35
+ properties: {
36
+ query: { type: "string", description: "What to search for" },
37
+ types: { type: "array", items: { type: "string", enum: ["facts", "topics", "people", "quotes"] }, description: "Filter to specific types" },
38
+ limit: { type: "number", description: "Max results (1-20, default 10)" },
39
+ recent: { type: "boolean", description: "Sort by most recently mentioned instead of relevance" },
40
+ persona: { type: "string", description: "Filter to what a specific persona has learned. Use their display name." },
41
+ },
42
+ required: ["query"],
43
+ },
44
+ runtime: "any",
45
+ builtin: true,
46
+ enabled: true,
47
+ created_at: new Date(0).toISOString(),
48
+ max_calls_per_interaction: 3,
49
+ },
50
+ {
51
+ id: "builtin-fetch-memory",
52
+ provider_id: "ei",
53
+ name: "fetch_memory",
54
+ display_name: "Fetch Memory",
55
+ description: "Retrieve the full record for a specific memory by its ID. Use when find_memory returns an item and you need its complete details, or when a system prompt references a memory ID. Returns the full Fact, Topic, Person, or Quote record.",
56
+ input_schema: {
57
+ type: "object",
58
+ properties: {
59
+ id: { type: "string", description: "The ID of the memory to retrieve" },
60
+ },
61
+ required: ["id"],
62
+ },
63
+ runtime: "any",
64
+ builtin: true,
65
+ enabled: true,
66
+ created_at: new Date(0).toISOString(),
67
+ max_calls_per_interaction: 10,
68
+ },
69
+ {
70
+ id: "builtin-fetch-message",
71
+ provider_id: "ei",
72
+ name: "fetch_message",
73
+ display_name: "Fetch Message",
74
+ description: "Retrieve a specific message by its ID, with optional surrounding context. Use when find_memory returns a quote with a message_id and you want to read the original conversation, or when a temporal anchor references a message ID. The 'before' and 'after' parameters return that many additional messages for context (default 0).",
75
+ input_schema: {
76
+ type: "object",
77
+ properties: {
78
+ id: { type: "string", description: "The message ID to retrieve" },
79
+ before: { type: "number", description: "Number of preceding messages to include for context (default 0)" },
80
+ after: { type: "number", description: "Number of following messages to include for context (default 0)" },
81
+ },
82
+ required: ["id"],
83
+ },
84
+ runtime: "any",
85
+ builtin: true,
86
+ enabled: true,
87
+ created_at: new Date(0).toISOString(),
88
+ max_calls_per_interaction: 5,
89
+ },
90
+ ];
91
+
21
92
  /** Default max calls per tool if not set on the ToolDefinition. */
22
93
  const DEFAULT_MAX_CALLS = 3;
23
94
 
@@ -32,7 +103,7 @@ export function registerExecutor(executor: ToolExecutor): void {
32
103
  executorRegistry.set(executor.name, executor);
33
104
  }
34
105
 
35
- // Register builtins. read_memory is registered lazily via registerReadMemoryExecutor()
106
+ // Register builtins. find_memory is registered lazily via registerFindMemoryExecutor()
36
107
  // because it requires Processor.searchHumanData injection.
37
108
  registerExecutor(tavilyWebSearchExecutor);
38
109
  registerExecutor(tavilyNewsSearchExecutor);
@@ -42,10 +113,18 @@ registerExecutor(webFetchExecutor);
42
113
  // file_read and list_directory are registered lazily via registerFileReadExecutor() — Node/TUI only.
43
114
 
44
115
  /**
45
- * Register the read_memory executor — called by Processor after it's initialized,
116
+ * Register the find_memory executor — called by Processor after it's initialized,
46
117
  * injecting its own searchHumanData method to avoid circular imports.
47
118
  */
48
- export function registerReadMemoryExecutor(executor: ToolExecutor): void {
119
+ export function registerFindMemoryExecutor(executor: ToolExecutor): void {
120
+ executorRegistry.set(executor.name, executor);
121
+ }
122
+
123
+ export function registerFetchMemoryExecutor(executor: ToolExecutor): void {
124
+ executorRegistry.set(executor.name, executor);
125
+ }
126
+
127
+ export function registerFetchMessageExecutor(executor: ToolExecutor): void {
49
128
  executorRegistry.set(executor.name, executor);
50
129
  }
51
130
 
@@ -122,7 +201,9 @@ export async function executeToolCalls(
122
201
  const toolsByName = new Map(tools.map(t => [t.name, t]));
123
202
 
124
203
  for (const call of calls) {
125
- if (totalCalls.count >= HARD_TOOL_CALL_LIMIT) {
204
+ const isSystemTool = SYSTEM_TOOLS.some(t => t.name === call.name);
205
+
206
+ if (!isSystemTool && totalCalls.count >= HARD_TOOL_CALL_LIMIT) {
126
207
  console.log(`[Tools] Hard limit (${HARD_TOOL_CALL_LIMIT}) reached — skipping remaining tool calls`);
127
208
  break;
128
209
  }
@@ -168,7 +249,9 @@ export async function executeToolCalls(
168
249
  }
169
250
 
170
251
  callCounts.set(call.name, currentCount + 1);
171
- totalCalls.count++;
252
+ if (!isSystemTool) {
253
+ totalCalls.count++;
254
+ }
172
255
 
173
256
  const newCount = currentCount + 1;
174
257
  if (newCount >= maxCalls) {
@@ -6,7 +6,7 @@
6
6
  /** A single tool call the LLM wants to make (from the API response). */
7
7
  export interface ToolCall {
8
8
  id: string; // call_abc123 — must be echoed back in tool result message
9
- name: string; // snake_case tool name ("web_search", "read_memory")
9
+ name: string; // snake_case tool name ("web_search", "find_memory")
10
10
  arguments: Record<string, unknown>; // Parsed from JSON string in the API response
11
11
  }
12
12
 
@@ -18,7 +18,7 @@ export interface DataItemBase {
18
18
  sources?: string[]; // Namespaced source identifiers — where items were learned from. Format: "provider:id" (e.g., "opencode:ses_abc123", "cursor:composerId"). Grow-only union.
19
19
  persona_groups?: string[];
20
20
  embedding?: number[];
21
- rewrite_checked?: boolean; // True after rewrite scan finds no changes. Cleared automatically when extraction upserts a fresh item.
21
+ rewrite_length_floor?: number; // Set after every rewrite scan: ceil(description.length * 1.1). Item is skipped by ceremony until description grows past this floor. Preserved across extraction upserts only cleared when description exceeds it.
22
22
  }
23
23
 
24
24
  export interface Fact extends DataItemBase {
@@ -20,9 +20,15 @@ export interface OpenCodeSettings {
20
20
  processed_sessions?: Record<string, string>; // sessionId → ISO timestamp of last import
21
21
  }
22
22
 
23
+ export interface DocumentRecord {
24
+ created_at: string;
25
+ type: "imported" | "generated";
26
+ subject?: string;
27
+ }
28
+
23
29
  export interface DocumentSettings {
24
30
  extraction_model?: string;
25
- processed_documents?: Record<string, string>;
31
+ processed_documents?: Record<string, DocumentRecord>;
26
32
  }
27
33
 
28
34
  export interface CeremonyConfig {
@@ -53,6 +53,7 @@ export enum LLMNextStep {
53
53
  HandleTopicValidate = "handleTopicValidate",
54
54
  HandleReflectionCritic = "handleReflectionCritic",
55
55
  HandleDocumentSegmentation = "handleDocumentSegmentation",
56
+ HandleKnowledgeSynthesis = "handleKnowledgeSynthesis",
56
57
  }
57
58
 
58
59
  export enum ProviderType {
@@ -29,7 +29,7 @@ export interface ToolProvider {
29
29
  export interface ToolDefinition {
30
30
  id: string; // UUID
31
31
  provider_id: string; // FK → ToolProvider.id (required)
32
- name: string; // Snake_case machine name ("web_search", "read_memory")
32
+ name: string; // Snake_case machine name ("web_search", "find_memory")
33
33
  display_name: string; // Human label
34
34
  description: string; // What the LLM reads to decide whether to call this tool
35
35
  input_schema: Record<string, unknown>; // JSON Schema for parameters the LLM can pass
@@ -77,6 +77,7 @@ export interface QueueStatus {
77
77
  embedding_warning?: boolean;
78
78
  pending_documents?: Array<{ batchId: string; filename: string; count: number }>;
79
79
  extracting_documents?: string[];
80
+ generating_documents?: string[];
80
81
  }
81
82
 
82
83
  export interface EiError {
@@ -121,6 +122,7 @@ export interface Ei_Interface {
121
122
  onRoomMessageAdded?: (roomId: string) => void;
122
123
  onRoomMessageQueued?: (roomId: string) => void;
123
124
  onRoomMessageProcessing?: (roomId: string) => void;
125
+ onDocumentGenerated?: (slug: string) => void;
124
126
  }
125
127
 
126
128
  // =============================================================================
@@ -11,6 +11,10 @@ import {
11
11
  queueAllScans,
12
12
  type ExtractionContext,
13
13
  } from "../../core/orchestrators/human-extraction.js";
14
+ import {
15
+ queuePersonRewritePhase,
16
+ queueTopicRewritePhase,
17
+ } from "../../core/orchestrators/ceremony.js";
14
18
  import { isProcessRunning } from "../process-check.js";
15
19
  import { getMachineId } from "../machine-id.js";
16
20
 
@@ -268,6 +272,8 @@ export async function importClaudeCodeSessions(
268
272
  sources: [`claudecode:${getMachineId()}:${targetSession.id}`],
269
273
  };
270
274
 
275
+ queuePersonRewritePhase(stateManager);
276
+ queueTopicRewritePhase(stateManager);
271
277
  const ccSettings = stateManager.getHuman().settings?.claudeCode;
272
278
  queueAllScans(context, stateManager, {
273
279
  extraction_model: ccSettings?.extraction_model,
@@ -13,6 +13,10 @@ import {
13
13
  queueAllScans,
14
14
  type ExtractionContext,
15
15
  } from "../../core/orchestrators/human-extraction.js";
16
+ import {
17
+ queuePersonRewritePhase,
18
+ queueTopicRewritePhase,
19
+ } from "../../core/orchestrators/ceremony.js";
16
20
 
17
21
  export interface CursorImportResult {
18
22
  sessionsProcessed: number;
@@ -227,6 +231,8 @@ export async function importCursorSessions(
227
231
  sources: [`cursor:${getMachineId()}:${targetSession.id}`],
228
232
  };
229
233
 
234
+ queuePersonRewritePhase(stateManager);
235
+ queueTopicRewritePhase(stateManager);
230
236
  queueAllScans(context, stateManager, { external_filter: "only" });
231
237
  result.extractionScansQueued += 4;
232
238
  }
@@ -150,13 +150,15 @@ export async function executeUnsource(
150
150
  stateManager.messages_remove("emmet", sourceMessageIds);
151
151
  }
152
152
 
153
- const filename = preview.sourceTag.startsWith("import:document:")
153
+ const key = preview.sourceTag.startsWith("import:document:")
154
154
  ? preview.sourceTag.slice("import:document:".length)
155
- : preview.sourceTag;
155
+ : preview.sourceTag.startsWith("generate:document:")
156
+ ? preview.sourceTag.slice("generate:document:".length)
157
+ : preview.sourceTag;
156
158
 
157
159
  const human = stateManager.getHuman();
158
160
  if (human.settings?.document?.processed_documents) {
159
- delete human.settings.document.processed_documents[filename];
161
+ delete human.settings.document.processed_documents[key];
160
162
  stateManager.setHuman(human);
161
163
  }
162
164
 
@@ -8,6 +8,10 @@ import {
8
8
  queueAllScans,
9
9
  type ExtractionContext,
10
10
  } from "../../core/orchestrators/human-extraction.js";
11
+ import {
12
+ queuePersonRewritePhase,
13
+ queueTopicRewritePhase,
14
+ } from "../../core/orchestrators/ceremony.js";
11
15
  import { isProcessRunning } from "../process-check.js";
12
16
  import { getMachineId } from "../machine-id.js";
13
17
 
@@ -195,6 +199,7 @@ export async function importOpenCodeSessions(
195
199
 
196
200
  const cutoffIso = processedSessions[targetSession.id] ?? null;
197
201
  const cutoffMs = cutoffIso ? new Date(cutoffIso).getTime() : null;
202
+ let anyPersonaHasChanges = false;
198
203
 
199
204
  for (const [, { persona, msgs: agentMsgs, isNew, agentName }] of byPersonaId) {
200
205
  if (isNew) {
@@ -252,6 +257,7 @@ export async function importOpenCodeSessions(
252
257
  };
253
258
 
254
259
  if (!signal?.aborted) {
260
+ anyPersonaHasChanges = true;
255
261
  const openCodeSettings = stateManager.getHuman().settings?.opencode;
256
262
  queueAllScans(context, stateManager, {
257
263
  extraction_model: openCodeSettings?.extraction_model,
@@ -264,7 +270,13 @@ export async function importOpenCodeSessions(
264
270
 
265
271
  result.sessionsProcessed = 1;
266
272
 
267
- // ─── Step 6: Advance extraction state ────────────────────────────────
273
+ // ─── Step 6: Queue rewrite checks if any persona had new messages ─────
274
+ if (anyPersonaHasChanges && !signal?.aborted) {
275
+ queuePersonRewritePhase(stateManager);
276
+ queueTopicRewritePhase(stateManager);
277
+ }
278
+
279
+ // ─── Step 7: Advance extraction state ────────────────────────────────
268
280
  updateExtractionState(stateManager, targetSession);
269
281
 
270
282
  console.log(
@@ -3,8 +3,13 @@ import type { Message } from "../../core/types.js";
3
3
  import {
4
4
  queueTopicScan,
5
5
  queuePersonScan,
6
+ queueFactFind,
6
7
  type ExtractionContext,
7
8
  } from "../../core/orchestrators/human-extraction.js";
9
+ import {
10
+ queuePersonRewritePhase,
11
+ queueTopicRewritePhase,
12
+ } from "../../core/orchestrators/ceremony.js";
8
13
 
9
14
  export interface PersonaHistoryImportResult {
10
15
  daysQueued: number;
@@ -101,11 +106,12 @@ export async function importPersonaHistory(
101
106
  };
102
107
 
103
108
  const extractionModel = settings?.extraction_model;
109
+ queueFactFind(context, stateManager, { extraction_model: extractionModel });
104
110
  queueTopicScan(context, stateManager, { extraction_model: extractionModel });
105
111
  queuePersonScan(context, stateManager, { extraction_model: extractionModel });
106
112
 
107
113
  result.personasProcessed++;
108
- result.scansQueued += 2;
114
+ result.scansQueued += 3;
109
115
  }
110
116
 
111
117
  for (const room of Object.values((stateManager.getStorageState() as any).rooms ?? {})) {
@@ -139,6 +145,11 @@ export async function importPersonaHistory(
139
145
 
140
146
  result.daysQueued = 1;
141
147
 
148
+ if (result.scansQueued > 0) {
149
+ queuePersonRewritePhase(stateManager);
150
+ queueTopicRewritePhase(stateManager);
151
+ }
152
+
142
153
  const isLastDay = currentDate >= today;
143
154
  advanceProgress(stateManager, currentDate, isLastDay);
144
155