ei-tui 0.9.4 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +22 -3
  2. package/package.json +5 -1
  3. package/src/README.md +9 -25
  4. package/src/core/handlers/document-segmentation.ts +113 -0
  5. package/src/core/handlers/human-extraction.ts +16 -16
  6. package/src/core/handlers/index.ts +2 -0
  7. package/src/core/handlers/rewrite.ts +13 -9
  8. package/src/core/heartbeat-manager.ts +2 -2
  9. package/src/core/llm-client.ts +66 -6
  10. package/src/core/message-manager.ts +20 -18
  11. package/src/core/orchestrators/ceremony.ts +83 -40
  12. package/src/core/orchestrators/human-extraction.ts +5 -1
  13. package/src/core/persona-manager.ts +4 -0
  14. package/src/core/processor.ts +90 -1
  15. package/src/core/queue-manager.ts +35 -0
  16. package/src/core/queue-processor.ts +13 -13
  17. package/src/core/state/queue.ts +9 -1
  18. package/src/core/state-manager.ts +10 -6
  19. package/src/core/types/entities.ts +15 -0
  20. package/src/core/types/enums.ts +1 -0
  21. package/src/core/types/integrations.ts +2 -0
  22. package/src/core/types/llm.ts +9 -0
  23. package/src/integrations/document/chunker.ts +88 -0
  24. package/src/integrations/document/importer.ts +82 -0
  25. package/src/integrations/document/index.ts +2 -0
  26. package/src/integrations/document/invoice.ts +63 -0
  27. package/src/integrations/document/types.ts +16 -0
  28. package/src/integrations/document/unsource.ts +164 -0
  29. package/src/integrations/persona-history/importer.ts +197 -0
  30. package/src/integrations/persona-history/index.ts +3 -0
  31. package/src/integrations/persona-history/types.ts +7 -0
  32. package/src/prompts/ceremony/dedup.ts +7 -3
  33. package/src/prompts/ceremony/index.ts +2 -1
  34. package/src/prompts/ceremony/people-rewrite.ts +190 -0
  35. package/src/prompts/ceremony/{rewrite.ts → topic-rewrite.ts} +103 -78
  36. package/src/prompts/human/person-scan.ts +13 -4
  37. package/src/prompts/human/topic-scan.ts +16 -2
  38. package/src/prompts/human/topic-update.ts +36 -4
  39. package/src/prompts/human/types.ts +1 -0
  40. package/src/storage/indexed.ts +4 -0
  41. package/src/storage/interface.ts +1 -0
  42. package/src/storage/local.ts +4 -0
  43. package/src/templates/emmett.ts +49 -0
  44. package/tui/README.md +25 -2
  45. package/tui/src/app.tsx +9 -6
  46. package/tui/src/commands/delete.tsx +7 -1
  47. package/tui/src/commands/import.tsx +30 -0
  48. package/tui/src/commands/unsource.tsx +115 -0
  49. package/tui/src/components/PromptInput.tsx +4 -0
  50. package/tui/src/components/WelcomeOverlay.tsx +58 -32
  51. package/tui/src/context/ei.tsx +80 -60
  52. package/tui/src/index.tsx +14 -0
  53. package/tui/src/storage/file.ts +11 -5
  54. package/tui/src/util/e2e-flags.ts +4 -3
  55. package/tui/src/util/help-content.ts +20 -0
  56. package/tui/src/util/logger.ts +1 -1
  57. package/tui/src/util/provider-detection.ts +251 -0
  58. package/tui/src/util/yaml-human.ts +7 -1
package/README.md CHANGED
@@ -83,7 +83,7 @@ Ei can operate with three types of input, and three types of output.
83
83
  ^
84
84
  Sessions
85
85
  |
86
- [OpenCode]
86
+ [OpenCode / Claude Code / Cursor]
87
87
  ```
88
88
 
89
89
  ```
@@ -169,6 +169,22 @@ All sessions map to a single "Cursor" persona.
169
169
 
170
170
  Sessions are processed oldest-first, one per queue cycle, so Ei won't overwhelm your LLM provider on first run. See [TUI Readme](tui/README.md)
171
171
 
172
+ ## Document Import
173
+
174
+ Got notes, journals, markdown files? You can feed them directly to Ei.
175
+
176
+ **Web**: Open **☰ menu** → **My Data** → **Documents** tab. Drop a `.txt`, `.md`, or `.markdown` file and Ei gets to work.
177
+
178
+ **TUI**:
179
+ ```bash
180
+ /import ~/notes/my-journal.md
181
+ /import /path/to/report.pdf
182
+ ```
183
+
184
+ Ei splits the document into segments, runs them through the extraction pipeline, and pulls out facts, topics, people, and quotes — exactly like it does with your conversations. The extracted knowledge is attributed to a reserved persona called **Emmett** so it doesn't pollute your chat history.
185
+
186
+ Both surfaces show you which documents have been imported and let you remove their extracted knowledge (web: Delete button in the Documents tab; TUI: `/unsource <source_tag>`).
187
+
172
188
  ## Built-in Tool Integrations
173
189
 
174
190
  Personas can use tools. Not just read-from-memory tools — *actual* tools. Web search. Your music. Your filesystem. Here's what ships with Ei out of the box:
@@ -184,6 +200,7 @@ Personas can use tools. Not just read-from-memory tools — *actual* tools. Web
184
200
  | `search_files` | Find files by name pattern *(TUI only)* |
185
201
  | `grep` | Search file contents by regex *(TUI only)* |
186
202
  | `get_file_info` | File/directory metadata *(TUI only)* |
203
+ | `web_fetch` | Fetch a URL and return its text content *(TUI only — blocked by CORS in browsers)* |
187
204
 
188
205
  The filesystem tools make Ei a legitimate coding assistant in the TUI. Ask a persona to review a file, understand a project structure, or track down where something is defined — it can actually look.
189
206
 
@@ -264,13 +281,15 @@ Tag a version to publish automatically:
264
281
 
265
282
  ```bash
266
283
  # bump version in package.json
267
- git commit -am "chore: bump to v0.1.4"
268
- git tag v0.1.4
284
+ git commit -am "chore: bump to v1.0.0"
285
+ git tag v1.0.0
269
286
  git push && git push --tags
270
287
  ```
271
288
 
272
289
  GitHub Actions picks up the tag and publishes to npm with provenance via OIDC. No stored secrets.
273
290
 
291
+ > **Note**: Run the pre-flight checklist in `AGENTS.md` (or use the `release` skill in OpenCode) before tagging. The v0.1.9 incident is a cautionary tale.
292
+
274
293
  ## Project Structure
275
294
 
276
295
  See `AGENTS.md` for detailed architecture and contribution guidelines.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ei-tui",
3
- "version": "0.9.4",
3
+ "version": "1.0.1",
4
4
  "author": "Flare576",
5
5
  "repository": {
6
6
  "type": "git",
@@ -57,6 +57,10 @@
57
57
  "test:evals:topic-scan": "vite-node tests/evals/topic-scan.eval.ts",
58
58
  "test:evals:topic-match": "vite-node tests/evals/topic-match.eval.ts",
59
59
  "test:evals:topic-update": "vite-node tests/evals/topic-update.eval.ts",
60
+ "test:evals:topic-technical": "vite-node tests/evals/topic-technical.eval.ts",
61
+ "test:evals:rewrite-scan": "vite-node tests/evals/rewrite-scan.eval.ts",
62
+ "test:evals:rewrite-rewrite": "vite-node tests/evals/rewrite-rewrite.eval.ts",
63
+ "test:evals:rewrite-real-data": "vite-node tests/evals/rewrite-real-data.eval.ts",
60
64
  "test:evals:topic-validate": "vite-node tests/evals/topic-validate.eval.ts",
61
65
  "test:evals:person-scan": "vite-node tests/evals/person-scan.eval.ts",
62
66
  "test:evals:person-update": "vite-node tests/evals/person-update.eval.ts",
package/src/README.md CHANGED
@@ -57,39 +57,23 @@ Each Topic will have an "exposure" rating similar to those on Human Data points.
57
57
 
58
58
  # Ceremony Intent
59
59
 
60
- Every 24 hours, we want to freshen up the system. We do this in 4 parts: Exposure, Decay, Expire, Explore
60
+ Every 24 hours, the system runs a ceremony to keep knowledge fresh and healthy. Phases run sequentially via `ceremony_progress`:
61
61
 
62
- ## Exposure
62
+ **Phase 1 → Dedup**: User-triggered only (not automated). Merges confirmed duplicate records.
63
63
 
64
- I also frequently refer to this as "Extract," but this is the first step where we determine what the human and that Persona talked about that day. It serves two purposes:
64
+ **Phase 2 Expose**: Human extraction catch-up (facts, topics, people) + persona topic rating for any messages that didn't hit the per-send threshold during the day.
65
65
 
66
- ### Detail Extraction
66
+ **Phase 3 → EventSummary**: Summarizes significant events from recent conversations.
67
67
 
68
- Since we also pull out details during normal discourse (see above), this is the less-important step at this point, but still vital for catching up with the last few messages, or Personas that only received a few messages during the day and may not have hit the current limit for natural extraction.
68
+ **Decay** (synchronous after Phase 3): Applies exposure decay to persona topics + prunes old messages. Human ceremony (decay for human topics/people) runs here too.
69
69
 
70
- ### Exposure Adjustment
70
+ **Phase 4 → Person Rewrite**: Scans bloated Person records (>750 chars) and extracts non-relationship content into Topics. Gated so Topic Rewrite can snapshot the updated Topic list afterward.
71
71
 
72
- Exposure is calculated by two metrics - `desired` and `current`. If an entity REALLY likes talking about a subject, their `desired` will be very high (1.0 max), ranging down to 0.0 for subjects which that entity does NOT wish to discuss. You may have guessed already, but `current` is how much they've recently talked about a topic.
72
+ **Topic Rewrite** (fire-and-forget after Phase 4): Scans bloated Topic records (>750 chars) and splits them into focused sub-topics. Topics created by Person Rewrite are included.
73
73
 
74
- Adjusting the values is different for Human Topics/People than Persona Topics. The Human subjects are actually adjusted during the previous step, while extracting details.
74
+ **Reflection** (fire-and-forget alongside Phase 4): Persona-side critic pass on person records.
75
75
 
76
- The Persona Topic update only happens during the Ceremony, and really this step only increases exposure IF the subject was discussed, bumping the last_updated field accordingly.
77
-
78
- ## Decay
79
-
80
- After we determine if topics were discussed (increasing exposure), we adjust exposure the _other_ way. Based on some heuristics (like current level, desired level, and time-since-discussion), we decrease the current exposure levels down.
81
-
82
- ## Expire
83
-
84
- This and the following step (Explore) are exclusive to Persona Topics right now. In Expire, we analyze the Person Topics to determine if any of them have
85
- - Lost their meaning to the Persona
86
- - Been ignored or dismissed by the user
87
-
88
- This is largely tracked by exposure, but expiration is dictated by an Agent.
89
-
90
- ## Explore
91
-
92
- After we've removed irrelevant topics, this is the Agent's opportunity to add NEW topics that might be of interest to the Persona (and the user). Again, it's a prompt to an agent if the Persona doesn't have its full capacity of Topics.
76
+ > **Note**: Expire and Explore phases were removed in the Persona Ceremony Simplification (2026-04-05). Persona topics now only update `exposure_current` during ceremony. See CONTRACTS.md changelog for details.
93
77
 
94
78
  # Opencode Importer
95
79
 
@@ -0,0 +1,113 @@
1
+ import { ContextStatus } from "../types.js";
2
+ import type { LLMResponse, Message } from "../types.js";
3
+ import type { StateManager } from "../state-manager.js";
4
+ import {
5
+ queueAllScans,
6
+ type ExtractionContext,
7
+ } from "../orchestrators/human-extraction.js";
8
+
9
+ function parseSegmentArray(content: string): string[] | null {
10
+ const jsonMatch = content.match(/```json\s*([\s\S]*?)```/) ?? content.match(/```\s*([\s\S]*?)```/);
11
+ const jsonText = jsonMatch ? jsonMatch[1].trim() : content.trim();
12
+
13
+ const arrayMatch = jsonText.match(/\[[\s\S]*\]/);
14
+ if (!arrayMatch) return null;
15
+
16
+ try {
17
+ const parsed = JSON.parse(arrayMatch[0]);
18
+ if (!Array.isArray(parsed)) return null;
19
+ return parsed.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
20
+ } catch {
21
+ return null;
22
+ }
23
+ }
24
+
25
+ export function handleDocumentSegmentation(response: LLMResponse, state: StateManager): void {
26
+ const { batchId, filename, originalContent } = response.request.data as {
27
+ batchId: string;
28
+ filename: string;
29
+ originalContent: string;
30
+ };
31
+
32
+ if (!batchId || !filename) {
33
+ console.error("[handleDocumentSegmentation] Missing batchId or filename in request data");
34
+ return;
35
+ }
36
+
37
+ let segments: string[];
38
+ if (response.content) {
39
+ const parsed = parseSegmentArray(response.content);
40
+ segments = (parsed && parsed.length > 0) ? parsed : [originalContent];
41
+ } else {
42
+ segments = [originalContent];
43
+ }
44
+
45
+ const emmett = state.persona_getById("emmet");
46
+ if (!emmett) {
47
+ console.warn("[handleDocumentSegmentation] Emmett persona not found — skipping segment write");
48
+ return;
49
+ }
50
+
51
+ const now = new Date().toISOString();
52
+ const sourceTag = `import:document:${filename}`;
53
+
54
+ for (const segment of segments) {
55
+ const message: Message = {
56
+ id: crypto.randomUUID(),
57
+ role: "system",
58
+ content: segment,
59
+ timestamp: now,
60
+ read: true,
61
+ context_status: ContextStatus.Always,
62
+ external: true,
63
+ source_tag: sourceTag,
64
+ };
65
+ state.messages_append("emmet", message);
66
+ }
67
+
68
+ console.log(`[handleDocumentSegmentation] Wrote ${segments.length} segment(s) for batch ${batchId} (${filename})`);
69
+ }
70
+
71
+ export function finishDocumentBatch(batchId: string, filename: string, state: StateManager): void {
72
+ const sourceTag = `import:document:${filename}`;
73
+
74
+ const emmettMessages = state.messages_get("emmet");
75
+ const docMessages = emmettMessages.filter(m => m.external === true && m.source_tag === sourceTag);
76
+
77
+ if (docMessages.length === 0) {
78
+ console.warn(`[finishDocumentBatch] No messages found for ${sourceTag} — skipping extraction`);
79
+ } else {
80
+ const extractionContext: ExtractionContext = {
81
+ personaId: "emmet",
82
+ channelDisplayName: "Document",
83
+ messages_context: [],
84
+ messages_analyze: docMessages,
85
+ sources: [sourceTag],
86
+ };
87
+
88
+ const docSettings = state.getHuman().settings?.document;
89
+ queueAllScans(extractionContext, state, {
90
+ extraction_model: docSettings?.extraction_model,
91
+ external_filter: "only",
92
+ });
93
+
94
+ console.log(`[finishDocumentBatch] Queued extraction for ${docMessages.length} message(s) from ${filename}`);
95
+ }
96
+
97
+ const updatedHuman = state.getHuman();
98
+ state.setHuman({
99
+ ...updatedHuman,
100
+ settings: {
101
+ ...updatedHuman.settings,
102
+ document: {
103
+ ...updatedHuman.settings?.document,
104
+ processed_documents: {
105
+ ...(updatedHuman.settings?.document?.processed_documents ?? {}),
106
+ [filename]: new Date().toISOString(),
107
+ },
108
+ },
109
+ },
110
+ });
111
+
112
+ console.log(`[finishDocumentBatch] Batch ${batchId} complete, ${filename} marked processed`);
113
+ }
@@ -92,7 +92,7 @@ export async function handleFactFind(response: LLMResponse, state: StateManager)
92
92
  markMessagesExtracted(response, state, "f");
93
93
 
94
94
  if (!result?.facts || !Array.isArray(result.facts)) {
95
- console.log("[handleFactFind] No facts detected or invalid result");
95
+ console.debug("[handleFactFind] No facts detected or invalid result");
96
96
  return;
97
97
  }
98
98
 
@@ -106,26 +106,26 @@ export async function handleFactFind(response: LLMResponse, state: StateManager)
106
106
  for (const factResult of result.facts) {
107
107
  // Only upsert facts that match a built-in name
108
108
  if (!BUILT_IN_FACT_NAMES.has(factResult.name)) {
109
- console.log(`[handleFactFind] Skipping non-built-in fact: "${factResult.name}"`);
109
+ console.warn(`[handleFactFind] Skipping non-built-in fact: "${factResult.name}"`);
110
110
  continue;
111
111
  }
112
112
 
113
113
  // Find the existing fact in state
114
114
  const existingFact = human.facts.find(f => f.name === factResult.name);
115
115
  if (!existingFact) {
116
- console.log(`[handleFactFind] Skipping unknown fact: "${factResult.name}"`);
116
+ console.warn(`[handleFactFind] Skipping unknown fact: "${factResult.name}"`);
117
117
  continue;
118
118
  }
119
119
 
120
120
  // Skip facts that already have descriptions (only fill empty ones)
121
121
  if (existingFact.description && existingFact.description !== "") {
122
- console.log(`[handleFactFind] Skipping fact with existing description: "${factResult.name}"`);
122
+ console.debug(`[handleFactFind] Skipping fact with existing description: "${factResult.name}"`);
123
123
  continue;
124
124
  }
125
125
 
126
126
  // Skip if the LLM returned a null/empty/non-string value — don't store booleans or nulls
127
127
  if (!factResult.value || typeof factResult.value !== 'string') {
128
- console.log(`[handleFactFind] Skipping fact with null/empty/non-string value: "${factResult.name}" (got ${typeof factResult.value})`);
128
+ console.warn(`[handleFactFind] Skipping fact with null/empty/non-string value: "${factResult.name}" (got ${typeof factResult.value})`);
129
129
  continue;
130
130
  }
131
131
 
@@ -165,7 +165,7 @@ export async function handleHumanTopicScan(response: LLMResponse, state: StateMa
165
165
  markMessagesExtracted(response, state, "t");
166
166
 
167
167
  if (!result?.topics || !Array.isArray(result.topics)) {
168
- console.log("[handleHumanTopicScan] No topics detected or invalid result");
168
+ console.debug("[handleHumanTopicScan] No topics detected or invalid result");
169
169
  return;
170
170
  }
171
171
 
@@ -185,7 +185,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
185
185
  markMessagesExtracted(response, state, "p");
186
186
 
187
187
  if (!result?.people || !Array.isArray(result.people)) {
188
- console.log("[handleHumanPersonScan] No people detected or invalid result");
188
+ console.debug("[handleHumanPersonScan] No people detected or invalid result");
189
189
  return;
190
190
  }
191
191
 
@@ -231,7 +231,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
231
231
  }
232
232
  }
233
233
  if (!matchedPerson) {
234
- console.log(`[handleHumanPersonScan] Multi-match for "${candidate.name}" (${matches.length} hits) — no embedding above threshold, creating new record`);
234
+ console.debug(`[handleHumanPersonScan] Multi-match for "${candidate.name}" (${matches.length} hits) — no embedding above threshold, creating new record`);
235
235
  }
236
236
  } catch (err) {
237
237
  console.warn(`[handleHumanPersonScan] Multi-match embedding failed for "${candidate.name}", using first match:`, err);
@@ -253,7 +253,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
253
253
  if (isUnknownPlaceholder || isSingleton) {
254
254
  matchedPerson = existing;
255
255
  const reason = isUnknownPlaceholder ? 'unnamed placeholder' : 'singleton relationship';
256
- console.log(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
256
+ console.debug(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
257
257
  }
258
258
  } else {
259
259
  // N>1 same relationship → cosine within that subset.
@@ -267,7 +267,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
267
267
  : `all ${human.people.length} people`;
268
268
 
269
269
  if (searchPool.length > 0) {
270
- console.log(`[handleHumanPersonScan] "${candidate.name}": cosine against ${searchPool.length} embedded (${poolLabel})`);
270
+ console.debug(`[handleHumanPersonScan] "${candidate.name}": cosine against ${searchPool.length} embedded (${poolLabel})`);
271
271
  try {
272
272
  const embeddingService = getEmbeddingService();
273
273
  const candidateText = getPersonEmbeddingText({
@@ -288,15 +288,15 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
288
288
  }
289
289
  const top3 = scores.sort((a, b) => b.sim - a.sim).slice(0, 3).map(s => `"${s.name}"=${s.sim.toFixed(3)}`).join(', ');
290
290
  if (matchedPerson) {
291
- console.log(`[handleHumanPersonScan] Cosine matched "${candidate.name}" → "${matchedPerson.name}" (${bestSimilarity.toFixed(3)}) | top3: ${top3}`);
291
+ console.debug(`[handleHumanPersonScan] Cosine matched "${candidate.name}" → "${matchedPerson.name}" (${bestSimilarity.toFixed(3)}) | top3: ${top3}`);
292
292
  } else {
293
- console.log(`[handleHumanPersonScan] Cosine: no match above ${ZERO_MATCH_COSINE_THRESHOLD} for "${candidate.name}" | top3: ${top3}`);
293
+ console.debug(`[handleHumanPersonScan] Cosine: no match above ${ZERO_MATCH_COSINE_THRESHOLD} for "${candidate.name}" | top3: ${top3}`);
294
294
  }
295
295
  } catch (err) {
296
296
  console.warn(`[handleHumanPersonScan] Cosine failed for "${candidate.name}":`, err);
297
297
  }
298
298
  } else {
299
- console.log(`[handleHumanPersonScan] "${candidate.name}": no embedded people in pool (${poolLabel}) — new person`);
299
+ console.debug(`[handleHumanPersonScan] "${candidate.name}": no embedded people in pool (${poolLabel}) — new person`);
300
300
  }
301
301
  }
302
302
  }
@@ -305,7 +305,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
305
305
  const linkedPersonaId = matchedPerson.identifiers
306
306
  ?.find(i => i.type === "Ei Persona")?.value;
307
307
  if (linkedPersonaId) {
308
- console.log(`[handleHumanPersonScan] Skipping update for "${candidate.name}" — scan marked as reflection drain (reflection_progress=1)`);
308
+ console.debug(`[handleHumanPersonScan] Skipping update for "${candidate.name}" — scan marked as reflection drain (reflection_progress=1)`);
309
309
  continue;
310
310
  }
311
311
  }
@@ -326,7 +326,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
326
326
  : matches.length > 1
327
327
  ? `multi-match ambiguous (${matches.length} hits) — new record`
328
328
  : "no match (new person)";
329
- console.log(`[handleHumanPersonScan] person "${candidate.name}": ${matched}`);
329
+ console.debug(`[handleHumanPersonScan] person "${candidate.name}": ${matched}`);
330
330
  }
331
331
  console.log(`[handleHumanPersonScan] Processed ${result.people.length} person(s)`);
332
332
  }
@@ -337,7 +337,7 @@ export async function handleEventScan(response: LLMResponse, state: StateManager
337
337
  const result = response.parsed as { events?: Array<{ name: string; description: string; reason: string }> } | undefined;
338
338
 
339
339
  if (!result?.events || !Array.isArray(result.events) || result.events.length === 0) {
340
- console.log("[handleEventScan] No epic events detected");
340
+ console.debug("[handleEventScan] No epic events detected");
341
341
  return;
342
342
  }
343
343
 
@@ -15,6 +15,7 @@ import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
15
15
  import { handleDedupCurate } from "./dedup.js";
16
16
  import { handleRoomResponse, handleRoomJudge } from "./rooms.js";
17
17
  import { handlePersonaPreview } from "./persona-preview.js";
18
+ import { handleDocumentSegmentation } from "./document-segmentation.js";
18
19
 
19
20
  export const handlers: Record<LLMNextStep, ResponseHandler> = {
20
21
  handlePersonaResponse,
@@ -41,4 +42,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
41
42
  handlePersonaPreview,
42
43
  [LLMNextStep.HandleTopicValidate]: handleDedupCurate,
43
44
  [LLMNextStep.HandleReflectionCritic]: handleReflectionCritic,
45
+ [LLMNextStep.HandleDocumentSegmentation]: handleDocumentSegmentation,
44
46
  };
@@ -14,7 +14,8 @@ import type {
14
14
  RewriteResult,
15
15
  RewriteSubjectMatch,
16
16
  } from "../../prompts/ceremony/types.js";
17
- import { buildRewritePrompt } from "../../prompts/ceremony/rewrite.js";
17
+ import { buildPersonRewriteSplitPrompt } from "../../prompts/ceremony/people-rewrite.js";
18
+ import { buildTopicRewriteSplitPrompt } from "../../prompts/ceremony/topic-rewrite.js";
18
19
  import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
19
20
 
20
21
  import { searchHumanData } from "../human-data-manager.js";
@@ -79,12 +80,10 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
79
80
  }
80
81
  }
81
82
 
82
- // Build Phase 2 prompt and queue it
83
- const prompt = buildRewritePrompt({
84
- item: currentItem,
85
- itemType,
86
- subjects: subjectMatches,
87
- });
83
+ const splitData = { item: currentItem, itemType, subjects: subjectMatches };
84
+ const prompt = itemType === "person"
85
+ ? buildPersonRewriteSplitPrompt(splitData)
86
+ : buildTopicRewriteSplitPrompt(splitData);
88
87
 
89
88
  state.queue_enqueue({
90
89
  type: LLMRequestType.JSON,
@@ -125,6 +124,11 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
125
124
  const human = state.getHuman();
126
125
  const now = new Date().toISOString();
127
126
 
127
+ const originalItem = itemType === "topic"
128
+ ? human.topics.find(t => t.id === itemId)
129
+ : human.people.find(p => p.id === itemId);
130
+ const originalCategory = itemType === "topic" ? (originalItem as Topic | undefined)?.category : undefined;
131
+
128
132
  const allItems: DataItemBase[] = [
129
133
  ...human.topics, ...human.people,
130
134
  ];
@@ -228,11 +232,11 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
228
232
  switch (item.type) {
229
233
  case "topic": {
230
234
  if (!item.category) {
231
- console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);
235
+ console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — inheriting from original (${originalCategory ?? "Interest"})`);
232
236
  }
233
237
  const topic: Topic = {
234
238
  ...baseFields,
235
- category: item.category ?? "Interest",
239
+ category: item.category ?? originalCategory ?? "Interest",
236
240
  exposure_current: 0.5,
237
241
  exposure_desired: 0.5,
238
242
  };
@@ -143,7 +143,7 @@ export async function queueEiHeartbeat(
143
143
  }
144
144
 
145
145
  const activePersonas = personas
146
- .filter((p) => !p.is_archived && !p.is_paused && p.id !== "ei")
146
+ .filter((p) => !p.is_archived && !p.is_paused && !p.is_static && p.id !== "ei")
147
147
  .map((p) => {
148
148
  const msgs = sm.messages_get(p.id);
149
149
  const lastHuman = [...msgs].reverse().find((m) => m.role === "human");
@@ -169,7 +169,7 @@ export async function queueEiHeartbeat(
169
169
  }
170
170
 
171
171
  const personasWithPendingUpdate = personas.filter(
172
- (p) => !p.is_archived && !p.is_paused && p.id !== "ei" && p.pending_update?.critique
172
+ (p) => !p.is_archived && !p.is_paused && !p.is_static && p.id !== "ei" && p.pending_update?.critique
173
173
  );
174
174
  for (const p of personasWithPendingUpdate) {
175
175
  items.push({
@@ -2,6 +2,36 @@ import type { ChatMessage, ProviderAccount, ModelConfig } from "./types.js";
2
2
  const DEFAULT_TOKEN_LIMIT = 8192;
3
3
  const DEFAULT_MAX_OUTPUT_TOKENS = 8000;
4
4
 
5
+ // Lazy verbose network dump — only active when EI_DEBUG_NETWORK_VERBOSE=1.
6
+ // Uses dynamic import so the web bundle never pulls in node:fs.
7
+ async function writeNetworkDump(
8
+ callNumber: number,
9
+ nextStep: string,
10
+ meta: { model: string; provider: string; latency_ms: number; status_code: number; tokens_in: number; tokens_out: number },
11
+ request: unknown,
12
+ response: unknown
13
+ ): Promise<void> {
14
+ const dataPath = (typeof process !== "undefined" && process.env?.EI_DATA_PATH) ||
15
+ (typeof Bun !== "undefined" && (Bun as Record<string, unknown>).env && ((Bun as { env: Record<string, string> }).env.EI_DATA_PATH));
16
+ if (!dataPath) return;
17
+
18
+ try {
19
+ const { mkdirSync, writeFileSync } = await import("node:fs");
20
+ const { join } = await import("node:path");
21
+ const logsDir = join(dataPath as string, "logs");
22
+ mkdirSync(logsDir, { recursive: true });
23
+
24
+ const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
25
+ const safeName = nextStep.replace(/[^a-zA-Z0-9_-]/g, "_");
26
+ const filename = join(logsDir, `${timestamp}_call${callNumber}_${safeName}.json`);
27
+
28
+ const payload = JSON.stringify({ meta, request, response }, null, 2);
29
+ writeFileSync(filename, payload);
30
+ } catch {
31
+ // Silent — verbose dump failures must never crash the main path
32
+ }
33
+ }
34
+
5
35
  export interface ProviderConfig {
6
36
  baseURL: string;
7
37
  apiKey: string;
@@ -22,6 +52,8 @@ export interface LLMCallOptions {
22
52
  tools?: Record<string, unknown>[];
23
53
  /** Fire-and-forget callback invoked after a successful response to increment usage counters. */
24
54
  onUsageUpdate?: (modelId: string, usage: { calls: number; tokens_in: number; tokens_out: number }) => void;
55
+ /** Queue step name passed through to EI_DEBUG_NETWORK_VERBOSE file dumps. */
56
+ nextStep?: string;
25
57
  }
26
58
 
27
59
  export interface LLMRawResponse {
@@ -212,7 +244,7 @@ function logTokenLimit(model: string, source: string, tokens: number): void {
212
244
  if (source === "default") {
213
245
  console.warn(`[TokenLimit] Unknown model "${model}" — using conservative default (${DEFAULT_TOKEN_LIMIT})`);
214
246
  } else {
215
- console.log(`[TokenLimit] ${model}: ${source} → ${tokens} tokens (extraction budget: ${budget})`);
247
+ console.debug(`[TokenLimit] ${model}: ${source} → ${tokens} tokens (extraction budget: ${budget})`);
216
248
  }
217
249
  }
218
250
 
@@ -226,7 +258,7 @@ export async function callLLMRaw(
226
258
  ): Promise<LLMRawResponse> {
227
259
  llmCallCount++;
228
260
 
229
- const { signal, temperature = 0.7, onUsageUpdate } = options;
261
+ const { signal, temperature = 0.7, onUsageUpdate, nextStep = "unknown" } = options;
230
262
 
231
263
  if (signal?.aborted) {
232
264
  throw new Error("LLM call aborted");
@@ -251,7 +283,9 @@ export async function callLLMRaw(
251
283
 
252
284
  const totalChars = finalMessages.reduce((sum, m) => sum + (m.content?.length ?? 0), 0);
253
285
  const estimatedTokens = Math.ceil(totalChars / 4);
254
- console.log(`[LLM] Call #${llmCallCount} - ~${estimatedTokens} tokens (${totalChars} chars)`);
286
+ const modelLabel = model ?? "default";
287
+ console.log(`[LLM] Call #${llmCallCount} — ${config.name}:${modelLabel}, ~${estimatedTokens} tokens est.`);
288
+ const _llmCallStart = Date.now();
255
289
 
256
290
  const normalizedBaseURL = config.baseURL.replace(/\/+$/, "");
257
291
 
@@ -274,7 +308,18 @@ export async function callLLMRaw(
274
308
  };
275
309
 
276
310
  if (modelConfig?.thinking_budget !== undefined) {
277
- requestBody.think = { budget_tokens: modelConfig.thinking_budget };
311
+ if (modelConfig.thinking_budget === 0) {
312
+ // Universal kill switch across all known providers. Non-conflicting — each reads
313
+ // whichever field it understands and ignores the rest.
314
+ requestBody.reasoning_effort = "none"; // Ollama, OpenAI-compat
315
+ requestBody.enable_thinking = false; // Rapid-MLX
316
+ } else {
317
+ // Pass all on-signals: providers that honor the token budget get it (Qwen3, Anthropic),
318
+ // providers that reduce thinking to on/off use reasoning_effort or enable_thinking.
319
+ requestBody.reasoning_effort = "high";
320
+ requestBody.enable_thinking = true;
321
+ requestBody.think = { budget_tokens: modelConfig.thinking_budget };
322
+ }
278
323
  }
279
324
 
280
325
  if (options.tools && options.tools.length > 0) {
@@ -296,9 +341,24 @@ export async function callLLMRaw(
296
341
 
297
342
  const data = await response.json();
298
343
 
344
+ const _llmLatency = Date.now() - _llmCallStart;
345
+ const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
346
+ const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
347
+ console.log(`[LLM] Response #${llmCallCount} — ${response.status} ${_llmLatency}ms | in: ${tokensIn} out: ${tokensOut}`);
348
+
349
+ const isVerbose = (typeof process !== "undefined" && process.env?.EI_DEBUG_NETWORK_VERBOSE === "1") ||
350
+ (typeof Bun !== "undefined" && (Bun as { env: Record<string, string> }).env?.EI_DEBUG_NETWORK_VERBOSE === "1");
351
+ if (isVerbose) {
352
+ void writeNetworkDump(
353
+ llmCallCount,
354
+ nextStep,
355
+ { model: modelLabel, provider: config.name, latency_ms: _llmLatency, status_code: response.status, tokens_in: tokensIn, tokens_out: tokensOut },
356
+ requestBody,
357
+ data
358
+ );
359
+ }
360
+
299
361
  if (onUsageUpdate && modelConfig) {
300
- const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
301
- const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
302
362
  onUsageUpdate(modelConfig.id, { calls: 1, tokens_in: tokensIn, tokens_out: tokensOut });
303
363
  }
304
364
 
@@ -177,25 +177,27 @@ export async function sendMessage(
177
177
 
178
178
  const history = sm.messages_get(persona.id);
179
179
 
180
- const traitExtractionData: PersonaTraitExtractionPromptData = {
181
- persona_name: persona.display_name,
182
- current_traits: persona.traits,
183
- messages_context: history.slice(-11, -1),
184
- messages_analyze: [message],
185
- };
186
- const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
187
-
188
- sm.queue_enqueue({
189
- type: LLMRequestType.JSON,
190
- priority: LLMPriority.Low,
191
- system: traitPrompt.system,
192
- user: traitPrompt.user,
193
- next_step: LLMNextStep.HandlePersonaTraitExtraction,
194
- model: getModelForPersona(persona.id),
195
- data: { personaId: persona.id, personaDisplayName: persona.display_name },
196
- });
180
+ if (!persona.is_static) {
181
+ const traitExtractionData: PersonaTraitExtractionPromptData = {
182
+ persona_name: persona.display_name,
183
+ current_traits: persona.traits,
184
+ messages_context: history.slice(-11, -1),
185
+ messages_analyze: [message],
186
+ };
187
+ const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
188
+
189
+ sm.queue_enqueue({
190
+ type: LLMRequestType.JSON,
191
+ priority: LLMPriority.Low,
192
+ system: traitPrompt.system,
193
+ user: traitPrompt.user,
194
+ next_step: LLMNextStep.HandlePersonaTraitExtraction,
195
+ model: getModelForPersona(persona.id),
196
+ data: { personaId: persona.id, personaDisplayName: persona.display_name },
197
+ });
197
198
 
198
- checkAndQueueHumanExtraction(sm, persona.id, persona.display_name, history);
199
+ checkAndQueueHumanExtraction(sm, persona.id, persona.display_name, history);
200
+ }
199
201
  }
200
202
 
201
203
  // =============================================================================