ei-tui 0.9.4 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/package.json +5 -1
- package/src/README.md +9 -25
- package/src/core/handlers/document-segmentation.ts +113 -0
- package/src/core/handlers/human-extraction.ts +16 -16
- package/src/core/handlers/index.ts +2 -0
- package/src/core/handlers/rewrite.ts +13 -9
- package/src/core/heartbeat-manager.ts +2 -2
- package/src/core/llm-client.ts +66 -6
- package/src/core/message-manager.ts +20 -18
- package/src/core/orchestrators/ceremony.ts +83 -40
- package/src/core/orchestrators/human-extraction.ts +5 -1
- package/src/core/persona-manager.ts +4 -0
- package/src/core/processor.ts +90 -1
- package/src/core/queue-manager.ts +35 -0
- package/src/core/queue-processor.ts +13 -13
- package/src/core/state/queue.ts +9 -1
- package/src/core/state-manager.ts +10 -6
- package/src/core/types/entities.ts +15 -0
- package/src/core/types/enums.ts +1 -0
- package/src/core/types/integrations.ts +2 -0
- package/src/core/types/llm.ts +9 -0
- package/src/integrations/document/chunker.ts +88 -0
- package/src/integrations/document/importer.ts +82 -0
- package/src/integrations/document/index.ts +2 -0
- package/src/integrations/document/invoice.ts +63 -0
- package/src/integrations/document/types.ts +16 -0
- package/src/integrations/document/unsource.ts +164 -0
- package/src/integrations/persona-history/importer.ts +197 -0
- package/src/integrations/persona-history/index.ts +3 -0
- package/src/integrations/persona-history/types.ts +7 -0
- package/src/prompts/ceremony/dedup.ts +7 -3
- package/src/prompts/ceremony/index.ts +2 -1
- package/src/prompts/ceremony/people-rewrite.ts +190 -0
- package/src/prompts/ceremony/{rewrite.ts → topic-rewrite.ts} +103 -78
- package/src/prompts/human/person-scan.ts +13 -4
- package/src/prompts/human/topic-scan.ts +16 -2
- package/src/prompts/human/topic-update.ts +36 -4
- package/src/prompts/human/types.ts +1 -0
- package/src/storage/indexed.ts +4 -0
- package/src/storage/interface.ts +1 -0
- package/src/storage/local.ts +4 -0
- package/src/templates/emmett.ts +49 -0
- package/tui/README.md +25 -2
- package/tui/src/app.tsx +9 -6
- package/tui/src/commands/delete.tsx +7 -1
- package/tui/src/commands/import.tsx +30 -0
- package/tui/src/commands/unsource.tsx +115 -0
- package/tui/src/components/PromptInput.tsx +4 -0
- package/tui/src/components/WelcomeOverlay.tsx +58 -32
- package/tui/src/context/ei.tsx +80 -60
- package/tui/src/index.tsx +14 -0
- package/tui/src/storage/file.ts +11 -5
- package/tui/src/util/e2e-flags.ts +4 -3
- package/tui/src/util/help-content.ts +20 -0
- package/tui/src/util/logger.ts +1 -1
- package/tui/src/util/provider-detection.ts +251 -0
- package/tui/src/util/yaml-human.ts +7 -1
package/README.md
CHANGED
|
@@ -83,7 +83,7 @@ Ei can operate with three types of input, and three types of output.
|
|
|
83
83
|
^
|
|
84
84
|
Sessions
|
|
85
85
|
|
|
|
86
|
-
|
|
86
|
+
[OpenCode / Claude Code / Cursor]
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
```
|
|
@@ -169,6 +169,22 @@ All sessions map to a single "Cursor" persona.
|
|
|
169
169
|
|
|
170
170
|
Sessions are processed oldest-first, one per queue cycle, so Ei won't overwhelm your LLM provider on first run. See [TUI Readme](tui/README.md)
|
|
171
171
|
|
|
172
|
+
## Document Import
|
|
173
|
+
|
|
174
|
+
Got notes, journals, markdown files? You can feed them directly to Ei.
|
|
175
|
+
|
|
176
|
+
**Web**: Open **☰ menu** → **My Data** → **Documents** tab. Drop a `.txt`, `.md`, or `.markdown` file and Ei gets to work.
|
|
177
|
+
|
|
178
|
+
**TUI**:
|
|
179
|
+
```bash
|
|
180
|
+
/import ~/notes/my-journal.md
|
|
181
|
+
/import /path/to/report.pdf
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Ei splits the document into segments, runs them through the extraction pipeline, and pulls out facts, topics, people, and quotes — exactly like it does with your conversations. The extracted knowledge is attributed to a reserved persona called **Emmett** so it doesn't pollute your chat history.
|
|
185
|
+
|
|
186
|
+
Both surfaces show you which documents have been imported and let you remove their extracted knowledge (web: Delete button in the Documents tab; TUI: `/unsource <source_tag>`).
|
|
187
|
+
|
|
172
188
|
## Built-in Tool Integrations
|
|
173
189
|
|
|
174
190
|
Personas can use tools. Not just read-from-memory tools — *actual* tools. Web search. Your music. Your filesystem. Here's what ships with Ei out of the box:
|
|
@@ -184,6 +200,7 @@ Personas can use tools. Not just read-from-memory tools — *actual* tools. Web
|
|
|
184
200
|
| `search_files` | Find files by name pattern *(TUI only)* |
|
|
185
201
|
| `grep` | Search file contents by regex *(TUI only)* |
|
|
186
202
|
| `get_file_info` | File/directory metadata *(TUI only)* |
|
|
203
|
+
| `web_fetch` | Fetch a URL and return its text content *(TUI only — blocked by CORS in browsers)* |
|
|
187
204
|
|
|
188
205
|
The filesystem tools make Ei a legitimate coding assistant in the TUI. Ask a persona to review a file, understand a project structure, or track down where something is defined — it can actually look.
|
|
189
206
|
|
|
@@ -264,13 +281,15 @@ Tag a version to publish automatically:
|
|
|
264
281
|
|
|
265
282
|
```bash
|
|
266
283
|
# bump version in package.json
|
|
267
|
-
git commit -am "chore: bump to
|
|
268
|
-
git tag
|
|
284
|
+
git commit -am "chore: bump to v1.0.0"
|
|
285
|
+
git tag v1.0.0
|
|
269
286
|
git push && git push --tags
|
|
270
287
|
```
|
|
271
288
|
|
|
272
289
|
GitHub Actions picks up the tag and publishes to npm with provenance via OIDC. No stored secrets.
|
|
273
290
|
|
|
291
|
+
> **Note**: Run the pre-flight checklist in `AGENTS.md` (or use the `release` skill in OpenCode) before tagging. The v0.1.9 incident is a cautionary tale.
|
|
292
|
+
|
|
274
293
|
## Project Structure
|
|
275
294
|
|
|
276
295
|
See `AGENTS.md` for detailed architecture and contribution guidelines.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ei-tui",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"author": "Flare576",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -57,6 +57,10 @@
|
|
|
57
57
|
"test:evals:topic-scan": "vite-node tests/evals/topic-scan.eval.ts",
|
|
58
58
|
"test:evals:topic-match": "vite-node tests/evals/topic-match.eval.ts",
|
|
59
59
|
"test:evals:topic-update": "vite-node tests/evals/topic-update.eval.ts",
|
|
60
|
+
"test:evals:topic-technical": "vite-node tests/evals/topic-technical.eval.ts",
|
|
61
|
+
"test:evals:rewrite-scan": "vite-node tests/evals/rewrite-scan.eval.ts",
|
|
62
|
+
"test:evals:rewrite-rewrite": "vite-node tests/evals/rewrite-rewrite.eval.ts",
|
|
63
|
+
"test:evals:rewrite-real-data": "vite-node tests/evals/rewrite-real-data.eval.ts",
|
|
60
64
|
"test:evals:topic-validate": "vite-node tests/evals/topic-validate.eval.ts",
|
|
61
65
|
"test:evals:person-scan": "vite-node tests/evals/person-scan.eval.ts",
|
|
62
66
|
"test:evals:person-update": "vite-node tests/evals/person-update.eval.ts",
|
package/src/README.md
CHANGED
|
@@ -57,39 +57,23 @@ Each Topic will have an "exposure" rating similar to those on Human Data points.
|
|
|
57
57
|
|
|
58
58
|
# Ceremony Intent
|
|
59
59
|
|
|
60
|
-
Every 24 hours,
|
|
60
|
+
Every 24 hours, the system runs a ceremony to keep knowledge fresh and healthy. Phases run sequentially via `ceremony_progress`:
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
**Phase 1 → Dedup**: User-triggered only (not automated). Merges confirmed duplicate records.
|
|
63
63
|
|
|
64
|
-
|
|
64
|
+
**Phase 2 → Expose**: Human extraction catch-up (facts, topics, people) + persona topic rating for any messages that didn't hit the per-send threshold during the day.
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
**Phase 3 → EventSummary**: Summarizes significant events from recent conversations.
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
**Decay** (synchronous after Phase 3): Applies exposure decay to persona topics + prunes old messages. Human ceremony (decay for human topics/people) runs here too.
|
|
69
69
|
|
|
70
|
-
|
|
70
|
+
**Phase 4 → Person Rewrite**: Scans bloated Person records (>750 chars) and extracts non-relationship content into Topics. Gated so Topic Rewrite can snapshot the updated Topic list afterward.
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
**Topic Rewrite** (fire-and-forget after Phase 4): Scans bloated Topic records (>750 chars) and splits them into focused sub-topics. Topics created by Person Rewrite are included.
|
|
73
73
|
|
|
74
|
-
|
|
74
|
+
**Reflection** (fire-and-forget alongside Phase 4): Persona-side critic pass on person records.
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
## Decay
|
|
79
|
-
|
|
80
|
-
After we determine if topics were discussed (increasing exposure), we adjust exposure the _other_ way. Based on some heuristics (like current level, desired level, and time-since-discussion), we decrease the current exposure levels down.
|
|
81
|
-
|
|
82
|
-
## Expire
|
|
83
|
-
|
|
84
|
-
This and the following step (Explore) are exclusive to Persona Topics right now. In Expire, we analyze the Person Topics to determine if any of them have
|
|
85
|
-
- Lost their meaning to the Persona
|
|
86
|
-
- Been ignored or dismissed by the user
|
|
87
|
-
|
|
88
|
-
This is largely tracked by exposure, but expiration is dictated by an Agent.
|
|
89
|
-
|
|
90
|
-
## Explore
|
|
91
|
-
|
|
92
|
-
After we've removed irrelevant topics, this is the Agent's opportunity to add NEW topics that might be of interest to the Persona (and the user). Again, it's a prompt to an agent if the Persona doesn't have its full capacity of Topics.
|
|
76
|
+
> **Note**: Expire and Explore phases were removed in the Persona Ceremony Simplification (2026-04-05). Persona topics now only update `exposure_current` during ceremony. See CONTRACTS.md changelog for details.
|
|
93
77
|
|
|
94
78
|
# Opencode Importer
|
|
95
79
|
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { ContextStatus } from "../types.js";
|
|
2
|
+
import type { LLMResponse, Message } from "../types.js";
|
|
3
|
+
import type { StateManager } from "../state-manager.js";
|
|
4
|
+
import {
|
|
5
|
+
queueAllScans,
|
|
6
|
+
type ExtractionContext,
|
|
7
|
+
} from "../orchestrators/human-extraction.js";
|
|
8
|
+
|
|
9
|
+
function parseSegmentArray(content: string): string[] | null {
|
|
10
|
+
const jsonMatch = content.match(/```json\s*([\s\S]*?)```/) ?? content.match(/```\s*([\s\S]*?)```/);
|
|
11
|
+
const jsonText = jsonMatch ? jsonMatch[1].trim() : content.trim();
|
|
12
|
+
|
|
13
|
+
const arrayMatch = jsonText.match(/\[[\s\S]*\]/);
|
|
14
|
+
if (!arrayMatch) return null;
|
|
15
|
+
|
|
16
|
+
try {
|
|
17
|
+
const parsed = JSON.parse(arrayMatch[0]);
|
|
18
|
+
if (!Array.isArray(parsed)) return null;
|
|
19
|
+
return parsed.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
|
|
20
|
+
} catch {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function handleDocumentSegmentation(response: LLMResponse, state: StateManager): void {
|
|
26
|
+
const { batchId, filename, originalContent } = response.request.data as {
|
|
27
|
+
batchId: string;
|
|
28
|
+
filename: string;
|
|
29
|
+
originalContent: string;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
if (!batchId || !filename) {
|
|
33
|
+
console.error("[handleDocumentSegmentation] Missing batchId or filename in request data");
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
let segments: string[];
|
|
38
|
+
if (response.content) {
|
|
39
|
+
const parsed = parseSegmentArray(response.content);
|
|
40
|
+
segments = (parsed && parsed.length > 0) ? parsed : [originalContent];
|
|
41
|
+
} else {
|
|
42
|
+
segments = [originalContent];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const emmett = state.persona_getById("emmet");
|
|
46
|
+
if (!emmett) {
|
|
47
|
+
console.warn("[handleDocumentSegmentation] Emmett persona not found — skipping segment write");
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const now = new Date().toISOString();
|
|
52
|
+
const sourceTag = `import:document:${filename}`;
|
|
53
|
+
|
|
54
|
+
for (const segment of segments) {
|
|
55
|
+
const message: Message = {
|
|
56
|
+
id: crypto.randomUUID(),
|
|
57
|
+
role: "system",
|
|
58
|
+
content: segment,
|
|
59
|
+
timestamp: now,
|
|
60
|
+
read: true,
|
|
61
|
+
context_status: ContextStatus.Always,
|
|
62
|
+
external: true,
|
|
63
|
+
source_tag: sourceTag,
|
|
64
|
+
};
|
|
65
|
+
state.messages_append("emmet", message);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
console.log(`[handleDocumentSegmentation] Wrote ${segments.length} segment(s) for batch ${batchId} (${filename})`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function finishDocumentBatch(batchId: string, filename: string, state: StateManager): void {
|
|
72
|
+
const sourceTag = `import:document:${filename}`;
|
|
73
|
+
|
|
74
|
+
const emmettMessages = state.messages_get("emmet");
|
|
75
|
+
const docMessages = emmettMessages.filter(m => m.external === true && m.source_tag === sourceTag);
|
|
76
|
+
|
|
77
|
+
if (docMessages.length === 0) {
|
|
78
|
+
console.warn(`[finishDocumentBatch] No messages found for ${sourceTag} — skipping extraction`);
|
|
79
|
+
} else {
|
|
80
|
+
const extractionContext: ExtractionContext = {
|
|
81
|
+
personaId: "emmet",
|
|
82
|
+
channelDisplayName: "Document",
|
|
83
|
+
messages_context: [],
|
|
84
|
+
messages_analyze: docMessages,
|
|
85
|
+
sources: [sourceTag],
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const docSettings = state.getHuman().settings?.document;
|
|
89
|
+
queueAllScans(extractionContext, state, {
|
|
90
|
+
extraction_model: docSettings?.extraction_model,
|
|
91
|
+
external_filter: "only",
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
console.log(`[finishDocumentBatch] Queued extraction for ${docMessages.length} message(s) from ${filename}`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const updatedHuman = state.getHuman();
|
|
98
|
+
state.setHuman({
|
|
99
|
+
...updatedHuman,
|
|
100
|
+
settings: {
|
|
101
|
+
...updatedHuman.settings,
|
|
102
|
+
document: {
|
|
103
|
+
...updatedHuman.settings?.document,
|
|
104
|
+
processed_documents: {
|
|
105
|
+
...(updatedHuman.settings?.document?.processed_documents ?? {}),
|
|
106
|
+
[filename]: new Date().toISOString(),
|
|
107
|
+
},
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
console.log(`[finishDocumentBatch] Batch ${batchId} complete, ${filename} marked processed`);
|
|
113
|
+
}
|
|
@@ -92,7 +92,7 @@ export async function handleFactFind(response: LLMResponse, state: StateManager)
|
|
|
92
92
|
markMessagesExtracted(response, state, "f");
|
|
93
93
|
|
|
94
94
|
if (!result?.facts || !Array.isArray(result.facts)) {
|
|
95
|
-
console.
|
|
95
|
+
console.debug("[handleFactFind] No facts detected or invalid result");
|
|
96
96
|
return;
|
|
97
97
|
}
|
|
98
98
|
|
|
@@ -106,26 +106,26 @@ export async function handleFactFind(response: LLMResponse, state: StateManager)
|
|
|
106
106
|
for (const factResult of result.facts) {
|
|
107
107
|
// Only upsert facts that match a built-in name
|
|
108
108
|
if (!BUILT_IN_FACT_NAMES.has(factResult.name)) {
|
|
109
|
-
console.
|
|
109
|
+
console.warn(`[handleFactFind] Skipping non-built-in fact: "${factResult.name}"`);
|
|
110
110
|
continue;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
// Find the existing fact in state
|
|
114
114
|
const existingFact = human.facts.find(f => f.name === factResult.name);
|
|
115
115
|
if (!existingFact) {
|
|
116
|
-
console.
|
|
116
|
+
console.warn(`[handleFactFind] Skipping unknown fact: "${factResult.name}"`);
|
|
117
117
|
continue;
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
// Skip facts that already have descriptions (only fill empty ones)
|
|
121
121
|
if (existingFact.description && existingFact.description !== "") {
|
|
122
|
-
console.
|
|
122
|
+
console.debug(`[handleFactFind] Skipping fact with existing description: "${factResult.name}"`);
|
|
123
123
|
continue;
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
// Skip if the LLM returned a null/empty/non-string value — don't store booleans or nulls
|
|
127
127
|
if (!factResult.value || typeof factResult.value !== 'string') {
|
|
128
|
-
console.
|
|
128
|
+
console.warn(`[handleFactFind] Skipping fact with null/empty/non-string value: "${factResult.name}" (got ${typeof factResult.value})`);
|
|
129
129
|
continue;
|
|
130
130
|
}
|
|
131
131
|
|
|
@@ -165,7 +165,7 @@ export async function handleHumanTopicScan(response: LLMResponse, state: StateMa
|
|
|
165
165
|
markMessagesExtracted(response, state, "t");
|
|
166
166
|
|
|
167
167
|
if (!result?.topics || !Array.isArray(result.topics)) {
|
|
168
|
-
console.
|
|
168
|
+
console.debug("[handleHumanTopicScan] No topics detected or invalid result");
|
|
169
169
|
return;
|
|
170
170
|
}
|
|
171
171
|
|
|
@@ -185,7 +185,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
185
185
|
markMessagesExtracted(response, state, "p");
|
|
186
186
|
|
|
187
187
|
if (!result?.people || !Array.isArray(result.people)) {
|
|
188
|
-
console.
|
|
188
|
+
console.debug("[handleHumanPersonScan] No people detected or invalid result");
|
|
189
189
|
return;
|
|
190
190
|
}
|
|
191
191
|
|
|
@@ -231,7 +231,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
231
231
|
}
|
|
232
232
|
}
|
|
233
233
|
if (!matchedPerson) {
|
|
234
|
-
console.
|
|
234
|
+
console.debug(`[handleHumanPersonScan] Multi-match for "${candidate.name}" (${matches.length} hits) — no embedding above threshold, creating new record`);
|
|
235
235
|
}
|
|
236
236
|
} catch (err) {
|
|
237
237
|
console.warn(`[handleHumanPersonScan] Multi-match embedding failed for "${candidate.name}", using first match:`, err);
|
|
@@ -253,7 +253,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
253
253
|
if (isUnknownPlaceholder || isSingleton) {
|
|
254
254
|
matchedPerson = existing;
|
|
255
255
|
const reason = isUnknownPlaceholder ? 'unnamed placeholder' : 'singleton relationship';
|
|
256
|
-
console.
|
|
256
|
+
console.debug(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
|
|
257
257
|
}
|
|
258
258
|
} else {
|
|
259
259
|
// N>1 same relationship → cosine within that subset.
|
|
@@ -267,7 +267,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
267
267
|
: `all ${human.people.length} people`;
|
|
268
268
|
|
|
269
269
|
if (searchPool.length > 0) {
|
|
270
|
-
console.
|
|
270
|
+
console.debug(`[handleHumanPersonScan] "${candidate.name}": cosine against ${searchPool.length} embedded (${poolLabel})`);
|
|
271
271
|
try {
|
|
272
272
|
const embeddingService = getEmbeddingService();
|
|
273
273
|
const candidateText = getPersonEmbeddingText({
|
|
@@ -288,15 +288,15 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
288
288
|
}
|
|
289
289
|
const top3 = scores.sort((a, b) => b.sim - a.sim).slice(0, 3).map(s => `"${s.name}"=${s.sim.toFixed(3)}`).join(', ');
|
|
290
290
|
if (matchedPerson) {
|
|
291
|
-
console.
|
|
291
|
+
console.debug(`[handleHumanPersonScan] Cosine matched "${candidate.name}" → "${matchedPerson.name}" (${bestSimilarity.toFixed(3)}) | top3: ${top3}`);
|
|
292
292
|
} else {
|
|
293
|
-
console.
|
|
293
|
+
console.debug(`[handleHumanPersonScan] Cosine: no match above ${ZERO_MATCH_COSINE_THRESHOLD} for "${candidate.name}" | top3: ${top3}`);
|
|
294
294
|
}
|
|
295
295
|
} catch (err) {
|
|
296
296
|
console.warn(`[handleHumanPersonScan] Cosine failed for "${candidate.name}":`, err);
|
|
297
297
|
}
|
|
298
298
|
} else {
|
|
299
|
-
console.
|
|
299
|
+
console.debug(`[handleHumanPersonScan] "${candidate.name}": no embedded people in pool (${poolLabel}) — new person`);
|
|
300
300
|
}
|
|
301
301
|
}
|
|
302
302
|
}
|
|
@@ -305,7 +305,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
305
305
|
const linkedPersonaId = matchedPerson.identifiers
|
|
306
306
|
?.find(i => i.type === "Ei Persona")?.value;
|
|
307
307
|
if (linkedPersonaId) {
|
|
308
|
-
console.
|
|
308
|
+
console.debug(`[handleHumanPersonScan] Skipping update for "${candidate.name}" — scan marked as reflection drain (reflection_progress=1)`);
|
|
309
309
|
continue;
|
|
310
310
|
}
|
|
311
311
|
}
|
|
@@ -326,7 +326,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
326
326
|
: matches.length > 1
|
|
327
327
|
? `multi-match ambiguous (${matches.length} hits) — new record`
|
|
328
328
|
: "no match (new person)";
|
|
329
|
-
console.
|
|
329
|
+
console.debug(`[handleHumanPersonScan] person "${candidate.name}": ${matched}`);
|
|
330
330
|
}
|
|
331
331
|
console.log(`[handleHumanPersonScan] Processed ${result.people.length} person(s)`);
|
|
332
332
|
}
|
|
@@ -337,7 +337,7 @@ export async function handleEventScan(response: LLMResponse, state: StateManager
|
|
|
337
337
|
const result = response.parsed as { events?: Array<{ name: string; description: string; reason: string }> } | undefined;
|
|
338
338
|
|
|
339
339
|
if (!result?.events || !Array.isArray(result.events) || result.events.length === 0) {
|
|
340
|
-
console.
|
|
340
|
+
console.debug("[handleEventScan] No epic events detected");
|
|
341
341
|
return;
|
|
342
342
|
}
|
|
343
343
|
|
|
@@ -15,6 +15,7 @@ import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
|
|
|
15
15
|
import { handleDedupCurate } from "./dedup.js";
|
|
16
16
|
import { handleRoomResponse, handleRoomJudge } from "./rooms.js";
|
|
17
17
|
import { handlePersonaPreview } from "./persona-preview.js";
|
|
18
|
+
import { handleDocumentSegmentation } from "./document-segmentation.js";
|
|
18
19
|
|
|
19
20
|
export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
20
21
|
handlePersonaResponse,
|
|
@@ -41,4 +42,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
41
42
|
handlePersonaPreview,
|
|
42
43
|
[LLMNextStep.HandleTopicValidate]: handleDedupCurate,
|
|
43
44
|
[LLMNextStep.HandleReflectionCritic]: handleReflectionCritic,
|
|
45
|
+
[LLMNextStep.HandleDocumentSegmentation]: handleDocumentSegmentation,
|
|
44
46
|
};
|
|
@@ -14,7 +14,8 @@ import type {
|
|
|
14
14
|
RewriteResult,
|
|
15
15
|
RewriteSubjectMatch,
|
|
16
16
|
} from "../../prompts/ceremony/types.js";
|
|
17
|
-
import {
|
|
17
|
+
import { buildPersonRewriteSplitPrompt } from "../../prompts/ceremony/people-rewrite.js";
|
|
18
|
+
import { buildTopicRewriteSplitPrompt } from "../../prompts/ceremony/topic-rewrite.js";
|
|
18
19
|
import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
|
|
19
20
|
|
|
20
21
|
import { searchHumanData } from "../human-data-manager.js";
|
|
@@ -79,12 +80,10 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
79
80
|
}
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
|
|
83
|
-
const prompt =
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
subjects: subjectMatches,
|
|
87
|
-
});
|
|
83
|
+
const splitData = { item: currentItem, itemType, subjects: subjectMatches };
|
|
84
|
+
const prompt = itemType === "person"
|
|
85
|
+
? buildPersonRewriteSplitPrompt(splitData)
|
|
86
|
+
: buildTopicRewriteSplitPrompt(splitData);
|
|
88
87
|
|
|
89
88
|
state.queue_enqueue({
|
|
90
89
|
type: LLMRequestType.JSON,
|
|
@@ -125,6 +124,11 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
125
124
|
const human = state.getHuman();
|
|
126
125
|
const now = new Date().toISOString();
|
|
127
126
|
|
|
127
|
+
const originalItem = itemType === "topic"
|
|
128
|
+
? human.topics.find(t => t.id === itemId)
|
|
129
|
+
: human.people.find(p => p.id === itemId);
|
|
130
|
+
const originalCategory = itemType === "topic" ? (originalItem as Topic | undefined)?.category : undefined;
|
|
131
|
+
|
|
128
132
|
const allItems: DataItemBase[] = [
|
|
129
133
|
...human.topics, ...human.people,
|
|
130
134
|
];
|
|
@@ -228,11 +232,11 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
228
232
|
switch (item.type) {
|
|
229
233
|
case "topic": {
|
|
230
234
|
if (!item.category) {
|
|
231
|
-
console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category —
|
|
235
|
+
console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — inheriting from original (${originalCategory ?? "Interest"})`);
|
|
232
236
|
}
|
|
233
237
|
const topic: Topic = {
|
|
234
238
|
...baseFields,
|
|
235
|
-
category: item.category ?? "Interest",
|
|
239
|
+
category: item.category ?? originalCategory ?? "Interest",
|
|
236
240
|
exposure_current: 0.5,
|
|
237
241
|
exposure_desired: 0.5,
|
|
238
242
|
};
|
|
@@ -143,7 +143,7 @@ export async function queueEiHeartbeat(
|
|
|
143
143
|
}
|
|
144
144
|
|
|
145
145
|
const activePersonas = personas
|
|
146
|
-
.filter((p) => !p.is_archived && !p.is_paused && p.id !== "ei")
|
|
146
|
+
.filter((p) => !p.is_archived && !p.is_paused && !p.is_static && p.id !== "ei")
|
|
147
147
|
.map((p) => {
|
|
148
148
|
const msgs = sm.messages_get(p.id);
|
|
149
149
|
const lastHuman = [...msgs].reverse().find((m) => m.role === "human");
|
|
@@ -169,7 +169,7 @@ export async function queueEiHeartbeat(
|
|
|
169
169
|
}
|
|
170
170
|
|
|
171
171
|
const personasWithPendingUpdate = personas.filter(
|
|
172
|
-
(p) => !p.is_archived && !p.is_paused && p.id !== "ei" && p.pending_update?.critique
|
|
172
|
+
(p) => !p.is_archived && !p.is_paused && !p.is_static && p.id !== "ei" && p.pending_update?.critique
|
|
173
173
|
);
|
|
174
174
|
for (const p of personasWithPendingUpdate) {
|
|
175
175
|
items.push({
|
package/src/core/llm-client.ts
CHANGED
|
@@ -2,6 +2,36 @@ import type { ChatMessage, ProviderAccount, ModelConfig } from "./types.js";
|
|
|
2
2
|
const DEFAULT_TOKEN_LIMIT = 8192;
|
|
3
3
|
const DEFAULT_MAX_OUTPUT_TOKENS = 8000;
|
|
4
4
|
|
|
5
|
+
// Lazy verbose network dump — only active when EI_DEBUG_NETWORK_VERBOSE=1.
|
|
6
|
+
// Uses dynamic import so the web bundle never pulls in node:fs.
|
|
7
|
+
async function writeNetworkDump(
|
|
8
|
+
callNumber: number,
|
|
9
|
+
nextStep: string,
|
|
10
|
+
meta: { model: string; provider: string; latency_ms: number; status_code: number; tokens_in: number; tokens_out: number },
|
|
11
|
+
request: unknown,
|
|
12
|
+
response: unknown
|
|
13
|
+
): Promise<void> {
|
|
14
|
+
const dataPath = (typeof process !== "undefined" && process.env?.EI_DATA_PATH) ||
|
|
15
|
+
(typeof Bun !== "undefined" && (Bun as Record<string, unknown>).env && ((Bun as { env: Record<string, string> }).env.EI_DATA_PATH));
|
|
16
|
+
if (!dataPath) return;
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
const { mkdirSync, writeFileSync } = await import("node:fs");
|
|
20
|
+
const { join } = await import("node:path");
|
|
21
|
+
const logsDir = join(dataPath as string, "logs");
|
|
22
|
+
mkdirSync(logsDir, { recursive: true });
|
|
23
|
+
|
|
24
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
25
|
+
const safeName = nextStep.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
26
|
+
const filename = join(logsDir, `${timestamp}_call${callNumber}_${safeName}.json`);
|
|
27
|
+
|
|
28
|
+
const payload = JSON.stringify({ meta, request, response }, null, 2);
|
|
29
|
+
writeFileSync(filename, payload);
|
|
30
|
+
} catch {
|
|
31
|
+
// Silent — verbose dump failures must never crash the main path
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
5
35
|
export interface ProviderConfig {
|
|
6
36
|
baseURL: string;
|
|
7
37
|
apiKey: string;
|
|
@@ -22,6 +52,8 @@ export interface LLMCallOptions {
|
|
|
22
52
|
tools?: Record<string, unknown>[];
|
|
23
53
|
/** Fire-and-forget callback invoked after a successful response to increment usage counters. */
|
|
24
54
|
onUsageUpdate?: (modelId: string, usage: { calls: number; tokens_in: number; tokens_out: number }) => void;
|
|
55
|
+
/** Queue step name passed through to EI_DEBUG_NETWORK_VERBOSE file dumps. */
|
|
56
|
+
nextStep?: string;
|
|
25
57
|
}
|
|
26
58
|
|
|
27
59
|
export interface LLMRawResponse {
|
|
@@ -212,7 +244,7 @@ function logTokenLimit(model: string, source: string, tokens: number): void {
|
|
|
212
244
|
if (source === "default") {
|
|
213
245
|
console.warn(`[TokenLimit] Unknown model "${model}" — using conservative default (${DEFAULT_TOKEN_LIMIT})`);
|
|
214
246
|
} else {
|
|
215
|
-
console.
|
|
247
|
+
console.debug(`[TokenLimit] ${model}: ${source} → ${tokens} tokens (extraction budget: ${budget})`);
|
|
216
248
|
}
|
|
217
249
|
}
|
|
218
250
|
|
|
@@ -226,7 +258,7 @@ export async function callLLMRaw(
|
|
|
226
258
|
): Promise<LLMRawResponse> {
|
|
227
259
|
llmCallCount++;
|
|
228
260
|
|
|
229
|
-
const { signal, temperature = 0.7, onUsageUpdate } = options;
|
|
261
|
+
const { signal, temperature = 0.7, onUsageUpdate, nextStep = "unknown" } = options;
|
|
230
262
|
|
|
231
263
|
if (signal?.aborted) {
|
|
232
264
|
throw new Error("LLM call aborted");
|
|
@@ -251,7 +283,9 @@ export async function callLLMRaw(
|
|
|
251
283
|
|
|
252
284
|
const totalChars = finalMessages.reduce((sum, m) => sum + (m.content?.length ?? 0), 0);
|
|
253
285
|
const estimatedTokens = Math.ceil(totalChars / 4);
|
|
254
|
-
|
|
286
|
+
const modelLabel = model ?? "default";
|
|
287
|
+
console.log(`[LLM] Call #${llmCallCount} — ${config.name}:${modelLabel}, ~${estimatedTokens} tokens est.`);
|
|
288
|
+
const _llmCallStart = Date.now();
|
|
255
289
|
|
|
256
290
|
const normalizedBaseURL = config.baseURL.replace(/\/+$/, "");
|
|
257
291
|
|
|
@@ -274,7 +308,18 @@ export async function callLLMRaw(
|
|
|
274
308
|
};
|
|
275
309
|
|
|
276
310
|
if (modelConfig?.thinking_budget !== undefined) {
|
|
277
|
-
|
|
311
|
+
if (modelConfig.thinking_budget === 0) {
|
|
312
|
+
// Universal kill switch across all known providers. Non-conflicting — each reads
|
|
313
|
+
// whichever field it understands and ignores the rest.
|
|
314
|
+
requestBody.reasoning_effort = "none"; // Ollama, OpenAI-compat
|
|
315
|
+
requestBody.enable_thinking = false; // Rapid-MLX
|
|
316
|
+
} else {
|
|
317
|
+
// Pass all on-signals: providers that honor the token budget get it (Qwen3, Anthropic),
|
|
318
|
+
// providers that reduce thinking to on/off use reasoning_effort or enable_thinking.
|
|
319
|
+
requestBody.reasoning_effort = "high";
|
|
320
|
+
requestBody.enable_thinking = true;
|
|
321
|
+
requestBody.think = { budget_tokens: modelConfig.thinking_budget };
|
|
322
|
+
}
|
|
278
323
|
}
|
|
279
324
|
|
|
280
325
|
if (options.tools && options.tools.length > 0) {
|
|
@@ -296,9 +341,24 @@ export async function callLLMRaw(
|
|
|
296
341
|
|
|
297
342
|
const data = await response.json();
|
|
298
343
|
|
|
344
|
+
const _llmLatency = Date.now() - _llmCallStart;
|
|
345
|
+
const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
|
|
346
|
+
const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
|
|
347
|
+
console.log(`[LLM] Response #${llmCallCount} — ${response.status} ${_llmLatency}ms | in: ${tokensIn} out: ${tokensOut}`);
|
|
348
|
+
|
|
349
|
+
const isVerbose = (typeof process !== "undefined" && process.env?.EI_DEBUG_NETWORK_VERBOSE === "1") ||
|
|
350
|
+
(typeof Bun !== "undefined" && (Bun as { env: Record<string, string> }).env?.EI_DEBUG_NETWORK_VERBOSE === "1");
|
|
351
|
+
if (isVerbose) {
|
|
352
|
+
void writeNetworkDump(
|
|
353
|
+
llmCallCount,
|
|
354
|
+
nextStep,
|
|
355
|
+
{ model: modelLabel, provider: config.name, latency_ms: _llmLatency, status_code: response.status, tokens_in: tokensIn, tokens_out: tokensOut },
|
|
356
|
+
requestBody,
|
|
357
|
+
data
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
299
361
|
if (onUsageUpdate && modelConfig) {
|
|
300
|
-
const tokensIn = data.usage?.prompt_tokens ?? data.usage?.input_tokens ?? 0;
|
|
301
|
-
const tokensOut = data.usage?.completion_tokens ?? data.usage?.output_tokens ?? 0;
|
|
302
362
|
onUsageUpdate(modelConfig.id, { calls: 1, tokens_in: tokensIn, tokens_out: tokensOut });
|
|
303
363
|
}
|
|
304
364
|
|
|
@@ -177,25 +177,27 @@ export async function sendMessage(
|
|
|
177
177
|
|
|
178
178
|
const history = sm.messages_get(persona.id);
|
|
179
179
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
180
|
+
if (!persona.is_static) {
|
|
181
|
+
const traitExtractionData: PersonaTraitExtractionPromptData = {
|
|
182
|
+
persona_name: persona.display_name,
|
|
183
|
+
current_traits: persona.traits,
|
|
184
|
+
messages_context: history.slice(-11, -1),
|
|
185
|
+
messages_analyze: [message],
|
|
186
|
+
};
|
|
187
|
+
const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
|
|
188
|
+
|
|
189
|
+
sm.queue_enqueue({
|
|
190
|
+
type: LLMRequestType.JSON,
|
|
191
|
+
priority: LLMPriority.Low,
|
|
192
|
+
system: traitPrompt.system,
|
|
193
|
+
user: traitPrompt.user,
|
|
194
|
+
next_step: LLMNextStep.HandlePersonaTraitExtraction,
|
|
195
|
+
model: getModelForPersona(persona.id),
|
|
196
|
+
data: { personaId: persona.id, personaDisplayName: persona.display_name },
|
|
197
|
+
});
|
|
197
198
|
|
|
198
|
-
|
|
199
|
+
checkAndQueueHumanExtraction(sm, persona.id, persona.display_name, history);
|
|
200
|
+
}
|
|
199
201
|
}
|
|
200
202
|
|
|
201
203
|
// =============================================================================
|