ei-tui 0.1.22 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/AGENTS.md +16 -0
- package/src/core/handlers/dedup.ts +212 -0
- package/src/core/handlers/heartbeat.ts +16 -6
- package/src/core/handlers/human-matching.ts +29 -3
- package/src/core/handlers/index.ts +2 -0
- package/src/core/handlers/persona-response.ts +29 -3
- package/src/core/heartbeat-manager.ts +4 -0
- package/src/core/llm-client.ts +2 -1
- package/src/core/orchestrators/ceremony.ts +41 -33
- package/src/core/orchestrators/dedup-phase.ts +198 -0
- package/src/core/orchestrators/human-extraction.ts +2 -1
- package/src/core/orchestrators/index.ts +2 -2
- package/src/core/processor.ts +39 -9
- package/src/core/queue-processor.ts +9 -2
- package/src/core/state/personas.ts +10 -0
- package/src/core/state/queue.ts +1 -1
- package/src/core/state-manager.ts +6 -0
- package/src/core/types/entities.ts +10 -0
- package/src/core/types/enums.ts +2 -0
- package/src/core/types/llm.ts +3 -0
- package/src/prompts/ceremony/dedup.ts +258 -0
- package/src/prompts/ceremony/index.ts +3 -0
- package/src/prompts/ceremony/types.ts +45 -0
- package/src/prompts/heartbeat/check.ts +1 -0
- package/src/prompts/message-utils.ts +7 -1
- package/src/prompts/response/index.ts +3 -3
- package/src/prompts/response/sections.ts +3 -2
- package/tui/src/context/ei.tsx +13 -0
- package/tui/src/context/keyboard.tsx +2 -1
package/package.json
CHANGED
package/src/core/AGENTS.md
CHANGED
|
@@ -49,6 +49,22 @@ Priority queue for LLM requests:
|
|
|
49
49
|
|
|
50
50
|
**Async model**: Handlers queue work, don't await results inline.
|
|
51
51
|
|
|
52
|
+
### llm-client.ts
|
|
53
|
+
|
|
54
|
+
Multi-provider LLM abstraction layer:
|
|
55
|
+
- Handles requests to Anthropic, OpenAI, Bedrock, local models
|
|
56
|
+
- **Sets `max_tokens: 64000`** for all requests
|
|
57
|
+
- Prevents unbounded generation (test showed timeout after 2min without limit)
|
|
58
|
+
- Local models silently clamp to their configured maximums
|
|
59
|
+
- Anthropic Opus 4 accepts 64K (200K total context - 64K output = 136K input budget)
|
|
60
|
+
|
|
61
|
+
**JSON Response Parsing** (`parseJSONResponse()`):
|
|
62
|
+
- **Strategy 1**: Extract from markdown code blocks (```json)
|
|
63
|
+
- **Strategy 2**: Auto-repair malformed JSON (trailing commas, etc.)
|
|
64
|
+
- **Strategy 3**: Extract outermost `{...}` from mixed prose/JSON (handles LLM preamble)
|
|
65
|
+
|
|
66
|
+
No prompt changes needed for JSON-only output—parser handles natural language gracefully.
|
|
67
|
+
|
|
52
68
|
### handlers/index.ts (1000+ lines)
|
|
53
69
|
|
|
54
70
|
All `LLMNextStep` handlers in one file. Each handler:
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { StateManager } from "../state-manager.js";
|
|
2
|
+
import { LLMResponse } from "../types.js";
|
|
3
|
+
import type { DedupResult } from "../../prompts/ceremony/types.js";
|
|
4
|
+
import type { DataItemType, Fact, Trait, Topic, Person, Quote } from "../types/data-items.js";
|
|
5
|
+
import { getEmbeddingService } from "../embedding-service.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* handleDedupCurate — Process Opus deduplication decisions
|
|
9
|
+
*
|
|
10
|
+
* This handler receives merge decisions from Opus and applies them:
|
|
11
|
+
* 1. Updates: Entities with revised descriptions/merged data
|
|
12
|
+
* 2. Removes: Duplicate entities to delete (with foreign key updates)
|
|
13
|
+
* 3. Adds: New entities created from consolidation
|
|
14
|
+
*
|
|
15
|
+
* CRITICAL: Quote foreign keys must be updated BEFORE deletions to maintain
|
|
16
|
+
* referential integrity.
|
|
17
|
+
*/
|
|
18
|
+
export async function handleDedupCurate(
|
|
19
|
+
response: LLMResponse,
|
|
20
|
+
stateManager: StateManager
|
|
21
|
+
): Promise<void> {
|
|
22
|
+
const entity_type = response.request.data.entity_type as DataItemType;
|
|
23
|
+
const entity_ids = response.request.data.entity_ids as string[];
|
|
24
|
+
const state = stateManager.getHuman();
|
|
25
|
+
|
|
26
|
+
// Parse Opus response
|
|
27
|
+
let decisions: DedupResult;
|
|
28
|
+
try {
|
|
29
|
+
decisions = response.parsed as DedupResult;
|
|
30
|
+
if (!decisions || typeof decisions !== 'object') {
|
|
31
|
+
throw new Error("Invalid response format");
|
|
32
|
+
}
|
|
33
|
+
} catch (err) {
|
|
34
|
+
console.error(`[Dedup] Failed to parse Opus response:`, err);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Validate response structure
|
|
39
|
+
if (!Array.isArray(decisions.update) || !Array.isArray(decisions.remove) || !Array.isArray(decisions.add)) {
|
|
40
|
+
console.error(`[Dedup] Invalid response structure - missing update/remove/add arrays`);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
console.log(`[Dedup] Processing cluster: ${decisions.update.length} updates, ${decisions.remove.length} removals, ${decisions.add.length} additions`);
|
|
45
|
+
|
|
46
|
+
// HYDRATION: Fetch entities by ID (graceful degradation for missing)
|
|
47
|
+
const entityList = state[`${entity_type}s` as 'facts' | 'traits' | 'topics' | 'people'];
|
|
48
|
+
const entities = entity_ids
|
|
49
|
+
.map((id: string) => entityList.find((e: Fact | Trait | Topic | Person) => e.id === id))
|
|
50
|
+
.filter((e: Fact | Trait | Topic | Person | undefined): e is (Fact | Trait | Topic | Person) => e !== undefined);
|
|
51
|
+
|
|
52
|
+
if (entities.length === 0) {
|
|
53
|
+
console.warn(`[Dedup] No entities found for cluster (already merged?)`);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// =========================================================================
|
|
58
|
+
// PHASE 1: Update Quote foreign keys FIRST (before deletions)
|
|
59
|
+
// =========================================================================
|
|
60
|
+
|
|
61
|
+
for (const removal of decisions.remove) {
|
|
62
|
+
const quotes = state.quotes.filter((q: Quote) =>
|
|
63
|
+
q.data_item_ids.includes(removal.to_be_removed)
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
for (const quote of quotes) {
|
|
67
|
+
const updatedIds = quote.data_item_ids
|
|
68
|
+
.map((id: string) => id === removal.to_be_removed ? removal.replaced_by : id)
|
|
69
|
+
.filter((id: string, idx: number, arr: string[]) => arr.indexOf(id) === idx); // Dedupe
|
|
70
|
+
|
|
71
|
+
stateManager.human_quote_update(quote.id, {
|
|
72
|
+
data_item_ids: updatedIds
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (quotes.length > 0) {
|
|
77
|
+
console.log(`[Dedup] Updated ${quotes.length} quotes referencing ${removal.to_be_removed}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// =========================================================================
|
|
82
|
+
// PHASE 2: Apply updates (merge decisions)
|
|
83
|
+
// =========================================================================
|
|
84
|
+
|
|
85
|
+
for (const update of decisions.update) {
|
|
86
|
+
const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === update.id);
|
|
87
|
+
|
|
88
|
+
if (!entity) {
|
|
89
|
+
console.warn(`[Dedup] Entity ${update.id} not found (already merged?)`);
|
|
90
|
+
continue; // Graceful skip
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Recalculate embedding if description changed
|
|
94
|
+
let embedding = entity.embedding;
|
|
95
|
+
if (update.description !== entity.description) {
|
|
96
|
+
try {
|
|
97
|
+
const embeddingService = getEmbeddingService();
|
|
98
|
+
embedding = await embeddingService.embed(update.description);
|
|
99
|
+
} catch (err) {
|
|
100
|
+
console.warn(`[Dedup] Failed to recalculate embedding for ${update.id}`, err);
|
|
101
|
+
// Fallback to old embedding if recalculation fails
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Build complete entity with updates (preserve original fields if LLM omits them)
|
|
106
|
+
const updatedEntity = {
|
|
107
|
+
...entity,
|
|
108
|
+
name: update.name ?? entity.name,
|
|
109
|
+
description: update.description ?? entity.description,
|
|
110
|
+
sentiment: update.sentiment ?? entity.sentiment,
|
|
111
|
+
last_updated: new Date().toISOString(),
|
|
112
|
+
embedding,
|
|
113
|
+
// Type-specific fields
|
|
114
|
+
...(update.strength !== undefined && { strength: update.strength }),
|
|
115
|
+
...(update.confidence !== undefined && { confidence: update.confidence }),
|
|
116
|
+
...(update.exposure_current !== undefined && { exposure_current: update.exposure_current }),
|
|
117
|
+
...(update.exposure_desired !== undefined && { exposure_desired: update.exposure_desired }),
|
|
118
|
+
...(update.relationship !== undefined && { relationship: update.relationship }),
|
|
119
|
+
...(update.category !== undefined && { category: update.category }),
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
// Type-safe cast based on entity_type
|
|
123
|
+
if (entity_type === 'fact') {
|
|
124
|
+
stateManager.human_fact_upsert(updatedEntity as Fact);
|
|
125
|
+
} else if (entity_type === 'trait') {
|
|
126
|
+
stateManager.human_trait_upsert(updatedEntity as Trait);
|
|
127
|
+
} else if (entity_type === 'topic') {
|
|
128
|
+
stateManager.human_topic_upsert(updatedEntity as Topic);
|
|
129
|
+
} else if (entity_type === 'person') {
|
|
130
|
+
stateManager.human_person_upsert(updatedEntity as Person);
|
|
131
|
+
}
|
|
132
|
+
console.log(`[Dedup] Updated ${entity_type} "${update.name}"`);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// =========================================================================
|
|
136
|
+
// PHASE 3: Apply removals (soft-delete with replaced_by tracking)
|
|
137
|
+
// =========================================================================
|
|
138
|
+
|
|
139
|
+
for (const removal of decisions.remove) {
|
|
140
|
+
const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === removal.to_be_removed);
|
|
141
|
+
|
|
142
|
+
if (!entity) {
|
|
143
|
+
console.warn(`[Dedup] Entity ${removal.to_be_removed} already deleted`);
|
|
144
|
+
continue; // Graceful skip
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Remove via StateManager (also cleans up quote references)
|
|
148
|
+
const removeMethod = `human_${entity_type}_remove` as
|
|
149
|
+
'human_fact_remove' | 'human_trait_remove' | 'human_topic_remove' | 'human_person_remove';
|
|
150
|
+
|
|
151
|
+
const removed = stateManager[removeMethod](removal.to_be_removed);
|
|
152
|
+
if (removed) {
|
|
153
|
+
console.log(`[Dedup] Removed ${entity_type} "${entity.name}" (merged into ${removal.replaced_by})`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// =========================================================================
|
|
158
|
+
// PHASE 4: Apply additions (new entities from consolidation)
|
|
159
|
+
// =========================================================================
|
|
160
|
+
|
|
161
|
+
for (const addition of decisions.add) {
|
|
162
|
+
// Compute embedding for new entity
|
|
163
|
+
let embedding: number[] | undefined;
|
|
164
|
+
try {
|
|
165
|
+
const embeddingService = getEmbeddingService();
|
|
166
|
+
embedding = await embeddingService.embed(addition.description);
|
|
167
|
+
} catch (err) {
|
|
168
|
+
console.warn(`[Dedup] Failed to compute embedding for new entity "${addition.name}"`, err);
|
|
169
|
+
continue; // Skip this addition if embedding fails
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Generate ID for new entity
|
|
173
|
+
const id = crypto.randomUUID();
|
|
174
|
+
|
|
175
|
+
// Build complete entity
|
|
176
|
+
const newEntity = {
|
|
177
|
+
id,
|
|
178
|
+
type: entity_type,
|
|
179
|
+
name: addition.name,
|
|
180
|
+
description: addition.description,
|
|
181
|
+
sentiment: addition.sentiment ?? 0.0,
|
|
182
|
+
last_updated: new Date().toISOString(),
|
|
183
|
+
embedding,
|
|
184
|
+
// Type-specific fields with defaults
|
|
185
|
+
...(entity_type === 'trait' && { strength: addition.strength ?? 0.5 }),
|
|
186
|
+
...(entity_type === 'fact' && {
|
|
187
|
+
confidence: addition.confidence ?? 0.5,
|
|
188
|
+
validated: 'unknown' as import("../types/enums.js").ValidationLevel,
|
|
189
|
+
validated_date: ''
|
|
190
|
+
}),
|
|
191
|
+
...((entity_type === 'topic' || entity_type === 'person') && {
|
|
192
|
+
exposure_current: addition.exposure_current ?? 0.0,
|
|
193
|
+
exposure_desired: addition.exposure_desired ?? 0.5,
|
|
194
|
+
last_ei_asked: null
|
|
195
|
+
}),
|
|
196
|
+
...(entity_type === 'person' && { relationship: addition.relationship ?? 'Unknown' }),
|
|
197
|
+
...(entity_type === 'topic' && { category: addition.category ?? 'Interest' }),
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
// Type-safe cast based on entity_type
|
|
201
|
+
if (entity_type === 'fact') {
|
|
202
|
+
stateManager.human_fact_upsert(newEntity as Fact);
|
|
203
|
+
} else if (entity_type === 'trait') {
|
|
204
|
+
stateManager.human_trait_upsert(newEntity as Trait);
|
|
205
|
+
} else if (entity_type === 'topic') {
|
|
206
|
+
stateManager.human_topic_upsert(newEntity as Topic);
|
|
207
|
+
} else if (entity_type === 'person') {
|
|
208
|
+
stateManager.human_person_upsert(newEntity as Person);
|
|
209
|
+
}
|
|
210
|
+
console.log(`[Dedup] Added new ${entity_type} "${addition.name}"`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ContextStatus,
|
|
3
|
+
LLMNextStep,
|
|
3
4
|
ValidationLevel,
|
|
4
5
|
type LLMResponse,
|
|
5
6
|
type Message,
|
|
@@ -18,15 +19,17 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
|
|
|
18
19
|
|
|
19
20
|
const result = response.parsed as HeartbeatCheckResult | undefined;
|
|
20
21
|
if (!result) {
|
|
21
|
-
console.error(
|
|
22
|
+
console.error(`[HeartbeatCheck ${personaDisplayName}] No parsed result`);
|
|
22
23
|
return;
|
|
23
24
|
}
|
|
25
|
+
console.log(`[HeartbeatCheck ${personaDisplayName}] Parsed result - should_respond: ${result.should_respond}, topic: ${result.topic ?? '(none)'}, message: ${result.message ? '(present)' : '(none)'}`);
|
|
24
26
|
|
|
25
27
|
const now = new Date().toISOString();
|
|
26
28
|
state.persona_update(personaId, { last_heartbeat: now });
|
|
29
|
+
state.queue_clearPersonaResponses(personaId, LLMNextStep.HandleHeartbeatCheck);
|
|
27
30
|
|
|
28
31
|
if (!result.should_respond) {
|
|
29
|
-
console.log(`[
|
|
32
|
+
console.log(`[HeartbeatCheck ${personaDisplayName}] Chose not to reach out (should_respond=false)`);
|
|
30
33
|
return;
|
|
31
34
|
}
|
|
32
35
|
|
|
@@ -40,20 +43,24 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
|
|
|
40
43
|
context_status: ContextStatus.Default,
|
|
41
44
|
};
|
|
42
45
|
state.messages_append(personaId, message);
|
|
43
|
-
console.log(`[
|
|
46
|
+
console.log(`[HeartbeatCheck ${personaDisplayName}] Added proactive message - topic: ${result.topic ?? 'general'}, message: "${result.message.substring(0, 100)}${result.message.length > 100 ? '...' : ''}"`);
|
|
47
|
+
} else {
|
|
48
|
+
console.log(`[HeartbeatCheck ${personaDisplayName}] should_respond=true but no message provided`);
|
|
44
49
|
}
|
|
45
50
|
}
|
|
46
51
|
|
|
47
52
|
export function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
|
|
48
53
|
const result = response.parsed as EiHeartbeatResult | undefined;
|
|
49
54
|
if (!result) {
|
|
50
|
-
console.error("[
|
|
55
|
+
console.error("[EiHeartbeat] No parsed result");
|
|
51
56
|
return;
|
|
52
57
|
}
|
|
58
|
+
console.log(`[EiHeartbeat] Parsed result - should_respond: ${result.should_respond}, id: ${result.id ?? '(none)'}, my_response: ${result.my_response ? '(present)' : '(none)'}`);
|
|
53
59
|
const now = new Date().toISOString();
|
|
54
60
|
state.persona_update("ei", { last_heartbeat: now });
|
|
61
|
+
state.queue_clearPersonaResponses("ei", LLMNextStep.HandleEiHeartbeat);
|
|
55
62
|
if (!result.should_respond || !result.id) {
|
|
56
|
-
console.log("[
|
|
63
|
+
console.log("[EiHeartbeat] Chose not to reach out (should_respond=false or no id)");
|
|
57
64
|
return;
|
|
58
65
|
}
|
|
59
66
|
const isTUI = response.request.data.isTUI as boolean;
|
|
@@ -81,7 +88,10 @@ export function handleEiHeartbeat(response: LLMResponse, state: StateManager): v
|
|
|
81
88
|
return;
|
|
82
89
|
}
|
|
83
90
|
|
|
84
|
-
if (result.my_response)
|
|
91
|
+
if (result.my_response) {
|
|
92
|
+
console.log(`[EiHeartbeat] Sending message: "${result.my_response.substring(0, 100)}${result.my_response.length > 100 ? '...' : ''}"`);
|
|
93
|
+
sendMessage(result.my_response);
|
|
94
|
+
}
|
|
85
95
|
|
|
86
96
|
switch (found.type) {
|
|
87
97
|
case "person":
|
|
@@ -27,9 +27,27 @@ export function handleHumanItemMatch(response: LLMResponse, state: StateManager)
|
|
|
27
27
|
const candidateType = response.request.data.candidateType as DataItemType;
|
|
28
28
|
const personaId = response.request.data.personaId as string;
|
|
29
29
|
const personaDisplayName = response.request.data.personaDisplayName as string;
|
|
30
|
-
const
|
|
30
|
+
const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
|
|
31
31
|
const allMessages = state.messages_get(personaId);
|
|
32
|
-
|
|
32
|
+
|
|
33
|
+
let messages_context: Message[];
|
|
34
|
+
let messages_analyze: Message[];
|
|
35
|
+
|
|
36
|
+
if (messageIdsToMark && messageIdsToMark.length > 0) {
|
|
37
|
+
const messageIdSet = new Set(messageIdsToMark);
|
|
38
|
+
messages_analyze = allMessages.filter(m => messageIdSet.has(m.id));
|
|
39
|
+
const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
|
|
40
|
+
messages_context = allMessages.filter(m =>
|
|
41
|
+
!messageIdSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
|
|
42
|
+
);
|
|
43
|
+
} else {
|
|
44
|
+
// Fallback to existing behavior
|
|
45
|
+
const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
|
|
46
|
+
const split = splitMessagesByTimestamp(allMessages, analyzeFrom);
|
|
47
|
+
messages_context = split.messages_context;
|
|
48
|
+
messages_analyze = split.messages_analyze;
|
|
49
|
+
}
|
|
50
|
+
|
|
33
51
|
const context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string } = {
|
|
34
52
|
personaId,
|
|
35
53
|
personaDisplayName,
|
|
@@ -212,6 +230,12 @@ export async function handleHumanItemUpdate(response: LLMResponse, state: StateM
|
|
|
212
230
|
console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
|
|
213
231
|
}
|
|
214
232
|
|
|
233
|
+
function normalizeQuotes(text: string): string {
|
|
234
|
+
return text
|
|
235
|
+
.replace(/[\u201C\u201D]/g, '"') // Curly double quotes to straight
|
|
236
|
+
.replace(/[\u2018\u2019]/g, "'"); // Curly single quotes to straight
|
|
237
|
+
}
|
|
238
|
+
|
|
215
239
|
async function validateAndStoreQuotes(
|
|
216
240
|
candidates: Array<{ text: string; reason: string }> | undefined,
|
|
217
241
|
messages: Message[],
|
|
@@ -226,7 +250,9 @@ async function validateAndStoreQuotes(
|
|
|
226
250
|
let found = false;
|
|
227
251
|
for (const message of messages) {
|
|
228
252
|
const msgText = getMessageText(message);
|
|
229
|
-
const
|
|
253
|
+
const normalizedMsg = normalizeQuotes(msgText);
|
|
254
|
+
const normalizedQuote = normalizeQuotes(candidate.text);
|
|
255
|
+
const start = normalizedMsg.indexOf(normalizedQuote);
|
|
230
256
|
if (start !== -1) {
|
|
231
257
|
const end = start + candidate.text.length;
|
|
232
258
|
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
import { handleHumanFactScan, handleHumanTraitScan, handleHumanTopicScan, handleHumanPersonScan } from "./human-extraction.js";
|
|
18
18
|
import { handleHumanItemMatch, handleHumanItemUpdate } from "./human-matching.js";
|
|
19
19
|
import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
|
|
20
|
+
import { handleDedupCurate } from "./dedup.js";
|
|
20
21
|
|
|
21
22
|
export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
22
23
|
handlePersonaResponse,
|
|
@@ -41,4 +42,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
41
42
|
handleToolContinuation,
|
|
42
43
|
handleRewriteScan,
|
|
43
44
|
handleRewriteRewrite,
|
|
45
|
+
handleDedupCurate,
|
|
44
46
|
};
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ContextStatus,
|
|
3
|
+
LLMNextStep,
|
|
3
4
|
type LLMResponse,
|
|
4
5
|
type Message,
|
|
5
6
|
} from "../types.js";
|
|
6
7
|
import type { StateManager } from "../state-manager.js";
|
|
7
8
|
import type { PersonaResponseResult } from "../../prompts/response/index.js";
|
|
9
|
+
import { handlers } from "./index.js";
|
|
8
10
|
|
|
9
11
|
export type ResponseHandler = (response: LLMResponse, state: StateManager) => void | Promise<void>;
|
|
10
12
|
|
|
@@ -87,11 +89,35 @@ export function handlePersonaResponse(response: LLMResponse, state: StateManager
|
|
|
87
89
|
/**
|
|
88
90
|
* handleToolContinuation — second LLM call in the tool flow (may loop if LLM calls more tools).
|
|
89
91
|
* The QueueProcessor already injected tool history into messages and got the
|
|
90
|
-
* final persona response.
|
|
92
|
+
* final persona response. Route to the original handler based on originalNextStep in data.
|
|
91
93
|
*/
|
|
92
94
|
export function handleToolContinuation(response: LLMResponse, state: StateManager): void {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
const originalStep = response.request.data.originalNextStep as LLMNextStep | undefined;
|
|
96
|
+
|
|
97
|
+
if (!originalStep) {
|
|
98
|
+
console.error(`[handleToolContinuation] No originalNextStep in data, falling back to handlePersonaResponse`);
|
|
99
|
+
handlePersonaResponse(response, state);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
console.log(`[handleToolContinuation] Original request was ${originalStep}, routing accordingly`);
|
|
104
|
+
|
|
105
|
+
const handler = handlers[originalStep];
|
|
106
|
+
|
|
107
|
+
if (!handler) {
|
|
108
|
+
console.error(`[handleToolContinuation] No handler found for ${originalStep}, falling back to handlePersonaResponse`);
|
|
109
|
+
handlePersonaResponse(response, state);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Avoid infinite loop - if original was already HandleToolContinuation, go to PersonaResponse
|
|
114
|
+
if (originalStep === "handleToolContinuation") {
|
|
115
|
+
console.log(`[handleToolContinuation] Original was tool continuation, routing to handlePersonaResponse`);
|
|
116
|
+
handlePersonaResponse(response, state);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
handler(response, state);
|
|
95
121
|
}
|
|
96
122
|
|
|
97
123
|
export function handleOneShot(_response: LLMResponse, _state: StateManager): void {
|
|
@@ -188,6 +188,8 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
|
|
|
188
188
|
const persona = sm.persona_getById(personaId);
|
|
189
189
|
if (!persona) return;
|
|
190
190
|
sm.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
|
|
191
|
+
const model = getModelForPersona(sm, personaId);
|
|
192
|
+
console.log(`[HeartbeatCheck ${persona.display_name}] Queueing heartbeat check (model: ${model})`);
|
|
191
193
|
const human = sm.getHuman();
|
|
192
194
|
const history = sm.messages_get(personaId);
|
|
193
195
|
const contextWindowHours = persona.context_window_hours ?? DEFAULT_CONTEXT_WINDOW_HOURS;
|
|
@@ -228,6 +230,7 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
|
|
|
228
230
|
};
|
|
229
231
|
|
|
230
232
|
const prompt = buildHeartbeatCheckPrompt(promptData);
|
|
233
|
+
console.log(`[HeartbeatCheck ${persona.display_name}] Prompt data - topics: ${promptData.human.topics.length}, people: ${promptData.human.people.length}, inactive_days: ${inactiveDays}`);
|
|
231
234
|
|
|
232
235
|
sm.queue_enqueue({
|
|
233
236
|
type: LLMRequestType.JSON,
|
|
@@ -238,4 +241,5 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
|
|
|
238
241
|
model: getModelForPersona(sm, personaId),
|
|
239
242
|
data: { personaId, personaDisplayName: persona.display_name },
|
|
240
243
|
});
|
|
244
|
+
console.log(`[HeartbeatCheck ${persona.display_name}] Request queued`);
|
|
241
245
|
}
|
package/src/core/llm-client.ts
CHANGED
|
@@ -60,7 +60,7 @@ export function resolveModel(modelSpec?: string, accounts?: ProviderAccount[]):
|
|
|
60
60
|
if (accounts) {
|
|
61
61
|
const searchName = provider || modelSpec; // If no ":", the whole spec might be an account name
|
|
62
62
|
const matchingAccount = accounts.find(
|
|
63
|
-
(acc) => acc.name.toLowerCase() === searchName.toLowerCase() && acc.enabled
|
|
63
|
+
(acc) => acc.name.toLowerCase() === searchName.toLowerCase() && acc.enabled && acc.type === "llm"
|
|
64
64
|
);
|
|
65
65
|
if (matchingAccount) {
|
|
66
66
|
// If bare account name was used, get model from account's default_model
|
|
@@ -189,6 +189,7 @@ export async function callLLMRaw(
|
|
|
189
189
|
model,
|
|
190
190
|
messages: finalMessages,
|
|
191
191
|
temperature,
|
|
192
|
+
max_tokens: 64000, // Opus 4: 128K max output, 200K total context. Local models clamp to their config. Prevents runaway generation.
|
|
192
193
|
};
|
|
193
194
|
|
|
194
195
|
if (options.tools && options.tools.length > 0) {
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
type ExtractionOptions,
|
|
11
11
|
} from "./human-extraction.js";
|
|
12
12
|
import { queuePersonaTopicScan, type PersonaTopicContext } from "./persona-topics.js";
|
|
13
|
+
import { queueDedupPhase } from "./dedup-phase.js";
|
|
13
14
|
import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt, buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
|
|
14
15
|
|
|
15
16
|
export function isNewDay(lastCeremony: string | undefined, now: Date): boolean {
|
|
@@ -69,40 +70,19 @@ export function startCeremony(state: StateManager): void {
|
|
|
69
70
|
},
|
|
70
71
|
});
|
|
71
72
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
!p.is_archived &&
|
|
76
|
-
!p.is_static
|
|
77
|
-
);
|
|
73
|
+
// PHASE 1: Deduplication (runs BEFORE Expose)
|
|
74
|
+
console.log("[ceremony] Starting Phase 1: Deduplication");
|
|
75
|
+
queueDedupPhase(state);
|
|
78
76
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const lastActivity = p.last_activity ? new Date(p.last_activity).getTime() : 0;
|
|
85
|
-
return lastActivity > lastCeremony;
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
console.log(`[ceremony] Processing ${personasWithActivity.length} personas with activity (of ${activePersonas.length} active)`);
|
|
89
|
-
|
|
90
|
-
const options: ExtractionOptions = { ceremony_progress: true };
|
|
91
|
-
|
|
92
|
-
for (let i = 0; i < personasWithActivity.length; i++) {
|
|
93
|
-
const persona = personasWithActivity[i];
|
|
94
|
-
const isLast = i === personasWithActivity.length - 1;
|
|
95
|
-
|
|
96
|
-
console.log(`[ceremony] Queuing exposure for ${persona.display_name} (${i + 1}/${personasWithActivity.length})${isLast ? " (last)" : ""}`);
|
|
97
|
-
queueExposurePhase(persona.id, state, options);
|
|
77
|
+
// Check if dedup work was queued
|
|
78
|
+
if (!state.queue_hasPendingCeremonies()) {
|
|
79
|
+
// No dedup work found → immediately advance to Expose phase
|
|
80
|
+
console.log("[ceremony] No dedup work, advancing to Expose phase");
|
|
81
|
+
handleCeremonyProgress(state, 1);
|
|
98
82
|
}
|
|
99
83
|
|
|
100
84
|
const duration = Date.now() - startTime;
|
|
101
|
-
console.log(`[ceremony]
|
|
102
|
-
|
|
103
|
-
// Check immediately — if zero messages were queued (no unextracted messages for any persona),
|
|
104
|
-
// this will see an empty queue and proceed directly to Decay → Expire.
|
|
105
|
-
handleCeremonyProgress(state);
|
|
85
|
+
console.log(`[ceremony] Dedup phase queued in ${duration}ms`);
|
|
106
86
|
}
|
|
107
87
|
|
|
108
88
|
/**
|
|
@@ -193,11 +173,40 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
|
|
|
193
173
|
* If any ceremony_progress items remain in the queue, does nothing — more work pending.
|
|
194
174
|
* If the queue is clear of ceremony items, advances to Decay → Prune → Expire.
|
|
195
175
|
*/
|
|
196
|
-
export function handleCeremonyProgress(state: StateManager): void {
|
|
176
|
+
export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
|
|
197
177
|
if (state.queue_hasPendingCeremonies()) {
|
|
198
|
-
return; // Still processing
|
|
178
|
+
return; // Still processing ceremony items
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (lastPhase === 1) {
|
|
182
|
+
// Dedup phase complete → start Expose phase
|
|
183
|
+
console.log("[ceremony:progress] Dedup complete, starting Expose phase");
|
|
184
|
+
|
|
185
|
+
const human = state.getHuman();
|
|
186
|
+
const personas = state.persona_getAll();
|
|
187
|
+
const activePersonas = personas.filter(p =>
|
|
188
|
+
!p.is_paused &&
|
|
189
|
+
!p.is_archived &&
|
|
190
|
+
!p.is_static
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
const lastCeremony = human.settings?.ceremony?.last_ceremony
|
|
194
|
+
? new Date(human.settings.ceremony.last_ceremony).getTime()
|
|
195
|
+
: 0;
|
|
196
|
+
|
|
197
|
+
const personasWithActivity = activePersonas.filter(p => {
|
|
198
|
+
const lastActivity = p.last_activity ? new Date(p.last_activity).getTime() : 0;
|
|
199
|
+
return lastActivity > lastCeremony;
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
const options: ExtractionOptions = { ceremony_progress: 2 };
|
|
203
|
+
for (const persona of personasWithActivity) {
|
|
204
|
+
queueExposurePhase(persona.id, state, options);
|
|
205
|
+
}
|
|
206
|
+
return;
|
|
199
207
|
}
|
|
200
208
|
|
|
209
|
+
// Phase 2 (Expose) complete → advance to Decay/Prune/Expire/Explore
|
|
201
210
|
console.log("[ceremony:progress] All exposure scans complete, advancing to Decay");
|
|
202
211
|
|
|
203
212
|
const personas = state.persona_getAll();
|
|
@@ -215,7 +224,6 @@ export function handleCeremonyProgress(state: StateManager): void {
|
|
|
215
224
|
if (eiIndex > -1) {
|
|
216
225
|
activePersonas.splice(eiIndex, 1);
|
|
217
226
|
}
|
|
218
|
-
|
|
219
227
|
// Decay phase: apply decay + prune for ALL active personas
|
|
220
228
|
for (const persona of activePersonas) {
|
|
221
229
|
applyDecayPhase(persona.id, state);
|