ei-tui 0.1.22 → 0.1.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ei-tui",
3
- "version": "0.1.22",
3
+ "version": "0.1.24",
4
4
  "author": "Flare576",
5
5
  "repository": {
6
6
  "type": "git",
@@ -49,6 +49,22 @@ Priority queue for LLM requests:
49
49
 
50
50
  **Async model**: Handlers queue work, don't await results inline.
51
51
 
52
+ ### llm-client.ts
53
+
54
+ Multi-provider LLM abstraction layer:
55
+ - Handles requests to Anthropic, OpenAI, Bedrock, local models
56
+ - **Sets `max_tokens: 64000`** for all requests
57
+ - Prevents unbounded generation (test showed timeout after 2min without limit)
58
+ - Local models silently clamp to their configured maximums
59
+ - Anthropic Opus 4 accepts 64K (200K total context - 64K output = 136K input budget)
60
+
61
+ **JSON Response Parsing** (`parseJSONResponse()`):
62
+ - **Strategy 1**: Extract from markdown code blocks (```json)
63
+ - **Strategy 2**: Auto-repair malformed JSON (trailing commas, etc.)
64
+ - **Strategy 3**: Extract outermost `{...}` from mixed prose/JSON (handles LLM preamble)
65
+
66
+ No prompt changes needed for JSON-only output—parser handles natural language gracefully.
67
+
52
68
  ### handlers/index.ts (1000+ lines)
53
69
 
54
70
  All `LLMNextStep` handlers in one file. Each handler:
@@ -0,0 +1,212 @@
1
+ import { StateManager } from "../state-manager.js";
2
+ import { LLMResponse } from "../types.js";
3
+ import type { DedupResult } from "../../prompts/ceremony/types.js";
4
+ import type { DataItemType, Fact, Trait, Topic, Person, Quote } from "../types/data-items.js";
5
+ import { getEmbeddingService } from "../embedding-service.js";
6
+
7
+ /**
8
+ * handleDedupCurate — Process Opus deduplication decisions
9
+ *
10
+ * This handler receives merge decisions from Opus and applies them:
11
+ * 1. Updates: Entities with revised descriptions/merged data
12
+ * 2. Removes: Duplicate entities to delete (with foreign key updates)
13
+ * 3. Adds: New entities created from consolidation
14
+ *
15
+ * CRITICAL: Quote foreign keys must be updated BEFORE deletions to maintain
16
+ * referential integrity.
17
+ */
18
+ export async function handleDedupCurate(
19
+ response: LLMResponse,
20
+ stateManager: StateManager
21
+ ): Promise<void> {
22
+ const entity_type = response.request.data.entity_type as DataItemType;
23
+ const entity_ids = response.request.data.entity_ids as string[];
24
+ const state = stateManager.getHuman();
25
+
26
+ // Parse Opus response
27
+ let decisions: DedupResult;
28
+ try {
29
+ decisions = response.parsed as DedupResult;
30
+ if (!decisions || typeof decisions !== 'object') {
31
+ throw new Error("Invalid response format");
32
+ }
33
+ } catch (err) {
34
+ console.error(`[Dedup] Failed to parse Opus response:`, err);
35
+ return;
36
+ }
37
+
38
+ // Validate response structure
39
+ if (!Array.isArray(decisions.update) || !Array.isArray(decisions.remove) || !Array.isArray(decisions.add)) {
40
+ console.error(`[Dedup] Invalid response structure - missing update/remove/add arrays`);
41
+ return;
42
+ }
43
+
44
+ console.log(`[Dedup] Processing cluster: ${decisions.update.length} updates, ${decisions.remove.length} removals, ${decisions.add.length} additions`);
45
+
46
+ // HYDRATION: Fetch entities by ID (graceful degradation for missing)
47
+ const entityList = state[`${entity_type}s` as 'facts' | 'traits' | 'topics' | 'people'];
48
+ const entities = entity_ids
49
+ .map((id: string) => entityList.find((e: Fact | Trait | Topic | Person) => e.id === id))
50
+ .filter((e: Fact | Trait | Topic | Person | undefined): e is (Fact | Trait | Topic | Person) => e !== undefined);
51
+
52
+ if (entities.length === 0) {
53
+ console.warn(`[Dedup] No entities found for cluster (already merged?)`);
54
+ return;
55
+ }
56
+
57
+ // =========================================================================
58
+ // PHASE 1: Update Quote foreign keys FIRST (before deletions)
59
+ // =========================================================================
60
+
61
+ for (const removal of decisions.remove) {
62
+ const quotes = state.quotes.filter((q: Quote) =>
63
+ q.data_item_ids.includes(removal.to_be_removed)
64
+ );
65
+
66
+ for (const quote of quotes) {
67
+ const updatedIds = quote.data_item_ids
68
+ .map((id: string) => id === removal.to_be_removed ? removal.replaced_by : id)
69
+ .filter((id: string, idx: number, arr: string[]) => arr.indexOf(id) === idx); // Dedupe
70
+
71
+ stateManager.human_quote_update(quote.id, {
72
+ data_item_ids: updatedIds
73
+ });
74
+ }
75
+
76
+ if (quotes.length > 0) {
77
+ console.log(`[Dedup] Updated ${quotes.length} quotes referencing ${removal.to_be_removed}`);
78
+ }
79
+ }
80
+
81
+ // =========================================================================
82
+ // PHASE 2: Apply updates (merge decisions)
83
+ // =========================================================================
84
+
85
+ for (const update of decisions.update) {
86
+ const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === update.id);
87
+
88
+ if (!entity) {
89
+ console.warn(`[Dedup] Entity ${update.id} not found (already merged?)`);
90
+ continue; // Graceful skip
91
+ }
92
+
93
+ // Recalculate embedding if description changed
94
+ let embedding = entity.embedding;
95
+ if (update.description !== entity.description) {
96
+ try {
97
+ const embeddingService = getEmbeddingService();
98
+ embedding = await embeddingService.embed(update.description);
99
+ } catch (err) {
100
+ console.warn(`[Dedup] Failed to recalculate embedding for ${update.id}`, err);
101
+ // Fallback to old embedding if recalculation fails
102
+ }
103
+ }
104
+
105
+ // Build complete entity with updates (preserve original fields if LLM omits them)
106
+ const updatedEntity = {
107
+ ...entity,
108
+ name: update.name ?? entity.name,
109
+ description: update.description ?? entity.description,
110
+ sentiment: update.sentiment ?? entity.sentiment,
111
+ last_updated: new Date().toISOString(),
112
+ embedding,
113
+ // Type-specific fields
114
+ ...(update.strength !== undefined && { strength: update.strength }),
115
+ ...(update.confidence !== undefined && { confidence: update.confidence }),
116
+ ...(update.exposure_current !== undefined && { exposure_current: update.exposure_current }),
117
+ ...(update.exposure_desired !== undefined && { exposure_desired: update.exposure_desired }),
118
+ ...(update.relationship !== undefined && { relationship: update.relationship }),
119
+ ...(update.category !== undefined && { category: update.category }),
120
+ };
121
+
122
+ // Type-safe cast based on entity_type
123
+ if (entity_type === 'fact') {
124
+ stateManager.human_fact_upsert(updatedEntity as Fact);
125
+ } else if (entity_type === 'trait') {
126
+ stateManager.human_trait_upsert(updatedEntity as Trait);
127
+ } else if (entity_type === 'topic') {
128
+ stateManager.human_topic_upsert(updatedEntity as Topic);
129
+ } else if (entity_type === 'person') {
130
+ stateManager.human_person_upsert(updatedEntity as Person);
131
+ }
132
+ console.log(`[Dedup] Updated ${entity_type} "${update.name}"`);
133
+ }
134
+
135
+ // =========================================================================
136
+ // PHASE 3: Apply removals (soft-delete with replaced_by tracking)
137
+ // =========================================================================
138
+
139
+ for (const removal of decisions.remove) {
140
+ const entity = entityList.find((e: Fact | Trait | Topic | Person) => e.id === removal.to_be_removed);
141
+
142
+ if (!entity) {
143
+ console.warn(`[Dedup] Entity ${removal.to_be_removed} already deleted`);
144
+ continue; // Graceful skip
145
+ }
146
+
147
+ // Remove via StateManager (also cleans up quote references)
148
+ const removeMethod = `human_${entity_type}_remove` as
149
+ 'human_fact_remove' | 'human_trait_remove' | 'human_topic_remove' | 'human_person_remove';
150
+
151
+ const removed = stateManager[removeMethod](removal.to_be_removed);
152
+ if (removed) {
153
+ console.log(`[Dedup] Removed ${entity_type} "${entity.name}" (merged into ${removal.replaced_by})`);
154
+ }
155
+ }
156
+
157
+ // =========================================================================
158
+ // PHASE 4: Apply additions (new entities from consolidation)
159
+ // =========================================================================
160
+
161
+ for (const addition of decisions.add) {
162
+ // Compute embedding for new entity
163
+ let embedding: number[] | undefined;
164
+ try {
165
+ const embeddingService = getEmbeddingService();
166
+ embedding = await embeddingService.embed(addition.description);
167
+ } catch (err) {
168
+ console.warn(`[Dedup] Failed to compute embedding for new entity "${addition.name}"`, err);
169
+ continue; // Skip this addition if embedding fails
170
+ }
171
+
172
+ // Generate ID for new entity
173
+ const id = crypto.randomUUID();
174
+
175
+ // Build complete entity
176
+ const newEntity = {
177
+ id,
178
+ type: entity_type,
179
+ name: addition.name,
180
+ description: addition.description,
181
+ sentiment: addition.sentiment ?? 0.0,
182
+ last_updated: new Date().toISOString(),
183
+ embedding,
184
+ // Type-specific fields with defaults
185
+ ...(entity_type === 'trait' && { strength: addition.strength ?? 0.5 }),
186
+ ...(entity_type === 'fact' && {
187
+ confidence: addition.confidence ?? 0.5,
188
+ validated: 'unknown' as import("../types/enums.js").ValidationLevel,
189
+ validated_date: ''
190
+ }),
191
+ ...((entity_type === 'topic' || entity_type === 'person') && {
192
+ exposure_current: addition.exposure_current ?? 0.0,
193
+ exposure_desired: addition.exposure_desired ?? 0.5,
194
+ last_ei_asked: null
195
+ }),
196
+ ...(entity_type === 'person' && { relationship: addition.relationship ?? 'Unknown' }),
197
+ ...(entity_type === 'topic' && { category: addition.category ?? 'Interest' }),
198
+ };
199
+
200
+ // Type-safe cast based on entity_type
201
+ if (entity_type === 'fact') {
202
+ stateManager.human_fact_upsert(newEntity as Fact);
203
+ } else if (entity_type === 'trait') {
204
+ stateManager.human_trait_upsert(newEntity as Trait);
205
+ } else if (entity_type === 'topic') {
206
+ stateManager.human_topic_upsert(newEntity as Topic);
207
+ } else if (entity_type === 'person') {
208
+ stateManager.human_person_upsert(newEntity as Person);
209
+ }
210
+ console.log(`[Dedup] Added new ${entity_type} "${addition.name}"`);
211
+ }
212
+ }
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  ContextStatus,
3
+ LLMNextStep,
3
4
  ValidationLevel,
4
5
  type LLMResponse,
5
6
  type Message,
@@ -18,15 +19,17 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
18
19
 
19
20
  const result = response.parsed as HeartbeatCheckResult | undefined;
20
21
  if (!result) {
21
- console.error("[handleHeartbeatCheck] No parsed result");
22
+ console.error(`[HeartbeatCheck ${personaDisplayName}] No parsed result`);
22
23
  return;
23
24
  }
25
+ console.log(`[HeartbeatCheck ${personaDisplayName}] Parsed result - should_respond: ${result.should_respond}, topic: ${result.topic ?? '(none)'}, message: ${result.message ? '(present)' : '(none)'}`);
24
26
 
25
27
  const now = new Date().toISOString();
26
28
  state.persona_update(personaId, { last_heartbeat: now });
29
+ state.queue_clearPersonaResponses(personaId, LLMNextStep.HandleHeartbeatCheck);
27
30
 
28
31
  if (!result.should_respond) {
29
- console.log(`[handleHeartbeatCheck] ${personaDisplayName} chose not to reach out`);
32
+ console.log(`[HeartbeatCheck ${personaDisplayName}] Chose not to reach out (should_respond=false)`);
30
33
  return;
31
34
  }
32
35
 
@@ -40,20 +43,24 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
40
43
  context_status: ContextStatus.Default,
41
44
  };
42
45
  state.messages_append(personaId, message);
43
- console.log(`[handleHeartbeatCheck] ${personaDisplayName} proactively messaged about: ${result.topic ?? "general"}`);
46
+ console.log(`[HeartbeatCheck ${personaDisplayName}] Added proactive message - topic: ${result.topic ?? 'general'}, message: "${result.message.substring(0, 100)}${result.message.length > 100 ? '...' : ''}"`);
47
+ } else {
48
+ console.log(`[HeartbeatCheck ${personaDisplayName}] should_respond=true but no message provided`);
44
49
  }
45
50
  }
46
51
 
47
52
  export function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
48
53
  const result = response.parsed as EiHeartbeatResult | undefined;
49
54
  if (!result) {
50
- console.error("[handleEiHeartbeat] No parsed result");
55
+ console.error("[EiHeartbeat] No parsed result");
51
56
  return;
52
57
  }
58
+ console.log(`[EiHeartbeat] Parsed result - should_respond: ${result.should_respond}, id: ${result.id ?? '(none)'}, my_response: ${result.my_response ? '(present)' : '(none)'}`);
53
59
  const now = new Date().toISOString();
54
60
  state.persona_update("ei", { last_heartbeat: now });
61
+ state.queue_clearPersonaResponses("ei", LLMNextStep.HandleEiHeartbeat);
55
62
  if (!result.should_respond || !result.id) {
56
- console.log("[handleEiHeartbeat] Ei chose not to reach out");
63
+ console.log("[EiHeartbeat] Chose not to reach out (should_respond=false or no id)");
57
64
  return;
58
65
  }
59
66
  const isTUI = response.request.data.isTUI as boolean;
@@ -81,7 +88,10 @@ export function handleEiHeartbeat(response: LLMResponse, state: StateManager): v
81
88
  return;
82
89
  }
83
90
 
84
- if (result.my_response) sendMessage(result.my_response);
91
+ if (result.my_response) {
92
+ console.log(`[EiHeartbeat] Sending message: "${result.my_response.substring(0, 100)}${result.my_response.length > 100 ? '...' : ''}"`);
93
+ sendMessage(result.my_response);
94
+ }
85
95
 
86
96
  switch (found.type) {
87
97
  case "person":
@@ -27,9 +27,27 @@ export function handleHumanItemMatch(response: LLMResponse, state: StateManager)
27
27
  const candidateType = response.request.data.candidateType as DataItemType;
28
28
  const personaId = response.request.data.personaId as string;
29
29
  const personaDisplayName = response.request.data.personaDisplayName as string;
30
- const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
30
+ const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
31
31
  const allMessages = state.messages_get(personaId);
32
- const { messages_context, messages_analyze } = splitMessagesByTimestamp(allMessages, analyzeFrom);
32
+
33
+ let messages_context: Message[];
34
+ let messages_analyze: Message[];
35
+
36
+ if (messageIdsToMark && messageIdsToMark.length > 0) {
37
+ const messageIdSet = new Set(messageIdsToMark);
38
+ messages_analyze = allMessages.filter(m => messageIdSet.has(m.id));
39
+ const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
40
+ messages_context = allMessages.filter(m =>
41
+ !messageIdSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
42
+ );
43
+ } else {
44
+ // Fallback to existing behavior
45
+ const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
46
+ const split = splitMessagesByTimestamp(allMessages, analyzeFrom);
47
+ messages_context = split.messages_context;
48
+ messages_analyze = split.messages_analyze;
49
+ }
50
+
33
51
  const context: ExtractionContext & { itemName: string; itemValue: string; itemCategory?: string } = {
34
52
  personaId,
35
53
  personaDisplayName,
@@ -212,6 +230,12 @@ export async function handleHumanItemUpdate(response: LLMResponse, state: StateM
212
230
  console.log(`[handleHumanItemUpdate] ${isNewItem ? "Created" : "Updated"} ${candidateType} "${result.name}"`);
213
231
  }
214
232
 
233
+ function normalizeQuotes(text: string): string {
234
+ return text
235
+ .replace(/[\u201C\u201D]/g, '"') // Curly double quotes to straight
236
+ .replace(/[\u2018\u2019]/g, "'"); // Curly single quotes to straight
237
+ }
238
+
215
239
  async function validateAndStoreQuotes(
216
240
  candidates: Array<{ text: string; reason: string }> | undefined,
217
241
  messages: Message[],
@@ -226,7 +250,9 @@ async function validateAndStoreQuotes(
226
250
  let found = false;
227
251
  for (const message of messages) {
228
252
  const msgText = getMessageText(message);
229
- const start = msgText.indexOf(candidate.text);
253
+ const normalizedMsg = normalizeQuotes(msgText);
254
+ const normalizedQuote = normalizeQuotes(candidate.text);
255
+ const start = normalizedMsg.indexOf(normalizedQuote);
230
256
  if (start !== -1) {
231
257
  const end = start + candidate.text.length;
232
258
 
@@ -17,6 +17,7 @@ import {
17
17
  import { handleHumanFactScan, handleHumanTraitScan, handleHumanTopicScan, handleHumanPersonScan } from "./human-extraction.js";
18
18
  import { handleHumanItemMatch, handleHumanItemUpdate } from "./human-matching.js";
19
19
  import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
20
+ import { handleDedupCurate } from "./dedup.js";
20
21
 
21
22
  export const handlers: Record<LLMNextStep, ResponseHandler> = {
22
23
  handlePersonaResponse,
@@ -41,4 +42,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
41
42
  handleToolContinuation,
42
43
  handleRewriteScan,
43
44
  handleRewriteRewrite,
45
+ handleDedupCurate,
44
46
  };
@@ -1,10 +1,12 @@
1
1
  import {
2
2
  ContextStatus,
3
+ LLMNextStep,
3
4
  type LLMResponse,
4
5
  type Message,
5
6
  } from "../types.js";
6
7
  import type { StateManager } from "../state-manager.js";
7
8
  import type { PersonaResponseResult } from "../../prompts/response/index.js";
9
+ import { handlers } from "./index.js";
8
10
 
9
11
  export type ResponseHandler = (response: LLMResponse, state: StateManager) => void | Promise<void>;
10
12
 
@@ -87,11 +89,35 @@ export function handlePersonaResponse(response: LLMResponse, state: StateManager
87
89
  /**
88
90
  * handleToolContinuation — second LLM call in the tool flow (may loop if LLM calls more tools).
89
91
  * The QueueProcessor already injected tool history into messages and got the
90
- * final persona response. Parse and store it exactly like handlePersonaResponse.
92
+ * final persona response. Route to the original handler based on originalNextStep in data.
91
93
  */
92
94
  export function handleToolContinuation(response: LLMResponse, state: StateManager): void {
93
- console.log(`[handleToolContinuation] Routing to handlePersonaResponse`);
94
- handlePersonaResponse(response, state);
95
+ const originalStep = response.request.data.originalNextStep as LLMNextStep | undefined;
96
+
97
+ if (!originalStep) {
98
+ console.error(`[handleToolContinuation] No originalNextStep in data, falling back to handlePersonaResponse`);
99
+ handlePersonaResponse(response, state);
100
+ return;
101
+ }
102
+
103
+ console.log(`[handleToolContinuation] Original request was ${originalStep}, routing accordingly`);
104
+
105
+ const handler = handlers[originalStep];
106
+
107
+ if (!handler) {
108
+ console.error(`[handleToolContinuation] No handler found for ${originalStep}, falling back to handlePersonaResponse`);
109
+ handlePersonaResponse(response, state);
110
+ return;
111
+ }
112
+
113
+ // Avoid infinite loop - if original was already HandleToolContinuation, go to PersonaResponse
114
+ if (originalStep === "handleToolContinuation") {
115
+ console.log(`[handleToolContinuation] Original was tool continuation, routing to handlePersonaResponse`);
116
+ handlePersonaResponse(response, state);
117
+ return;
118
+ }
119
+
120
+ handler(response, state);
95
121
  }
96
122
 
97
123
  export function handleOneShot(_response: LLMResponse, _state: StateManager): void {
@@ -188,6 +188,8 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
188
188
  const persona = sm.persona_getById(personaId);
189
189
  if (!persona) return;
190
190
  sm.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
191
+ const model = getModelForPersona(sm, personaId);
192
+ console.log(`[HeartbeatCheck ${persona.display_name}] Queueing heartbeat check (model: ${model})`);
191
193
  const human = sm.getHuman();
192
194
  const history = sm.messages_get(personaId);
193
195
  const contextWindowHours = persona.context_window_hours ?? DEFAULT_CONTEXT_WINDOW_HOURS;
@@ -228,6 +230,7 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
228
230
  };
229
231
 
230
232
  const prompt = buildHeartbeatCheckPrompt(promptData);
233
+ console.log(`[HeartbeatCheck ${persona.display_name}] Prompt data - topics: ${promptData.human.topics.length}, people: ${promptData.human.people.length}, inactive_days: ${inactiveDays}`);
231
234
 
232
235
  sm.queue_enqueue({
233
236
  type: LLMRequestType.JSON,
@@ -238,4 +241,5 @@ export async function queueHeartbeatCheck(sm: StateManager, personaId: string, i
238
241
  model: getModelForPersona(sm, personaId),
239
242
  data: { personaId, personaDisplayName: persona.display_name },
240
243
  });
244
+ console.log(`[HeartbeatCheck ${persona.display_name}] Request queued`);
241
245
  }
@@ -60,7 +60,7 @@ export function resolveModel(modelSpec?: string, accounts?: ProviderAccount[]):
60
60
  if (accounts) {
61
61
  const searchName = provider || modelSpec; // If no ":", the whole spec might be an account name
62
62
  const matchingAccount = accounts.find(
63
- (acc) => acc.name.toLowerCase() === searchName.toLowerCase() && acc.enabled
63
+ (acc) => acc.name.toLowerCase() === searchName.toLowerCase() && acc.enabled && acc.type === "llm"
64
64
  );
65
65
  if (matchingAccount) {
66
66
  // If bare account name was used, get model from account's default_model
@@ -189,6 +189,7 @@ export async function callLLMRaw(
189
189
  model,
190
190
  messages: finalMessages,
191
191
  temperature,
192
+ max_tokens: 64000, // Opus 4: 128K max output, 200K total context. Local models clamp to their config. Prevents runaway generation.
192
193
  };
193
194
 
194
195
  if (options.tools && options.tools.length > 0) {
@@ -10,6 +10,7 @@ import {
10
10
  type ExtractionOptions,
11
11
  } from "./human-extraction.js";
12
12
  import { queuePersonaTopicScan, type PersonaTopicContext } from "./persona-topics.js";
13
+ import { queueDedupPhase } from "./dedup-phase.js";
13
14
  import { buildPersonaExpirePrompt, buildPersonaExplorePrompt, buildDescriptionCheckPrompt, buildRewriteScanPrompt, type RewriteItemType } from "../../prompts/ceremony/index.js";
14
15
 
15
16
  export function isNewDay(lastCeremony: string | undefined, now: Date): boolean {
@@ -69,40 +70,19 @@ export function startCeremony(state: StateManager): void {
69
70
  },
70
71
  });
71
72
 
72
- const personas = state.persona_getAll();
73
- const activePersonas = personas.filter(p =>
74
- !p.is_paused &&
75
- !p.is_archived &&
76
- !p.is_static
77
- );
73
+ // PHASE 1: Deduplication (runs BEFORE Expose)
74
+ console.log("[ceremony] Starting Phase 1: Deduplication");
75
+ queueDedupPhase(state);
78
76
 
79
- const lastCeremony = human.settings?.ceremony?.last_ceremony
80
- ? new Date(human.settings.ceremony.last_ceremony).getTime()
81
- : 0;
82
-
83
- const personasWithActivity = activePersonas.filter(p => {
84
- const lastActivity = p.last_activity ? new Date(p.last_activity).getTime() : 0;
85
- return lastActivity > lastCeremony;
86
- });
87
-
88
- console.log(`[ceremony] Processing ${personasWithActivity.length} personas with activity (of ${activePersonas.length} active)`);
89
-
90
- const options: ExtractionOptions = { ceremony_progress: true };
91
-
92
- for (let i = 0; i < personasWithActivity.length; i++) {
93
- const persona = personasWithActivity[i];
94
- const isLast = i === personasWithActivity.length - 1;
95
-
96
- console.log(`[ceremony] Queuing exposure for ${persona.display_name} (${i + 1}/${personasWithActivity.length})${isLast ? " (last)" : ""}`);
97
- queueExposurePhase(persona.id, state, options);
77
+ // Check if dedup work was queued
78
+ if (!state.queue_hasPendingCeremonies()) {
79
+ // No dedup work found → immediately advance to Expose phase
80
+ console.log("[ceremony] No dedup work, advancing to Expose phase");
81
+ handleCeremonyProgress(state, 1);
98
82
  }
99
83
 
100
84
  const duration = Date.now() - startTime;
101
- console.log(`[ceremony] Exposure phase queued in ${duration}ms`);
102
-
103
- // Check immediately — if zero messages were queued (no unextracted messages for any persona),
104
- // this will see an empty queue and proceed directly to Decay → Expire.
105
- handleCeremonyProgress(state);
85
+ console.log(`[ceremony] Dedup phase queued in ${duration}ms`);
106
86
  }
107
87
 
108
88
  /**
@@ -193,11 +173,40 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
193
173
  * If any ceremony_progress items remain in the queue, does nothing — more work pending.
194
174
  * If the queue is clear of ceremony items, advances to Decay → Prune → Expire.
195
175
  */
196
- export function handleCeremonyProgress(state: StateManager): void {
176
+ export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
197
177
  if (state.queue_hasPendingCeremonies()) {
198
- return; // Still processing exposure scans
178
+ return; // Still processing ceremony items
179
+ }
180
+
181
+ if (lastPhase === 1) {
182
+ // Dedup phase complete → start Expose phase
183
+ console.log("[ceremony:progress] Dedup complete, starting Expose phase");
184
+
185
+ const human = state.getHuman();
186
+ const personas = state.persona_getAll();
187
+ const activePersonas = personas.filter(p =>
188
+ !p.is_paused &&
189
+ !p.is_archived &&
190
+ !p.is_static
191
+ );
192
+
193
+ const lastCeremony = human.settings?.ceremony?.last_ceremony
194
+ ? new Date(human.settings.ceremony.last_ceremony).getTime()
195
+ : 0;
196
+
197
+ const personasWithActivity = activePersonas.filter(p => {
198
+ const lastActivity = p.last_activity ? new Date(p.last_activity).getTime() : 0;
199
+ return lastActivity > lastCeremony;
200
+ });
201
+
202
+ const options: ExtractionOptions = { ceremony_progress: 2 };
203
+ for (const persona of personasWithActivity) {
204
+ queueExposurePhase(persona.id, state, options);
205
+ }
206
+ return;
199
207
  }
200
208
 
209
+ // Phase 2 (Expose) complete → advance to Decay/Prune/Expire/Explore
201
210
  console.log("[ceremony:progress] All exposure scans complete, advancing to Decay");
202
211
 
203
212
  const personas = state.persona_getAll();
@@ -215,7 +224,6 @@ export function handleCeremonyProgress(state: StateManager): void {
215
224
  if (eiIndex > -1) {
216
225
  activePersonas.splice(eiIndex, 1);
217
226
  }
218
-
219
227
  // Decay phase: apply decay + prune for ALL active personas
220
228
  for (const persona of activePersonas) {
221
229
  applyDecayPhase(persona.id, state);