ei-tui 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/package.json +2 -23
  2. package/src/core/handlers/dedup.ts +4 -15
  3. package/src/core/handlers/document-segmentation.ts +2 -3
  4. package/src/core/handlers/heartbeat.ts +5 -10
  5. package/src/core/handlers/human-matching.ts +8 -0
  6. package/src/core/handlers/index.ts +2 -0
  7. package/src/core/handlers/knowledge-synthesis.ts +50 -0
  8. package/src/core/handlers/persona-generation.ts +4 -8
  9. package/src/core/handlers/persona-response.ts +3 -4
  10. package/src/core/handlers/persona-topics.ts +2 -4
  11. package/src/core/handlers/rewrite.ts +26 -9
  12. package/src/core/handlers/rooms.ts +6 -12
  13. package/src/core/llm-client.ts +13 -3
  14. package/src/core/message-manager.ts +2 -4
  15. package/src/core/orchestrators/ceremony.ts +44 -13
  16. package/src/core/orchestrators/human-extraction.ts +10 -1
  17. package/src/core/processor.ts +155 -0
  18. package/src/core/queue-manager.ts +10 -0
  19. package/src/core/state-manager.ts +35 -0
  20. package/src/core/tools/builtin/fetch-memory.ts +6 -6
  21. package/src/core/tools/index.ts +1 -1
  22. package/src/core/tools/types.ts +1 -1
  23. package/src/core/types/data-items.ts +1 -1
  24. package/src/core/types/entities.ts +7 -1
  25. package/src/core/types/enums.ts +1 -0
  26. package/src/core/types/integrations.ts +3 -1
  27. package/src/integrations/claude-code/importer.ts +6 -0
  28. package/src/integrations/cursor/importer.ts +6 -0
  29. package/src/integrations/document/unsource.ts +5 -3
  30. package/src/integrations/opencode/importer.ts +13 -1
  31. package/src/integrations/persona-history/importer.ts +9 -0
  32. package/src/prompts/ceremony/people-rewrite.ts +2 -2
  33. package/src/prompts/ceremony/topic-rewrite.ts +2 -2
  34. package/src/prompts/index.ts +3 -0
  35. package/src/prompts/synthesis/index.ts +101 -0
  36. package/src/prompts/synthesis/types.ts +26 -0
  37. package/tui/src/commands/generate.tsx +98 -0
  38. package/tui/src/commands/unsource.tsx +17 -10
  39. package/tui/src/components/GeneratedDocsOverlay.tsx +136 -0
  40. package/tui/src/components/PromptInput.tsx +2 -0
  41. package/tui/src/context/ei.tsx +49 -2
  42. package/tui/src/util/logger.ts +22 -2
  43. package/tui/src/util/provider-detection.ts +5 -2
  44. package/tui/src/util/yaml-provider.ts +2 -8
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ei-tui",
3
- "version": "1.1.0",
3
+ "version": "1.2.0",
4
4
  "author": "Flare576",
5
5
  "repository": {
6
6
  "type": "git",
@@ -51,28 +51,7 @@
51
51
  "test:e2e:tui": "cd tui && npm run test:e2e",
52
52
  "test:e2e:ui": "playwright test --ui",
53
53
  "test:e2e:debug": "playwright test --debug",
54
- "test:evals": "vite-node tests/evals/reflection-critic.eval.ts",
55
- "test:evals:observe": "vite-node tests/evals/reflection-critic.observe.ts",
56
- "test:evals:fact-find": "vite-node tests/evals/fact-find.eval.ts",
57
- "test:evals:topic-scan": "vite-node tests/evals/topic-scan.eval.ts",
58
- "test:evals:topic-match": "vite-node tests/evals/topic-match.eval.ts",
59
- "test:evals:topic-update": "vite-node tests/evals/topic-update.eval.ts",
60
- "test:evals:topic-technical": "vite-node tests/evals/topic-technical.eval.ts",
61
- "test:evals:rewrite-scan": "vite-node tests/evals/rewrite-scan.eval.ts",
62
- "test:evals:rewrite-rewrite": "vite-node tests/evals/rewrite-rewrite.eval.ts",
63
- "test:evals:rewrite-real-data": "vite-node tests/evals/rewrite-real-data.eval.ts",
64
- "test:evals:topic-validate": "vite-node tests/evals/topic-validate.eval.ts",
65
- "test:evals:person-scan": "vite-node tests/evals/person-scan.eval.ts",
66
- "test:evals:person-scan-confidence": "vite-node tests/evals/person-scan-confidence.eval.ts",
67
-
68
- "test:evals:person-update": "vite-node tests/evals/person-update.eval.ts",
69
- "test:evals:persona-trait": "vite-node tests/evals/persona-trait-extraction.eval.ts",
70
- "test:evals:dedup": "vite-node tests/evals/dedup-tool-calls.eval.ts",
71
- "test:evals:response-read-memory": "vite-node tests/evals/response-read-memory.eval.ts",
72
- "test:evals:response-pending-update": "vite-node tests/evals/response-pending-update.eval.ts",
73
- "test:evals:heartbeat-pending-update": "vite-node tests/evals/heartbeat-pending-update.eval.ts",
74
- "test:evals:real-data": "vite-node tests/evals/real-data-example.eval.ts",
75
- "test:evals:persona-data-check": "vite-node tests/evals/persona-data-check.eval.ts",
54
+ "test:evals": "vite-node tests/evals/run.ts",
76
55
  "test:all": "npm run test && npm run test:e2e && npm run test:e2e:tui",
77
56
  "typecheck": "tsc --noEmit",
78
57
  "web": "cd web && npm run dev",
@@ -25,8 +25,7 @@ export async function handleDedupCurate(
25
25
 
26
26
  // Validate entity_type
27
27
  if (!entity_type || !['topic', 'person'].includes(entity_type)) {
28
- console.error(`[Dedup] Invalid entity_type: "${entity_type}" (from request data)`, response.request.data);
29
- return;
28
+ throw new Error(`[Dedup] Invalid entity_type: "${entity_type}" (from request data)`);
30
29
  }
31
30
 
32
31
  // Parse Opus response
@@ -37,14 +36,12 @@ export async function handleDedupCurate(
37
36
  throw new Error("Invalid response format");
38
37
  }
39
38
  } catch (err) {
40
- console.error(`[Dedup] Failed to parse Opus response:`, err);
41
- return;
39
+ throw new Error(`[Dedup] Failed to parse Opus response: ${err instanceof Error ? err.message : String(err)}`);
42
40
  }
43
41
 
44
42
  // Validate response structure
45
43
  if (!Array.isArray(decisions.update) || !Array.isArray(decisions.remove) || !Array.isArray(decisions.add)) {
46
- console.error(`[Dedup] Invalid response structure - missing update/remove/add arrays`);
47
- return;
44
+ throw new Error(`[Dedup] Invalid response structure - missing update/remove/add arrays`);
48
45
  }
49
46
 
50
47
  console.log(`[Dedup] Processing cluster: ${decisions.update.length} updates, ${decisions.remove.length} removals, ${decisions.add.length} additions`);
@@ -63,15 +60,7 @@ export async function handleDedupCurate(
63
60
 
64
61
  // Validate entityList exists
65
62
  if (!entityList || !Array.isArray(entityList)) {
66
- console.error(`[Dedup] entityList is ${entityList === undefined ? 'undefined' : 'not an array'} for entity_type="${entity_type}" (looking for state.${entity_type}s)`, {
67
- entity_type,
68
- entity_ids,
69
- stateKeys: Object.keys(state),
70
- factsExists: !!state.facts,
71
- topicsExists: !!state.topics,
72
- peopleExists: !!state.people
73
- });
74
- return;
63
+ throw new Error(`[Dedup] entityList is ${entityList === undefined ? 'undefined' : 'not an array'} for entity_type="${entity_type}" (looking for state.${entity_type}s)`);
75
64
  }
76
65
  const entities = entity_ids
77
66
  .map((id: string) => entityList.find((e: Fact | Topic | Person) => e.id === id))
@@ -30,8 +30,7 @@ export function handleDocumentSegmentation(response: LLMResponse, state: StateMa
30
30
  };
31
31
 
32
32
  if (!batchId || !filename) {
33
- console.error("[handleDocumentSegmentation] Missing batchId or filename in request data");
34
- return;
33
+ throw new Error("[handleDocumentSegmentation] Missing batchId or filename in request data");
35
34
  }
36
35
 
37
36
  let segments: string[];
@@ -103,7 +102,7 @@ export function finishDocumentBatch(batchId: string, filename: string, state: St
103
102
  ...updatedHuman.settings?.document,
104
103
  processed_documents: {
105
104
  ...(updatedHuman.settings?.document?.processed_documents ?? {}),
106
- [filename]: new Date().toISOString(),
105
+ [filename]: { created_at: new Date().toISOString(), type: "imported" },
107
106
  },
108
107
  },
109
108
  },
@@ -13,14 +13,12 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
13
13
  const personaId = response.request.data.personaId as string;
14
14
  const personaDisplayName = response.request.data.personaDisplayName as string;
15
15
  if (!personaId) {
16
- console.error("[handleHeartbeatCheck] No personaId in request data");
17
- return;
16
+ throw new Error("[handleHeartbeatCheck] No personaId in request data");
18
17
  }
19
18
 
20
19
  const result = response.parsed as HeartbeatCheckResult | undefined;
21
20
  if (!result) {
22
- console.error(`[HeartbeatCheck ${personaDisplayName}] No parsed result`);
23
- return;
21
+ throw new Error(`[HeartbeatCheck ${personaDisplayName}] No parsed result`);
24
22
  }
25
23
  console.log(`[HeartbeatCheck ${personaDisplayName}] Parsed result - should_respond: ${result.should_respond}, topic: ${result.topic ?? '(none)'}, message: ${result.message ? '(present)' : '(none)'}`);
26
24
 
@@ -52,8 +50,7 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
52
50
  export function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
53
51
  const result = response.parsed as EiHeartbeatResult | undefined;
54
52
  if (!result) {
55
- console.error("[EiHeartbeat] No parsed result");
56
- return;
53
+ throw new Error("[EiHeartbeat] No parsed result");
57
54
  }
58
55
  console.log(`[EiHeartbeat] Parsed result - should_respond: ${result.should_respond}, id: ${result.id ?? '(none)'}, my_response: ${result.my_response ? '(present)' : '(none)'}`);
59
56
  const now = new Date().toISOString();
@@ -127,8 +124,7 @@ export function handleReflectionCritic(response: LLMResponse, state: StateManage
127
124
 
128
125
  const result = response.parsed as ReflectionCriticResult | undefined;
129
126
  if (!result?.critique) {
130
- console.error(`[ReflectionCritic ${personaDisplayName}] Invalid or missing parsed result`);
131
- return;
127
+ throw new Error(`[ReflectionCritic ${personaDisplayName}] Invalid or missing parsed result`);
132
128
  }
133
129
 
134
130
  const personRecord = state.human_person_getByIdentifier("Ei Persona", personaId);
@@ -150,8 +146,7 @@ export function handleReflectionCritic(response: LLMResponse, state: StateManage
150
146
 
151
147
  const persona = state.persona_getById(personaId);
152
148
  if (!persona) {
153
- console.error(`[ReflectionCritic ${personaDisplayName}] Persona not found after critic`);
154
- return;
149
+ throw new Error(`[ReflectionCritic ${personaDisplayName}] Persona not found after critic`);
155
150
  }
156
151
 
157
152
  const mergedTopics = result.updated_identity.topics.map(updatedTopic => {
@@ -139,6 +139,8 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
139
139
  ? incomingSources
140
140
  : [...new Set([...(existingTopic?.sources ?? []), ...incomingSources])];
141
141
 
142
+ const newDescLen = resolvedDescription?.length ?? 0;
143
+ const existingFloor = existingTopic?.rewrite_length_floor;
142
144
  const topic: Topic = {
143
145
  id: itemId,
144
146
  name: resolvedName,
@@ -156,6 +158,7 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
156
158
  sources: sources.length > 0 ? sources : undefined,
157
159
  persona_groups: personaGroupsMerged,
158
160
  embedding,
161
+ rewrite_length_floor: existingFloor !== undefined && newDescLen < existingFloor ? existingFloor : undefined,
159
162
  };
160
163
  state.human_topic_upsert(topic);
161
164
 
@@ -333,6 +336,11 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
333
336
  sources: personSources.length > 0 ? personSources : undefined,
334
337
  persona_groups: personaGroupsMerged,
335
338
  embedding,
339
+ rewrite_length_floor: (() => {
340
+ const floor = existingPerson?.rewrite_length_floor;
341
+ const newLen = resolvedDescription?.length ?? 0;
342
+ return floor !== undefined && newLen < floor ? floor : undefined;
343
+ })(),
336
344
  };
337
345
  state.human_person_upsert(person);
338
346
 
@@ -16,6 +16,7 @@ import { handleDedupCurate } from "./dedup.js";
16
16
  import { handleRoomResponse, handleRoomJudge } from "./rooms.js";
17
17
  import { handlePersonaPreview } from "./persona-preview.js";
18
18
  import { handleDocumentSegmentation } from "./document-segmentation.js";
19
+ import { handleKnowledgeSynthesis } from "./knowledge-synthesis.js";
19
20
 
20
21
  export const handlers: Record<LLMNextStep, ResponseHandler> = {
21
22
  handlePersonaResponse,
@@ -43,4 +44,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
43
44
  [LLMNextStep.HandleTopicValidate]: handleDedupCurate,
44
45
  [LLMNextStep.HandleReflectionCritic]: handleReflectionCritic,
45
46
  [LLMNextStep.HandleDocumentSegmentation]: handleDocumentSegmentation,
47
+ [LLMNextStep.HandleKnowledgeSynthesis]: handleKnowledgeSynthesis,
46
48
  };
@@ -0,0 +1,50 @@
1
+ import { ContextStatus } from "../types.js";
2
+ import type { LLMResponse, Message } from "../types.js";
3
+ import type { StateManager } from "../state-manager.js";
4
+
5
+ export function handleKnowledgeSynthesis(response: LLMResponse, state: StateManager): void {
6
+ const { slug, subject } = response.request.data as {
7
+ slug: string;
8
+ subject: string;
9
+ };
10
+
11
+ if (!slug || !subject) {
12
+ throw new Error("[handleKnowledgeSynthesis] Missing slug or subject in request data");
13
+ }
14
+
15
+ const content = response.content?.trim() ?? "";
16
+ if (!content) {
17
+ throw new Error(`[handleKnowledgeSynthesis] Empty or null response content for slug "${slug}"`);
18
+ }
19
+
20
+ const now = new Date().toISOString();
21
+ const sourceTag = `generate:document:${slug}`;
22
+
23
+ const message: Message = {
24
+ id: crypto.randomUUID(),
25
+ role: "system",
26
+ content,
27
+ timestamp: now,
28
+ read: true,
29
+ context_status: ContextStatus.Always,
30
+ external: true,
31
+ source_tag: sourceTag,
32
+ };
33
+
34
+ state.messages_append("emmet", message);
35
+
36
+ const updatedHuman = state.getHuman();
37
+ state.setHuman({
38
+ ...updatedHuman,
39
+ settings: {
40
+ ...updatedHuman.settings,
41
+ document: {
42
+ ...updatedHuman.settings?.document,
43
+ processed_documents: {
44
+ ...(updatedHuman.settings?.document?.processed_documents ?? {}),
45
+ [slug]: { created_at: now, type: "generated", subject },
46
+ },
47
+ },
48
+ },
49
+ });
50
+ }
@@ -12,8 +12,7 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
12
12
  const personaId = response.request.data.personaId as string;
13
13
  const personaDisplayName = response.request.data.personaDisplayName as string;
14
14
  if (!personaId) {
15
- console.error("[handlePersonaGeneration] No personaId in request data");
16
- return;
15
+ throw new Error("[handlePersonaGeneration] No personaId in request data");
17
16
  }
18
17
 
19
18
  const result = response.parsed as PersonaGenerationResult | undefined;
@@ -115,14 +114,12 @@ export function handlePersonaTraitExtraction(response: LLMResponse, state: State
115
114
  const personaId = response.request.data.personaId as string;
116
115
  const personaDisplayName = response.request.data.personaDisplayName as string;
117
116
  if (!personaId) {
118
- console.error("[handlePersonaTraitExtraction] No personaId in request data");
119
- return;
117
+ throw new Error("[handlePersonaTraitExtraction] No personaId in request data");
120
118
  }
121
119
 
122
120
  const result = response.parsed as TraitResult[] | undefined;
123
121
  if (!result || !Array.isArray(result)) {
124
- console.error("[handlePersonaTraitExtraction] Invalid parsed result");
125
- return;
122
+ throw new Error("[handlePersonaTraitExtraction] Invalid parsed result");
126
123
  }
127
124
 
128
125
  if (result.length === 0) {
@@ -131,8 +128,7 @@ export function handlePersonaTraitExtraction(response: LLMResponse, state: State
131
128
 
132
129
  const persona = state.persona_getById(personaId);
133
130
  if (!persona) {
134
- console.error(`[handlePersonaTraitExtraction] Persona ${personaId} not found`);
135
- return;
131
+ throw new Error(`[handlePersonaTraitExtraction] Persona ${personaId} not found`);
136
132
  }
137
133
 
138
134
  const now = new Date().toISOString();
@@ -15,8 +15,7 @@ export function handlePersonaResponse(response: LLMResponse, state: StateManager
15
15
  const personaId = response.request.data.personaId as string;
16
16
  const personaDisplayName = response.request.data.personaDisplayName as string;
17
17
  if (!personaId) {
18
- console.error("[handlePersonaResponse] No personaId in request data");
19
- return;
18
+ throw new Error("[handlePersonaResponse] No personaId in request data");
20
19
  }
21
20
 
22
21
  state.messages_markPendingAsRead(personaId);
@@ -108,7 +107,7 @@ export function handleToolContinuation(response: LLMResponse, state: StateManage
108
107
  const originalStep = response.request.data.originalNextStep as LLMNextStep | undefined;
109
108
 
110
109
  if (!originalStep) {
111
- console.error(`[handleToolContinuation] No originalNextStep in data, falling back to handlePersonaResponse`);
110
+ console.warn(`[handleToolContinuation] No originalNextStep in data, falling back to handlePersonaResponse`);
112
111
  handlePersonaResponse(response, state);
113
112
  return;
114
113
  }
@@ -118,7 +117,7 @@ export function handleToolContinuation(response: LLMResponse, state: StateManage
118
117
  const handler = handlers[originalStep];
119
118
 
120
119
  if (!handler) {
121
- console.error(`[handleToolContinuation] No handler found for ${originalStep}, falling back to handlePersonaResponse`);
120
+ console.warn(`[handleToolContinuation] No handler found for ${originalStep}, falling back to handlePersonaResponse`);
122
121
  handlePersonaResponse(response, state);
123
122
  return;
124
123
  }
@@ -12,8 +12,7 @@ export function handlePersonaTopicRating(response: LLMResponse, state: StateMana
12
12
  const personaId = response.request.data.personaId as string;
13
13
  const personaDisplayName = response.request.data.personaDisplayName as string;
14
14
  if (!personaId || !personaDisplayName) {
15
- console.error("[handlePersonaTopicRating] Missing personaId or personaDisplayName in request data");
16
- return;
15
+ throw new Error("[handlePersonaTopicRating] Missing personaId or personaDisplayName in request data");
17
16
  }
18
17
 
19
18
  const result = response.parsed as PersonaTopicRatingResult | undefined;
@@ -35,8 +34,7 @@ export function handlePersonaTopicRating(response: LLMResponse, state: StateMana
35
34
 
36
35
  const persona = state.persona_getById(personaId);
37
36
  if (!persona) {
38
- console.error(`[handlePersonaTopicRating] Persona not found: ${personaDisplayName}`);
39
- return;
37
+ throw new Error(`[handlePersonaTopicRating] Persona not found: ${personaDisplayName}`);
40
38
  }
41
39
 
42
40
  const now = new Date().toISOString();
@@ -20,6 +20,8 @@ import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.
20
20
 
21
21
  import { searchHumanData } from "../human-data-manager.js";
22
22
 
23
+ const MIN_REWRITE_FLOOR = 750;
24
+
23
25
  /**
24
26
  * handleRewriteScan — Phase 1 of Rewrite.
25
27
  * LLM returns an array of subject strings found in the bloated item.
@@ -31,20 +33,25 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
31
33
  const rewriteModel = response.request.data.rewriteModel as string;
32
34
 
33
35
  if (!itemId || !itemType) {
34
- console.error("[handleRewriteScan] Missing itemId or itemType in request data");
35
- return;
36
+ throw new Error("[handleRewriteScan] Missing itemId or itemType in request data");
36
37
  }
37
38
 
38
39
  const subjects = response.parsed as RewriteScanResult | undefined;
39
40
  if (!subjects || !Array.isArray(subjects) || subjects.length === 0) {
40
- console.log(`[handleRewriteScan] No extra subjects found for ${itemType} "${itemId}" — marking rewrite_checked`);
41
+ console.log(`[handleRewriteScan] No extra subjects found for ${itemType} "${itemId}" — setting rewrite_length_floor`);
41
42
  const human = state.getHuman();
42
43
  if (itemType === "topic") {
43
44
  const topic = human.topics.find(t => t.id === itemId);
44
- if (topic) state.human_topic_upsert({ ...topic, rewrite_checked: true });
45
+ if (topic) state.human_topic_upsert({
46
+ ...topic,
47
+ rewrite_length_floor: Math.max(MIN_REWRITE_FLOOR, Math.ceil((topic.description?.length ?? 0) * 1.1)),
48
+ });
45
49
  } else if (itemType === "person") {
46
50
  const person = human.people.find(p => p.id === itemId);
47
- if (person) state.human_person_upsert({ ...person, rewrite_checked: true });
51
+ if (person) state.human_person_upsert({
52
+ ...person,
53
+ rewrite_length_floor: Math.max(MIN_REWRITE_FLOOR, Math.ceil((person.description?.length ?? 0) * 1.1)),
54
+ });
48
55
  }
49
56
  return;
50
57
  }
@@ -111,8 +118,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
111
118
  const itemType = response.request.data.itemType as RewriteItemType;
112
119
 
113
120
  if (!itemId || !itemType) {
114
- console.error("[handleRewriteRewrite] Missing itemId or itemType in request data");
115
- return;
121
+ throw new Error("[handleRewriteRewrite] Missing itemId or itemType in request data");
116
122
  }
117
123
 
118
124
  const result = response.parsed as RewriteResult | undefined;
@@ -171,6 +177,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
171
177
  console.warn(`[handleRewriteRewrite] Failed to compute embedding for existing ${resolvedType} "${item.name}":`, err);
172
178
  }
173
179
 
180
+ const existingFloor = Math.max(MIN_REWRITE_FLOOR, Math.ceil(item.description.length * 1.1));
174
181
  switch (resolvedType) {
175
182
  case "topic": {
176
183
  const existing = human.topics.find(t => t.id === item.id)!;
@@ -181,6 +188,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
181
188
  sentiment: item.sentiment ?? existing.sentiment,
182
189
  last_updated: now,
183
190
  embedding,
191
+ rewrite_length_floor: existingFloor,
184
192
  });
185
193
  break;
186
194
  }
@@ -193,6 +201,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
193
201
  sentiment: item.sentiment ?? existing.sentiment,
194
202
  last_updated: now,
195
203
  embedding,
204
+ rewrite_length_floor: existingFloor,
196
205
  });
197
206
  break;
198
207
  }
@@ -216,6 +225,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
216
225
  console.warn(`[handleRewriteRewrite] Failed to compute embedding for new ${item.type} "${item.name}":`, err);
217
226
  }
218
227
 
228
+ const newFloor = Math.max(MIN_REWRITE_FLOOR, Math.ceil(item.description.length * 1.1));
219
229
  const baseFields = {
220
230
  id: crypto.randomUUID(),
221
231
  name: item.name,
@@ -227,6 +237,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
227
237
  persona_groups: unionGroups,
228
238
  interested_personas: unionPersonas,
229
239
  embedding,
240
+ rewrite_length_floor: newFloor,
230
241
  };
231
242
 
232
243
  switch (item.type) {
@@ -267,10 +278,16 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
267
278
  const updatedHuman = state.getHuman();
268
279
  if (itemType === "topic") {
269
280
  const original = updatedHuman.topics.find(t => t.id === itemId);
270
- if (original) state.human_topic_upsert({ ...original, rewrite_checked: true });
281
+ if (original) state.human_topic_upsert({
282
+ ...original,
283
+ rewrite_length_floor: Math.max(MIN_REWRITE_FLOOR, Math.ceil((original.description?.length ?? 0) * 1.1)),
284
+ });
271
285
  } else if (itemType === "person") {
272
286
  const original = updatedHuman.people.find(p => p.id === itemId);
273
- if (original) state.human_person_upsert({ ...original, rewrite_checked: true });
287
+ if (original) state.human_person_upsert({
288
+ ...original,
289
+ rewrite_length_floor: Math.max(MIN_REWRITE_FLOOR, Math.ceil((original.description?.length ?? 0) * 1.1)),
290
+ });
274
291
  }
275
292
 
276
293
  console.log(`[handleRewriteRewrite] Complete for ${itemType} "${itemId}": ${existingCount} existing updated, ${newCount} new created`);
@@ -17,8 +17,7 @@ export function handleRoomResponse(response: LLMResponse, state: StateManager):
17
17
  const parentMessageId = response.request.data.parentMessageId as string | null ?? null;
18
18
 
19
19
  if (!roomId || !personaId) {
20
- console.error("[handleRoomResponse] Missing roomId or personaId in request data");
21
- return;
20
+ throw new Error("[handleRoomResponse] Missing roomId or personaId in request data");
22
21
  }
23
22
 
24
23
  const now = new Date().toISOString();
@@ -111,19 +110,16 @@ export async function handleRoomJudge(response: LLMResponse, state: StateManager
111
110
  const judgeDisplayName = response.request.data.judgePersonaDisplayName as string;
112
111
 
113
112
  if (!roomId) {
114
- console.error("[handleRoomJudge] Missing roomId in request data");
115
- return;
113
+ throw new Error("[handleRoomJudge] Missing roomId in request data");
116
114
  }
117
115
 
118
116
  if (!response.parsed) {
119
- console.error(`[handleRoomJudge] No parsed result from judge ${judgeDisplayName}`);
120
- return;
117
+ throw new Error(`[handleRoomJudge] No parsed result from judge ${judgeDisplayName}`);
121
118
  }
122
119
 
123
120
  const result = response.parsed as RoomJudgeResult;
124
121
  if (!result.winner_message_id) {
125
- console.error(`[handleRoomJudge] Judge ${judgeDisplayName} returned no winner_message_id`);
126
- return;
122
+ throw new Error(`[handleRoomJudge] Judge ${judgeDisplayName} returned no winner_message_id`);
127
123
  }
128
124
 
129
125
  const judgePersonaId = response.request.data.judgePersonaId as string;
@@ -131,16 +127,14 @@ export async function handleRoomJudge(response: LLMResponse, state: StateManager
131
127
  const allMessages = state.getRoomMessages(roomId);
132
128
  const winner = allMessages.find(m => m.id === result.winner_message_id);
133
129
  if (!winner) {
134
- console.error(`[handleRoomJudge] Winner message ${result.winner_message_id} not found in room ${roomId}`);
135
- return;
130
+ throw new Error(`[handleRoomJudge] Winner message ${result.winner_message_id} not found in room ${roomId}`);
136
131
  }
137
132
 
138
133
  const verdictParentId = winner.parent_id;
139
134
 
140
135
  const ok = state.setRoomActiveNode(roomId, result.winner_message_id);
141
136
  if (!ok) {
142
- console.error(`[handleRoomJudge] Could not set active node ${result.winner_message_id} in room ${roomId}`);
143
- return;
137
+ throw new Error(`[handleRoomJudge] Could not set active node ${result.winner_message_id} in room ${roomId}`);
144
138
  }
145
139
 
146
140
  const losers = allMessages
@@ -76,7 +76,17 @@ export interface LLMRawResponse {
76
76
 
77
77
  let llmCallCount = 0;
78
78
 
79
-
79
+ function resolveApiKey(raw: string | undefined): string {
80
+ if (!raw || !raw.startsWith("$")) return raw ?? "";
81
+ const varName = raw.slice(1);
82
+ const resolved =
83
+ (typeof Bun !== "undefined" && (Bun as { env: Record<string, string> }).env?.[varName]) ||
84
+ (typeof process !== "undefined" && process.env?.[varName]);
85
+ if (!resolved) {
86
+ throw new Error(`Provider API key references env var $${varName}, but it is not set.`);
87
+ }
88
+ return resolved;
89
+ }
80
90
 
81
91
  function isGuid(str: string): boolean {
82
92
  return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(str);
@@ -90,7 +100,7 @@ function buildResolvedModel(account: ProviderAccount, model: ModelConfig): Resol
90
100
  config: {
91
101
  name: account.name,
92
102
  baseURL: account.url,
93
- apiKey: account.api_key || "",
103
+ apiKey: resolveApiKey(account.api_key),
94
104
  },
95
105
  extraHeaders: account.extra_headers,
96
106
  };
@@ -171,7 +181,7 @@ export function resolveModel(modelSpec?: string, accounts?: ProviderAccount[]):
171
181
  config: {
172
182
  name: matchingAccount.name,
173
183
  baseURL: matchingAccount.url,
174
- apiKey: matchingAccount.api_key || "",
184
+ apiKey: resolveApiKey(matchingAccount.api_key),
175
185
  },
176
186
  extraHeaders: matchingAccount.extra_headers,
177
187
  };
@@ -255,8 +255,6 @@ export function checkAndQueueHumanExtraction(
255
255
  const unextractedPeople = sm.messages_getUnextracted(personaId, "p", undefined, "exclude");
256
256
  const peopleThreshold = Math.min(EXTRACTION_TAPER_CAP, human.people.length);
257
257
  if (unextractedPeople.length > 0 && unextractedPeople.length >= peopleThreshold) {
258
- const personaForScan = sm.persona_getById(personaId);
259
- const personScanOptions = personaForScan?.pending_update ? { reflection_progress: 1 } : undefined;
260
258
  const context: ExtractionContext = {
261
259
  personaId,
262
260
  channelDisplayName: personaDisplayName,
@@ -264,9 +262,9 @@ export function checkAndQueueHumanExtraction(
264
262
  messages_analyze: unextractedPeople,
265
263
  extraction_flag: "p",
266
264
  };
267
- queuePersonScan(context, sm, personScanOptions);
265
+ queuePersonScan(context, sm);
268
266
  console.log(
269
- `[Processor] Human Seed extraction: people (threshold: ${peopleThreshold}, unextracted: ${unextractedPeople.length}${personScanOptions ? ", reflection_progress=1" : ""})`
267
+ `[Processor] Human Seed extraction: people (threshold: ${peopleThreshold}, unextracted: ${unextractedPeople.length})`
270
268
  );
271
269
  }
272
270
  }
@@ -137,10 +137,7 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
137
137
  messages_analyze: unextractedPeople,
138
138
  extraction_flag: "p",
139
139
  };
140
- const personScanOptions = persona.pending_update
141
- ? { ...options, reflection_progress: 1 }
142
- : options;
143
- queuePersonScan(context, state, personScanOptions);
140
+ queuePersonScan(context, state, options);
144
141
  }
145
142
 
146
143
  const totalUnextracted = unextractedFacts.length + unextractedTopics.length + unextractedPeople.length;
@@ -287,7 +284,7 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
287
284
  // Person Rewrite phase (phase 4): scan bloated Person records, extract Topics from them.
288
285
  // Gated via ceremony_progress so Topic Rewrite can run after — Topics created here
289
286
  // need to be visible before Topic Rewrite snapshots the threshold.
290
- queuePersonRewritePhase(state);
287
+ queuePersonRewritePhase(state, { ceremonyProgress: 4 });
291
288
 
292
289
  // Zero-work guard: if no person rewrites queued, advance to topic rewrite immediately
293
290
  if (!state.queue_hasPendingCeremonies()) {
@@ -482,7 +479,7 @@ export function queueReflectionDrain(personaId: string, state: StateManager): vo
482
479
  messages_analyze: unextractedPeople,
483
480
  extraction_flag: "p",
484
481
  };
485
- queuePersonScan(context, state, { reflection_progress: 1 });
482
+ queuePersonScan(context, state);
486
483
  console.log(`[reflection:drain] Queued Person scan for ${persona.display_name} (${unextractedPeople.length} messages) — clears on completion`);
487
484
  }
488
485
 
@@ -490,7 +487,7 @@ function getRewriteModel(state: StateManager): string | undefined {
490
487
  return state.getHuman().settings?.rewrite_model;
491
488
  }
492
489
 
493
- export function queuePersonRewritePhase(state: StateManager): void {
490
+ export function queuePersonRewritePhase(state: StateManager, options?: { ceremonyProgress?: number }): void {
494
491
  const rewriteModel = getRewriteModel(state);
495
492
  if (!rewriteModel) {
496
493
  console.log("[ceremony:rewrite] rewrite_model not set — skipping person rewrite phase");
@@ -498,13 +495,30 @@ export function queuePersonRewritePhase(state: StateManager): void {
498
495
  }
499
496
 
500
497
  const human = state.getHuman();
501
- const personsToScan = human.people.filter(person => {
498
+ const allCandidates = human.people.filter(person => {
502
499
  const isPersonaLinked = (person.identifiers ?? []).some(
503
500
  i => i.type.toLowerCase() === 'ei persona'
504
501
  );
505
502
  return !isPersonaLinked
506
- && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
507
- && !person.rewrite_checked;
503
+ && (person.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD;
504
+ });
505
+
506
+ const alreadyChecked = allCandidates.filter(p => {
507
+ const descLen = p.description?.length ?? 0;
508
+ return p.rewrite_length_floor !== undefined && descLen < p.rewrite_length_floor;
509
+ });
510
+ if (alreadyChecked.length > 0) {
511
+ for (const person of alreadyChecked) {
512
+ console.log(
513
+ `[ceremony:rewrite] Person "${person.name}" is ${person.description?.length ?? 0} chars ` +
514
+ `(floor: ${person.rewrite_length_floor}) — already reviewed, skipping`
515
+ );
516
+ }
517
+ }
518
+
519
+ const personsToScan = allCandidates.filter(p => {
520
+ if (p.rewrite_length_floor === undefined) return true;
521
+ return (p.description?.length ?? 0) >= p.rewrite_length_floor;
508
522
  });
509
523
 
510
524
  if (personsToScan.length === 0) {
@@ -527,7 +541,7 @@ export function queuePersonRewritePhase(state: StateManager): void {
527
541
  itemId: person.id,
528
542
  itemType: "person" as RewriteItemType,
529
543
  rewriteModel,
530
- ceremony_progress: 4,
544
+ ...(options?.ceremonyProgress !== undefined && { ceremony_progress: options.ceremonyProgress }),
531
545
  },
532
546
  });
533
547
  }
@@ -543,11 +557,28 @@ export function queueTopicRewritePhase(state: StateManager): void {
543
557
  }
544
558
 
545
559
  const human = state.getHuman();
546
- const topicsToScan = human.topics.filter(topic =>
560
+ const allCandidateTopics = human.topics.filter(topic =>
547
561
  (topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD
548
- && !topic.rewrite_checked
549
562
  );
550
563
 
564
+ const alreadyCheckedTopics = allCandidateTopics.filter(t => {
565
+ const descLen = t.description?.length ?? 0;
566
+ return t.rewrite_length_floor !== undefined && descLen < t.rewrite_length_floor;
567
+ });
568
+ if (alreadyCheckedTopics.length > 0) {
569
+ for (const topic of alreadyCheckedTopics) {
570
+ console.log(
571
+ `[ceremony:rewrite] Topic "${topic.name}" is ${topic.description?.length ?? 0} chars ` +
572
+ `(floor: ${topic.rewrite_length_floor}) — already reviewed, skipping`
573
+ );
574
+ }
575
+ }
576
+
577
+ const topicsToScan = allCandidateTopics.filter(t => {
578
+ if (t.rewrite_length_floor === undefined) return true;
579
+ return (t.description?.length ?? 0) >= t.rewrite_length_floor;
580
+ });
581
+
551
582
  if (topicsToScan.length === 0) {
552
583
  console.log("[ceremony:rewrite] No topics above threshold — skipping topic rewrite phase");
553
584
  return;