ei-tui 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +170 -0
  3. package/package.json +63 -0
  4. package/src/README.md +96 -0
  5. package/src/cli/README.md +47 -0
  6. package/src/cli/commands/facts.ts +25 -0
  7. package/src/cli/commands/people.ts +25 -0
  8. package/src/cli/commands/quotes.ts +19 -0
  9. package/src/cli/commands/topics.ts +25 -0
  10. package/src/cli/commands/traits.ts +25 -0
  11. package/src/cli/retrieval.ts +269 -0
  12. package/src/cli.ts +176 -0
  13. package/src/core/AGENTS.md +104 -0
  14. package/src/core/embedding-service.ts +241 -0
  15. package/src/core/handlers/index.ts +1057 -0
  16. package/src/core/index.ts +4 -0
  17. package/src/core/llm-client.ts +265 -0
  18. package/src/core/model-context-windows.ts +49 -0
  19. package/src/core/orchestrators/ceremony.ts +500 -0
  20. package/src/core/orchestrators/extraction-chunker.ts +138 -0
  21. package/src/core/orchestrators/human-extraction.ts +457 -0
  22. package/src/core/orchestrators/index.ts +28 -0
  23. package/src/core/orchestrators/persona-generation.ts +76 -0
  24. package/src/core/orchestrators/persona-topics.ts +117 -0
  25. package/src/core/personas/index.ts +5 -0
  26. package/src/core/personas/opencode-agent.ts +81 -0
  27. package/src/core/processor.ts +1413 -0
  28. package/src/core/queue-processor.ts +197 -0
  29. package/src/core/state/checkpoints.ts +68 -0
  30. package/src/core/state/human.ts +176 -0
  31. package/src/core/state/index.ts +5 -0
  32. package/src/core/state/personas.ts +217 -0
  33. package/src/core/state/queue.ts +144 -0
  34. package/src/core/state-manager.ts +347 -0
  35. package/src/core/types.ts +421 -0
  36. package/src/core/utils/decay.ts +33 -0
  37. package/src/index.ts +1 -0
  38. package/src/integrations/opencode/importer.ts +896 -0
  39. package/src/integrations/opencode/index.ts +16 -0
  40. package/src/integrations/opencode/json-reader.ts +304 -0
  41. package/src/integrations/opencode/reader-factory.ts +35 -0
  42. package/src/integrations/opencode/sqlite-reader.ts +189 -0
  43. package/src/integrations/opencode/types.ts +244 -0
  44. package/src/prompts/AGENTS.md +62 -0
  45. package/src/prompts/ceremony/description-check.ts +47 -0
  46. package/src/prompts/ceremony/expire.ts +30 -0
  47. package/src/prompts/ceremony/explore.ts +60 -0
  48. package/src/prompts/ceremony/index.ts +11 -0
  49. package/src/prompts/ceremony/types.ts +42 -0
  50. package/src/prompts/generation/descriptions.ts +91 -0
  51. package/src/prompts/generation/index.ts +15 -0
  52. package/src/prompts/generation/persona.ts +155 -0
  53. package/src/prompts/generation/seeds.ts +31 -0
  54. package/src/prompts/generation/types.ts +47 -0
  55. package/src/prompts/heartbeat/check.ts +179 -0
  56. package/src/prompts/heartbeat/ei.ts +208 -0
  57. package/src/prompts/heartbeat/index.ts +15 -0
  58. package/src/prompts/heartbeat/types.ts +70 -0
  59. package/src/prompts/human/fact-scan.ts +152 -0
  60. package/src/prompts/human/index.ts +32 -0
  61. package/src/prompts/human/item-match.ts +74 -0
  62. package/src/prompts/human/item-update.ts +322 -0
  63. package/src/prompts/human/person-scan.ts +115 -0
  64. package/src/prompts/human/topic-scan.ts +135 -0
  65. package/src/prompts/human/trait-scan.ts +115 -0
  66. package/src/prompts/human/types.ts +127 -0
  67. package/src/prompts/index.ts +90 -0
  68. package/src/prompts/message-utils.ts +39 -0
  69. package/src/prompts/persona/index.ts +16 -0
  70. package/src/prompts/persona/topics-match.ts +69 -0
  71. package/src/prompts/persona/topics-scan.ts +98 -0
  72. package/src/prompts/persona/topics-update.ts +157 -0
  73. package/src/prompts/persona/traits.ts +117 -0
  74. package/src/prompts/persona/types.ts +74 -0
  75. package/src/prompts/response/index.ts +147 -0
  76. package/src/prompts/response/sections.ts +355 -0
  77. package/src/prompts/response/types.ts +38 -0
  78. package/src/prompts/validation/ei.ts +93 -0
  79. package/src/prompts/validation/index.ts +6 -0
  80. package/src/prompts/validation/types.ts +22 -0
  81. package/src/storage/crypto.ts +96 -0
  82. package/src/storage/index.ts +5 -0
  83. package/src/storage/interface.ts +9 -0
  84. package/src/storage/local.ts +79 -0
  85. package/src/storage/merge.ts +69 -0
  86. package/src/storage/remote.ts +145 -0
  87. package/src/templates/welcome.ts +91 -0
  88. package/tui/README.md +62 -0
  89. package/tui/bunfig.toml +4 -0
  90. package/tui/src/app.tsx +55 -0
  91. package/tui/src/commands/archive.tsx +93 -0
  92. package/tui/src/commands/context.tsx +124 -0
  93. package/tui/src/commands/delete.tsx +71 -0
  94. package/tui/src/commands/details.tsx +41 -0
  95. package/tui/src/commands/editor.tsx +46 -0
  96. package/tui/src/commands/help.tsx +12 -0
  97. package/tui/src/commands/me.tsx +145 -0
  98. package/tui/src/commands/model.ts +47 -0
  99. package/tui/src/commands/new.ts +31 -0
  100. package/tui/src/commands/pause.ts +46 -0
  101. package/tui/src/commands/persona.tsx +58 -0
  102. package/tui/src/commands/provider.tsx +124 -0
  103. package/tui/src/commands/quit.ts +22 -0
  104. package/tui/src/commands/quotes.tsx +172 -0
  105. package/tui/src/commands/registry.test.ts +137 -0
  106. package/tui/src/commands/registry.ts +130 -0
  107. package/tui/src/commands/resume.ts +39 -0
  108. package/tui/src/commands/setsync.tsx +43 -0
  109. package/tui/src/commands/settings.tsx +83 -0
  110. package/tui/src/components/ConfirmOverlay.tsx +51 -0
  111. package/tui/src/components/ConflictOverlay.tsx +78 -0
  112. package/tui/src/components/HelpOverlay.tsx +69 -0
  113. package/tui/src/components/Layout.tsx +24 -0
  114. package/tui/src/components/MessageList.tsx +174 -0
  115. package/tui/src/components/PersonaListOverlay.tsx +186 -0
  116. package/tui/src/components/PromptInput.tsx +145 -0
  117. package/tui/src/components/ProviderListOverlay.tsx +208 -0
  118. package/tui/src/components/QuotesOverlay.tsx +157 -0
  119. package/tui/src/components/Sidebar.tsx +95 -0
  120. package/tui/src/components/StatusBar.tsx +77 -0
  121. package/tui/src/components/WelcomeOverlay.tsx +73 -0
  122. package/tui/src/context/ei.tsx +623 -0
  123. package/tui/src/context/keyboard.tsx +164 -0
  124. package/tui/src/context/overlay.tsx +53 -0
  125. package/tui/src/index.tsx +8 -0
  126. package/tui/src/storage/file.ts +185 -0
  127. package/tui/src/util/duration.ts +32 -0
  128. package/tui/src/util/editor.ts +188 -0
  129. package/tui/src/util/logger.ts +109 -0
  130. package/tui/src/util/persona-editor.tsx +181 -0
  131. package/tui/src/util/provider-editor.tsx +168 -0
  132. package/tui/src/util/syntax.ts +35 -0
  133. package/tui/src/util/yaml-serializers.ts +755 -0
@@ -0,0 +1,1413 @@
1
+ import {
2
+ LLMRequestType,
3
+ LLMPriority,
4
+ LLMNextStep,
5
+ RESERVED_PERSONA_NAMES,
6
+ isReservedPersonaName,
7
+ type LLMRequest,
8
+ type Ei_Interface,
9
+ type PersonaSummary,
10
+ type PersonaEntity,
11
+ type PersonaCreationInput,
12
+ type Message,
13
+ type MessageQueryOptions,
14
+ type HumanEntity,
15
+ type Fact,
16
+ type Trait,
17
+ type Topic,
18
+ type Person,
19
+ type Quote,
20
+ type QueueStatus,
21
+ type ContextStatus,
22
+ type DataItemBase,
23
+ type LLMResponse,
24
+ type StorageState,
25
+ type StateConflictResolution,
26
+ type StateConflictData,
27
+ } from "./types.js";
28
+ import type { Storage } from "../storage/interface.js";
29
+ import { remoteSync } from "../storage/remote.js";
30
+ import { yoloMerge } from "../storage/merge.js";
31
+ import { StateManager } from "./state-manager.js";
32
+ import { QueueProcessor } from "./queue-processor.js";
33
+ import { handlers } from "./handlers/index.js";
34
+ import {
35
+ buildResponsePrompt,
36
+ buildPersonaTraitExtractionPrompt,
37
+ buildHeartbeatCheckPrompt,
38
+ type ResponsePromptData,
39
+ type PersonaTraitExtractionPromptData,
40
+ type HeartbeatCheckPromptData,
41
+ } from "../prompts/index.js";
42
+ import {
43
+ orchestratePersonaGeneration,
44
+ queueFactScan,
45
+ queueTopicScan,
46
+ queuePersonScan,
47
+ shouldStartCeremony,
48
+ startCeremony,
49
+ handleCeremonyProgress,
50
+ type ExtractionContext,
51
+ } from "./orchestrators/index.js";
52
+ import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
53
+ import { getEmbeddingService, findTopK, needsEmbeddingUpdate, needsQuoteEmbeddingUpdate, computeDataItemEmbedding, computeQuoteEmbedding } from "./embedding-service.js";
54
+ import { ContextStatus as ContextStatusEnum } from "./types.js";
55
+
56
+ // =============================================================================
57
+ // EMBEDDING STRIPPING - Remove embeddings from data items before returning to FE
58
+ // Embeddings are internal implementation details for similarity search.
59
+ // =============================================================================
60
+
61
+ function stripDataItemEmbedding<T extends DataItemBase>(item: T): T {
62
+ const { embedding, ...rest } = item;
63
+ return rest as T;
64
+ }
65
+
66
+ function stripQuoteEmbedding(quote: Quote): Quote {
67
+ const { embedding, ...rest } = quote;
68
+ return rest;
69
+ }
70
+
71
+ function stripHumanEmbeddings(human: HumanEntity): HumanEntity {
72
+ return {
73
+ ...human,
74
+ facts: (human.facts ?? []).map(stripDataItemEmbedding),
75
+ traits: (human.traits ?? []).map(stripDataItemEmbedding),
76
+ topics: (human.topics ?? []).map(stripDataItemEmbedding),
77
+ people: (human.people ?? []).map(stripDataItemEmbedding),
78
+ quotes: (human.quotes ?? []).map(stripQuoteEmbedding),
79
+ };
80
+ }
81
+
82
+ const DEFAULT_LOOP_INTERVAL_MS = 100;
83
+ const DEFAULT_CONTEXT_WINDOW_HOURS = 8;
84
+ const DEFAULT_OPENCODE_POLLING_MS = 1800000;
85
+
86
+ let processorInstanceCount = 0;
87
+
88
+ export function filterMessagesForContext(
89
+ messages: Message[],
90
+ contextBoundary: string | undefined,
91
+ contextWindowHours: number
92
+ ): Message[] {
93
+ if (messages.length === 0) return [];
94
+
95
+ const now = Date.now();
96
+ const windowStartMs = now - contextWindowHours * 60 * 60 * 1000;
97
+ const boundaryMs = contextBoundary ? new Date(contextBoundary).getTime() : 0;
98
+
99
+ return messages.filter((msg) => {
100
+ if (msg.context_status === ContextStatusEnum.Always) return true;
101
+ if (msg.context_status === ContextStatusEnum.Never) return false;
102
+
103
+ const msgMs = new Date(msg.timestamp).getTime();
104
+
105
+ if (contextBoundary) {
106
+ return msgMs >= boundaryMs;
107
+ }
108
+
109
+ return msgMs >= windowStartMs;
110
+ });
111
+ }
112
+
113
+ export class Processor {
114
+ private stateManager = new StateManager();
115
+ private queueProcessor = new QueueProcessor();
116
+ private interface: Ei_Interface;
117
+ private running = false;
118
+ private stopped = false;
119
+ private instanceId: number;
120
+ private currentRequest: LLMRequest | null = null;
121
+ private isTUI = false;
122
+ private lastOpenCodeSync = 0;
123
+ private openCodeImportInProgress = false;
124
+ private pendingConflict: StateConflictData | null = null;
125
+
126
+ constructor(ei: Ei_Interface) {
127
+ this.interface = ei;
128
+ this.instanceId = ++processorInstanceCount;
129
+ console.log(`[Processor ${this.instanceId}] CREATED`);
130
+ this.detectEnvironment();
131
+ }
132
+
133
+ private detectEnvironment(): void {
134
+ const hasProcess = typeof process !== "undefined" && typeof process.versions !== "undefined";
135
+ const hasBun = hasProcess && typeof process.versions.bun !== "undefined";
136
+ const hasNode = hasProcess && typeof process.versions.node !== "undefined";
137
+ const hasDocument = typeof document !== "undefined";
138
+
139
+ this.isTUI = (hasBun || hasNode) && !hasDocument;
140
+ }
141
+
142
+ async start(storage: Storage): Promise<void> {
143
+ console.log(`[Processor ${this.instanceId}] start() called`);
144
+ await this.stateManager.initialize(storage);
145
+ if (this.stopped) {
146
+ console.log(`[Processor ${this.instanceId}] stopped during init, not starting loop`);
147
+ return;
148
+ }
149
+
150
+ // === SYNC DECISION TREE ===
151
+ const primary = this.stateManager.hasExistingData();
152
+ const backup = await this.stateManager.loadBackup();
153
+ const syncCreds = primary
154
+ ? this.stateManager.getHuman().settings?.sync
155
+ : backup?.human?.settings?.sync;
156
+ // Sync creds can come from state/backup OR from pre-configuration
157
+ // (TUI pre-configures from env vars, Web from onboarding form)
158
+ const hasSyncCreds = !!(syncCreds?.username && syncCreds?.passphrase);
159
+ if (hasSyncCreds || remoteSync.isConfigured()) {
160
+ // State/backup creds always win over env var pre-config
161
+ // (env vars bootstrap you; once creds are in state, state is source of truth)
162
+ if (hasSyncCreds) {
163
+ await remoteSync.configure(syncCreds);
164
+ }
165
+
166
+ try {
167
+ const remoteInfo = await remoteSync.checkRemote(); // also captures etag
168
+ if (!primary && remoteInfo.exists) {
169
+ // CASE A: No primary state (clean exit or fresh install with env vars)
170
+ // → Silent pull, no questions asked
171
+ console.log(`[Processor ${this.instanceId}] No primary state, remote exists — silent pull`);
172
+ const result = await remoteSync.fetch(); // captures etag
173
+ if (result.success && result.state) {
174
+ this.stateManager.restoreFromState(result.state);
175
+ }
176
+ // If fetch fails, fall through to bootstrapFirstRun below
177
+ } else if (primary && remoteInfo.exists) {
178
+ // CASE B: Both primary AND remote exist
179
+ // This means: crash recovery, stale etag rejection, or multi-device conflict
180
+ // → ALWAYS ask user: [L]ocal / [S]erver / [Y]olo
181
+ console.log(`[Processor ${this.instanceId}] Both primary and remote exist — conflict`);
182
+ const localTimestamp = new Date(this.stateManager.getHuman().last_updated);
183
+ const remoteTimestamp = remoteInfo.lastModified ?? new Date();
184
+ this.pendingConflict = { localTimestamp, remoteTimestamp, hasLocalState: true };
185
+ this.interface.onStateConflict?.(this.pendingConflict);
186
+ // Loop does NOT start — waits for resolveStateConflict()
187
+ return;
188
+ }
189
+ // primary exists, no remote → normal boot (first time syncing from this device)
190
+ // no primary, no remote → fall through to bootstrapFirstRun
191
+ } catch (err) {
192
+ console.warn(`[Processor ${this.instanceId}] Sync check failed, continuing without sync:`, err);
193
+ }
194
+ }
195
+ // If still no data after sync attempts, bootstrap
196
+ if (!this.stateManager.hasExistingData() || this.stateManager.persona_getAll().length === 0) {
197
+ await this.bootstrapFirstRun();
198
+ }
199
+ this.running = true;
200
+ console.log(`[Processor ${this.instanceId}] initialized, starting loop`);
201
+ this.runLoop();
202
+ }
203
+
204
+ private async bootstrapFirstRun(): Promise<void> {
205
+ console.log(`[Processor ${this.instanceId}] First run detected, bootstrapping Ei`);
206
+
207
+ const human = this.stateManager.getHuman();
208
+ this.stateManager.setHuman({
209
+ ...human,
210
+ settings: {
211
+ ...human.settings,
212
+ ceremony: {
213
+ time: human.settings?.ceremony?.time ?? "09:00",
214
+ last_ceremony: new Date().toISOString(),
215
+ },
216
+ },
217
+ });
218
+
219
+ const eiEntity: PersonaEntity = {
220
+ ...EI_PERSONA_DEFINITION,
221
+ id: "ei",
222
+ display_name: "Ei",
223
+ last_updated: new Date().toISOString(),
224
+ last_activity: new Date().toISOString(),
225
+ };
226
+ this.stateManager.persona_add(eiEntity);
227
+
228
+ const welcomeMessage: Message = {
229
+ id: crypto.randomUUID(),
230
+ role: "system",
231
+ content: EI_WELCOME_MESSAGE,
232
+ timestamp: new Date().toISOString(),
233
+ read: false,
234
+ context_status: ContextStatusEnum.Always,
235
+ };
236
+ this.stateManager.messages_append(eiEntity.id, welcomeMessage);
237
+
238
+ this.interface.onPersonaAdded?.();
239
+ this.interface.onMessageAdded?.(eiEntity.id);
240
+ }
241
+
242
+ async stop(): Promise<void> {
243
+ console.log(`[Processor ${this.instanceId}] stop() called, running=${this.running}, stopped=${this.stopped}`);
244
+ this.stopped = true;
245
+
246
+ if (!this.running) {
247
+ console.log(`[Processor ${this.instanceId}] not running, skipping save`);
248
+ return;
249
+ }
250
+
251
+ this.running = false;
252
+ this.queueProcessor.abort();
253
+ await this.stateManager.flush();
254
+ console.log(`[Processor ${this.instanceId}] stopped`);
255
+ }
256
+
257
+ async saveAndExit(): Promise<{ success: boolean; error?: string }> {
258
+ console.log(`[Processor ${this.instanceId}] saveAndExit() called`);
259
+ this.interface.onSaveAndExitStart?.();
260
+
261
+ this.queueProcessor.abort();
262
+ if (this.openCodeImportInProgress) {
263
+ console.log(`[Processor ${this.instanceId}] Aborting OpenCode import in progress`);
264
+ this.openCodeImportInProgress = false;
265
+ }
266
+
267
+ await this.stateManager.flush();
268
+
269
+ const human = this.stateManager.getHuman();
270
+ const hasSyncCreds = !!human.settings?.sync?.username && !!human.settings?.sync?.passphrase;
271
+
272
+ if (hasSyncCreds && remoteSync.isConfigured()) {
273
+ const state = this.stateManager.getStorageState();
274
+ const result = await remoteSync.sync(state);
275
+
276
+ if (!result.success) {
277
+ // Push failed — likely 412 etag mismatch or network error
278
+ // Do NOT moveToBackup — leave state.json intact
279
+ // Next boot will detect primary + remote → conflict resolution
280
+ console.log(`[Processor ${this.instanceId}] Remote sync failed: ${result.error}`);
281
+ await this.stop();
282
+ this.interface.onSaveAndExitFinish?.();
283
+ return { success: false, error: result.error };
284
+ }
285
+
286
+ await this.stateManager.moveToBackup();
287
+ console.log(`[Processor ${this.instanceId}] State moved to backup after successful sync`);
288
+ }
289
+
290
+ await this.stop();
291
+ this.interface.onSaveAndExitFinish?.();
292
+ return { success: true };
293
+ }
294
+
295
+
296
+ async resolveStateConflict(resolution: StateConflictResolution): Promise<void> {
297
+ if (!this.pendingConflict) return;
298
+
299
+ switch (resolution) {
300
+ case "local":
301
+ // Keep local, push to server on next exit
302
+ break;
303
+ case "server": {
304
+ const result = await remoteSync.fetch(); // gets fresh etag
305
+ if (result.success && result.state) {
306
+ this.stateManager.restoreFromState(result.state);
307
+ }
308
+ break;
309
+ }
310
+ case "yolo": {
311
+ const localState = this.stateManager.getStorageState();
312
+ const remoteResult = await remoteSync.fetch(); // gets fresh etag
313
+ if (remoteResult.success && remoteResult.state) {
314
+ const merged = yoloMerge(localState, remoteResult.state);
315
+ this.stateManager.restoreFromState(merged);
316
+ }
317
+ break;
318
+ }
319
+ }
320
+
321
+ this.pendingConflict = null;
322
+ this.running = true;
323
+ this.runLoop();
324
+ this.interface.onStateImported?.();
325
+ }
326
+
327
+ private async runLoop(): Promise<void> {
328
+ console.log(`[Processor ${this.instanceId}] runLoop() started`);
329
+ while (this.running) {
330
+ await this.checkScheduledTasks();
331
+
332
+ if (this.queueProcessor.getState() === "idle") {
333
+ const request = this.stateManager.queue_peekHighest();
334
+ if (request) {
335
+ const personaId = request.data.personaId as string | undefined;
336
+ const personaDisplayName = request.data.personaDisplayName as string | undefined;
337
+ const personaSuffix = personaDisplayName ? ` [${personaDisplayName}]` : "";
338
+ console.log(`[Processor ${this.instanceId}] processing request: ${request.next_step}${personaSuffix}`);
339
+ this.currentRequest = request;
340
+
341
+ if (personaId && request.next_step === LLMNextStep.HandlePersonaResponse) {
342
+ this.interface.onMessageProcessing?.(personaId);
343
+ }
344
+
345
+ this.queueProcessor.start(request, async (response) => {
346
+ this.currentRequest = null;
347
+ await this.handleResponse(response);
348
+ const nextState = this.stateManager.queue_isPaused() ? "paused" : "idle";
349
+ // the processor state is set in the caller, so this needs a bit of delay
350
+ setTimeout(() => this.interface.onQueueStateChanged?.(nextState), 0);
351
+ }, {
352
+ accounts: this.stateManager.getHuman().settings?.accounts,
353
+ messageFetcher: (pName) => this.fetchMessagesForLLM(pName),
354
+ rawMessageFetcher: (pName) => this.stateManager.messages_get(pName),
355
+ });
356
+
357
+ this.interface.onQueueStateChanged?.("busy");
358
+ }
359
+ }
360
+
361
+ await this.sleep(DEFAULT_LOOP_INTERVAL_MS);
362
+ }
363
+ console.log(`[Processor ${this.instanceId}] runLoop() exited`);
364
+ }
365
+
366
+ private async checkScheduledTasks(): Promise<void> {
367
+ const now = Date.now();
368
+ const DEFAULT_HEARTBEAT_DELAY_MS = 1800000; //5 * 60 * 1000;//
369
+
370
+ const human = this.stateManager.getHuman();
371
+
372
+ if (this.isTUI && human.settings?.opencode?.integration) {
373
+ await this.checkAndSyncOpenCode(human, now);
374
+ }
375
+
376
+ if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
377
+ // Auto-backup to remote before ceremony (if configured)
378
+ if (human.settings?.sync && remoteSync.isConfigured()) {
379
+ const state = this.stateManager.getStorageState();
380
+ const result = await remoteSync.sync(state);
381
+ if (!result.success) {
382
+ console.warn(`[Processor] Pre-ceremony remote backup failed: ${result.error}`);
383
+ }
384
+ }
385
+ startCeremony(this.stateManager);
386
+ }
387
+
388
+ for (const persona of this.stateManager.persona_getAll()) {
389
+ if (persona.is_paused || persona.is_archived) continue;
390
+
391
+ const heartbeatDelay = persona.heartbeat_delay_ms ?? DEFAULT_HEARTBEAT_DELAY_MS;
392
+ const lastActivity = persona.last_activity ? new Date(persona.last_activity).getTime() : 0;
393
+ const timeSinceActivity = now - lastActivity;
394
+
395
+ if (timeSinceActivity >= heartbeatDelay) {
396
+ const lastHeartbeat = persona.last_heartbeat
397
+ ? new Date(persona.last_heartbeat).getTime()
398
+ : 0;
399
+ const timeSinceHeartbeat = now - lastHeartbeat;
400
+
401
+ if (timeSinceHeartbeat >= heartbeatDelay) {
402
+ this.queueHeartbeatCheck(persona.id);
403
+ }
404
+ }
405
+ }
406
+ }
407
+
408
+ private async checkAndSyncOpenCode(human: HumanEntity, now: number): Promise<void> {
409
+ if (this.openCodeImportInProgress) {
410
+ return;
411
+ }
412
+
413
+ const opencode = human.settings?.opencode;
414
+ const pollingInterval = opencode?.polling_interval_ms ?? DEFAULT_OPENCODE_POLLING_MS;
415
+ const lastSync = opencode?.last_sync
416
+ ? new Date(opencode.last_sync).getTime()
417
+ : 0;
418
+ const timeSinceSync = now - lastSync;
419
+
420
+ if (timeSinceSync < pollingInterval && this.lastOpenCodeSync > 0) {
421
+ return;
422
+ }
423
+
424
+ this.lastOpenCodeSync = now;
425
+ const syncTimestamp = new Date().toISOString();
426
+ this.stateManager.setHuman({
427
+ ...this.stateManager.getHuman(),
428
+ settings: {
429
+ ...this.stateManager.getHuman().settings,
430
+ opencode: {
431
+ ...opencode,
432
+ last_sync: syncTimestamp,
433
+ },
434
+ },
435
+ });
436
+
437
+ const since = lastSync > 0 ? new Date(lastSync) : new Date(0);
438
+
439
+ this.openCodeImportInProgress = true;
440
+ import("../integrations/opencode/importer.js")
441
+ .then(({ importOpenCodeSessions }) =>
442
+ importOpenCodeSessions(since, {
443
+ stateManager: this.stateManager,
444
+ interface: this.interface,
445
+ })
446
+ )
447
+ .then((result) => {
448
+ if (result.sessionsProcessed > 0) {
449
+ console.log(
450
+ `[Processor] OpenCode sync complete: ${result.sessionsProcessed} sessions, ` +
451
+ `${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
452
+ `${result.topicUpdatesQueued} topic updates queued`
453
+ );
454
+ }
455
+ })
456
+ .catch((err) => {
457
+ console.warn(`[Processor] OpenCode sync failed:`, err);
458
+ })
459
+ .finally(() => {
460
+ this.openCodeImportInProgress = false;
461
+ });
462
+ }
463
+
464
+ private getModelForPersona(personaId?: string): string | undefined {
465
+ const human = this.stateManager.getHuman();
466
+ if (personaId) {
467
+ const persona = this.stateManager.persona_getById(personaId);
468
+ return persona?.model || human.settings?.default_model;
469
+ }
470
+ return human.settings?.default_model;
471
+ }
472
+
473
+ private fetchMessagesForLLM(personaId: string): import("./types.js").ChatMessage[] {
474
+ const persona = this.stateManager.persona_getById(personaId);
475
+ if (!persona) return [];
476
+
477
+ const history = this.stateManager.messages_get(personaId);
478
+ const contextWindowHours = persona.context_window_hours ?? DEFAULT_CONTEXT_WINDOW_HOURS;
479
+ const filteredHistory = filterMessagesForContext(
480
+ history,
481
+ persona.context_boundary,
482
+ contextWindowHours
483
+ );
484
+
485
+ return filteredHistory.map((m) => ({
486
+ role: m.role === "human" ? "user" : "assistant",
487
+ content: m.content,
488
+ })) as import("./types.js").ChatMessage[];
489
+ }
490
+
491
+ private async queueHeartbeatCheck(personaId: string): Promise<void> {
492
+ const persona = this.stateManager.persona_getById(personaId);
493
+ if (!persona) return;
494
+
495
+ this.stateManager.persona_update(personaId, { last_heartbeat: new Date().toISOString() });
496
+
497
+ const human = this.stateManager.getHuman();
498
+ const history = this.stateManager.messages_get(personaId);
499
+ const filteredHuman = await this.filterHumanDataByVisibility(human, persona);
500
+
501
+ const inactiveDays = persona.last_activity
502
+ ? Math.floor((Date.now() - new Date(persona.last_activity).getTime()) / (1000 * 60 * 60 * 24))
503
+ : 0;
504
+
505
+ const sortByEngagementGap = <T extends { exposure_desired: number; exposure_current: number }>(items: T[]): T[] =>
506
+ [...items].sort((a, b) => (b.exposure_desired - b.exposure_current) - (a.exposure_desired - a.exposure_current));
507
+
508
+ const promptData: HeartbeatCheckPromptData = {
509
+ persona: {
510
+ name: persona.display_name,
511
+ traits: persona.traits,
512
+ topics: persona.topics,
513
+ },
514
+ human: {
515
+ topics: sortByEngagementGap(filteredHuman.topics).slice(0, 5),
516
+ people: sortByEngagementGap(filteredHuman.people).slice(0, 5),
517
+ },
518
+ recent_history: history.slice(-10),
519
+ inactive_days: inactiveDays,
520
+ };
521
+
522
+ const prompt = buildHeartbeatCheckPrompt(promptData);
523
+
524
+ this.stateManager.queue_enqueue({
525
+ type: LLMRequestType.JSON,
526
+ priority: LLMPriority.Low,
527
+ system: prompt.system,
528
+ user: prompt.user,
529
+ next_step: LLMNextStep.HandleHeartbeatCheck,
530
+ model: this.getModelForPersona(personaId),
531
+ data: { personaId, personaDisplayName: persona.display_name },
532
+ });
533
+ }
534
+
535
+
536
+ private classifyLLMError(error: string): string {
537
+ const match = error.match(/\((\d{3})\)/);
538
+ if (match) {
539
+ const status = parseInt(match[1], 10);
540
+ if (status === 429) return "LLM_RATE_LIMITED";
541
+ if (status === 401 || status === 403) return "LLM_AUTH_ERROR";
542
+ if (status >= 500) return "LLM_SERVER_ERROR";
543
+ if (status >= 400) return "LLM_REQUEST_ERROR";
544
+ }
545
+ if (/timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED/i.test(error)) {
546
+ return "LLM_TIMEOUT";
547
+ }
548
+ return "LLM_ERROR";
549
+ }
550
+ private async handleResponse(response: LLMResponse): Promise<void> {
551
+ if (!response.success) {
552
+ const errorMsg = response.error ?? "Unknown LLM error";
553
+ const result = this.stateManager.queue_fail(response.request.id, errorMsg);
554
+ const code = this.classifyLLMError(errorMsg);
555
+
556
+ let message = errorMsg;
557
+ if (!result.dropped && result.retryDelay != null) {
558
+ message += ` (attempt ${response.request.attempts}, retrying in ${Math.round(result.retryDelay / 1000)}s)`;
559
+ } else if (result.dropped) {
560
+ message += " (permanent failure \u2014 request removed)";
561
+ }
562
+
563
+ this.interface.onError?.({ code, message });
564
+ return;
565
+ }
566
+
567
+ const handler = handlers[response.request.next_step as LLMNextStep];
568
+ if (!handler) {
569
+ const errorMsg = `No handler for ${response.request.next_step}`;
570
+ this.stateManager.queue_fail(response.request.id, errorMsg, true);
571
+ this.interface.onError?.({
572
+ code: "HANDLER_NOT_FOUND",
573
+ message: `${errorMsg} (permanent failure \u2014 request removed)`,
574
+ });
575
+ return;
576
+ }
577
+
578
+ try {
579
+ await handler(response, this.stateManager);
580
+ this.stateManager.queue_complete(response.request.id);
581
+
582
+ if (response.request.next_step === LLMNextStep.HandlePersonaResponse) {
583
+ // Always notify FE - even without content, user's message was "read" by the persona
584
+ const personaId = response.request.data.personaId as string;
585
+ if (personaId) {
586
+ this.interface.onMessageAdded?.(personaId);
587
+ }
588
+ }
589
+
590
+ if (response.request.next_step === LLMNextStep.HandleOneShot) {
591
+ const guid = response.request.data.guid as string;
592
+ const content = response.content ?? "";
593
+ this.interface.onOneShotReturned?.(guid, content);
594
+ }
595
+
596
+ if (response.request.next_step === LLMNextStep.HandlePersonaGeneration) {
597
+ const personaId = response.request.data.personaId as string;
598
+ if (personaId) {
599
+ this.interface.onPersonaUpdated?.(personaId);
600
+ }
601
+ }
602
+
603
+ if (response.request.next_step === LLMNextStep.HandlePersonaDescriptions) {
604
+ const personaId = response.request.data.personaId as string;
605
+ if (personaId) {
606
+ this.interface.onPersonaUpdated?.(personaId);
607
+ }
608
+ }
609
+
610
+ if (
611
+ response.request.next_step === LLMNextStep.HandlePersonaTraitExtraction ||
612
+ response.request.next_step === LLMNextStep.HandlePersonaTopicScan ||
613
+ response.request.next_step === LLMNextStep.HandlePersonaTopicMatch ||
614
+ response.request.next_step === LLMNextStep.HandlePersonaTopicUpdate
615
+ ) {
616
+ const personaId = response.request.data.personaId as string;
617
+ if (personaId) {
618
+ this.interface.onPersonaUpdated?.(personaId);
619
+ }
620
+ }
621
+
622
+ if (response.request.next_step === LLMNextStep.HandleHeartbeatCheck ||
623
+ response.request.next_step === LLMNextStep.HandleEiHeartbeat) {
624
+ const personaId = response.request.data.personaId as string ?? "ei";
625
+ if (response.content) {
626
+ this.interface.onMessageAdded?.(personaId);
627
+ }
628
+ }
629
+
630
+ if (response.request.next_step === LLMNextStep.HandleEiValidation) {
631
+ this.interface.onHumanUpdated?.();
632
+ }
633
+
634
+ if (response.request.next_step === LLMNextStep.HandleHumanItemUpdate) {
635
+ this.interface.onHumanUpdated?.();
636
+ this.interface.onQuoteAdded?.();
637
+ }
638
+
639
+ if (response.request.data.ceremony_progress) {
640
+ handleCeremonyProgress(this.stateManager);
641
+ }
642
+ } catch (err) {
643
+ const errorMsg = err instanceof Error ? err.message : String(err);
644
+ const result = this.stateManager.queue_fail(response.request.id, errorMsg);
645
+
646
+ let message = errorMsg;
647
+ if (!result.dropped && result.retryDelay != null) {
648
+ message += ` (attempt ${response.request.attempts}, retrying in ${Math.round(result.retryDelay / 1000)}s)`;
649
+ } else if (result.dropped) {
650
+ message += " (permanent failure \u2014 request removed)";
651
+ }
652
+ this.interface.onError?.({
653
+ code: "HANDLER_ERROR",
654
+ message,
655
+ });
656
+ }
657
+ }
658
+
659
+ private sleep(ms: number): Promise<void> {
660
+ return new Promise((resolve) => setTimeout(resolve, ms));
661
+ }
662
+
663
+ async getPersonaList(): Promise<PersonaSummary[]> {
664
+ return this.stateManager.persona_getAll().map((entity) => {
665
+ return {
666
+ id: entity.id,
667
+ display_name: entity.display_name,
668
+ aliases: entity.aliases ?? [],
669
+ short_description: entity.short_description,
670
+ is_paused: entity.is_paused,
671
+ is_archived: entity.is_archived,
672
+ unread_count: this.stateManager.messages_countUnread(entity.id),
673
+ last_activity: entity.last_activity,
674
+ context_boundary: entity.context_boundary,
675
+ };
676
+ });
677
+ }
678
+
679
+ /**
680
+ * Resolve a persona name or alias to its ID.
681
+ * Use this when the user types a name (e.g., "/persona Bob").
682
+ * Returns null if no matching persona is found.
683
+ */
684
+ async resolvePersonaName(nameOrAlias: string): Promise<string | null> {
685
+ const persona = this.stateManager.persona_getByName(nameOrAlias);
686
+ return persona?.id ?? null;
687
+ }
688
+
689
+ async getPersona(personaId: string): Promise<PersonaEntity | null> {
690
+ return this.stateManager.persona_getById(personaId);
691
+ }
692
+
693
+ async createPersona(input: PersonaCreationInput): Promise<string> {
694
+ if (isReservedPersonaName(input.name)) {
695
+ throw new Error(`Cannot create persona with reserved name "${input.name}". Reserved names: ${RESERVED_PERSONA_NAMES.join(", ")}`);
696
+ }
697
+ const now = new Date().toISOString();
698
+ const DEFAULT_GROUP = "General";
699
+ const personaId = crypto.randomUUID();
700
+ const placeholder: PersonaEntity = {
701
+ id: personaId,
702
+ display_name: input.name,
703
+ entity: "system",
704
+ aliases: input.aliases ?? [input.name],
705
+ short_description: input.short_description,
706
+ long_description: input.long_description,
707
+ model: input.model,
708
+ group_primary: input.group_primary ?? DEFAULT_GROUP,
709
+ groups_visible: input.groups_visible ?? [DEFAULT_GROUP],
710
+ traits: [],
711
+ topics: [],
712
+ is_paused: false,
713
+ is_archived: false,
714
+ is_static: false,
715
+ last_updated: now,
716
+ last_activity: now,
717
+ };
718
+ this.stateManager.persona_add(placeholder);
719
+ this.interface.onPersonaAdded?.();
720
+
721
+ orchestratePersonaGeneration(
722
+ { ...input, id: personaId },
723
+ this.stateManager,
724
+ () => this.interface.onPersonaUpdated?.(placeholder.display_name)
725
+ );
726
+
727
+ return personaId;
728
+ }
729
+
730
+ async archivePersona(personaId: string): Promise<void> {
731
+ const persona = this.stateManager.persona_getById(personaId);
732
+ if (!persona) return;
733
+ this.stateManager.persona_archive(personaId);
734
+ this.interface.onPersonaRemoved?.();
735
+ }
736
+
737
+ async unarchivePersona(personaId: string): Promise<void> {
738
+ const persona = this.stateManager.persona_getById(personaId);
739
+ if (!persona) return;
740
+ this.stateManager.persona_unarchive(personaId);
741
+ this.interface.onPersonaAdded?.();
742
+ }
743
+
744
+ async deletePersona(personaId: string, _deleteHumanData: boolean): Promise<void> {
745
+ const persona = this.stateManager.persona_getById(personaId);
746
+ if (!persona) return;
747
+ this.stateManager.persona_delete(personaId);
748
+ this.interface.onPersonaRemoved?.();
749
+ }
750
+
751
+ async updatePersona(personaId: string, updates: Partial<PersonaEntity>): Promise<void> {
752
+ const persona = this.stateManager.persona_getById(personaId);
753
+ if (!persona) return;
754
+ this.stateManager.persona_update(personaId, updates);
755
+ this.interface.onPersonaUpdated?.(personaId);
756
+ }
757
+
758
+ async getGroupList(): Promise<string[]> {
759
+ const personas = this.stateManager.persona_getAll();
760
+ const groups = new Set<string>();
761
+ for (const p of personas) {
762
+ if (p.group_primary) groups.add(p.group_primary);
763
+ for (const g of p.groups_visible || []) groups.add(g);
764
+ }
765
+ return [...groups].sort();
766
+ }
767
+
768
+ async getMessages(personaId: string, _options?: MessageQueryOptions): Promise<Message[]> {
769
+ const persona = this.stateManager.persona_getById(personaId);
770
+ if (!persona) return [];
771
+ return this.stateManager.messages_get(personaId);
772
+ }
773
+
774
+ async markMessageRead(personaId: string, messageId: string): Promise<boolean> {
775
+ const persona = this.stateManager.persona_getById(personaId);
776
+ if (!persona) return false;
777
+ return this.stateManager.messages_markRead(personaId, messageId);
778
+ }
779
+
780
+ async markAllMessagesRead(personaId: string): Promise<number> {
781
+ const persona = this.stateManager.persona_getById(personaId);
782
+ if (!persona) return 0;
783
+ return this.stateManager.messages_markAllRead(personaId);
784
+ }
785
+
786
+ private clearPendingRequestsFor(personaId: string): boolean {
787
+ const responsesToClear = [
788
+ LLMNextStep.HandlePersonaResponse,
789
+ LLMNextStep.HandlePersonaTraitExtraction,
790
+ LLMNextStep.HandlePersonaTopicScan,
791
+ LLMNextStep.HandlePersonaTopicMatch,
792
+ LLMNextStep.HandlePersonaTopicUpdate,
793
+ ];
794
+
795
+ let removedAny = false;
796
+ for (const nextStep of responsesToClear) {
797
+ const removedIds = this.stateManager.queue_clearPersonaResponses(personaId, nextStep);
798
+ if (removedIds.length > 0) removedAny = true;
799
+ }
800
+
801
+ const currentMatchesPersona = this.currentRequest &&
802
+ responsesToClear.includes(this.currentRequest.next_step as LLMNextStep) &&
803
+ this.currentRequest.data.personaId === personaId;
804
+
805
+ if (currentMatchesPersona) {
806
+ this.queueProcessor.abort();
807
+ return true;
808
+ }
809
+
810
+ return removedAny;
811
+ }
812
+
813
+ async recallPendingMessages(personaId: string): Promise<string> {
814
+ const persona = this.stateManager.persona_getById(personaId);
815
+ if (!persona) return "";
816
+
817
+ this.clearPendingRequestsFor(personaId);
818
+ this.stateManager.queue_pause();
819
+
820
+ const messages = this.stateManager.messages_get(personaId);
821
+ const pendingIds = messages
822
+ .filter(m => m.role === "human" && !m.read)
823
+ .map(m => m.id);
824
+
825
+ if (pendingIds.length === 0) return "";
826
+
827
+ const removed = this.stateManager.messages_remove(personaId, pendingIds);
828
+ const recalledContent = removed.map(m => m.content).join("\n\n");
829
+
830
+ this.interface.onMessageAdded?.(personaId);
831
+ this.interface.onMessageRecalled?.(personaId, recalledContent);
832
+
833
+ return recalledContent;
834
+ }
835
+
836
+ async sendMessage(personaId: string, content: string): Promise<void> {
837
+ const persona = this.stateManager.persona_getById(personaId);
838
+ if (!persona) {
839
+ this.interface.onError?.({
840
+ code: "PERSONA_NOT_FOUND",
841
+ message: `Persona with ID "${personaId}" not found`,
842
+ });
843
+ return;
844
+ }
845
+
846
+ this.clearPendingRequestsFor(personaId);
847
+
848
+ const message: Message = {
849
+ id: crypto.randomUUID(),
850
+ role: "human",
851
+ content,
852
+ timestamp: new Date().toISOString(),
853
+ read: false,
854
+ context_status: "default" as ContextStatus,
855
+ };
856
+ this.stateManager.messages_append(persona.id, message);
857
+ this.interface.onMessageAdded?.(persona.id);
858
+
859
+ const promptData = await this.buildResponsePromptData(persona, content);
860
+ const prompt = buildResponsePrompt(promptData);
861
+
862
+ this.stateManager.queue_enqueue({
863
+ type: LLMRequestType.Response,
864
+ priority: LLMPriority.High,
865
+ system: prompt.system,
866
+ user: prompt.user,
867
+ next_step: LLMNextStep.HandlePersonaResponse,
868
+ model: this.getModelForPersona(persona.id),
869
+ data: { personaId: persona.id, personaDisplayName: persona.display_name },
870
+ });
871
+ this.interface.onMessageQueued?.(persona.id);
872
+
873
+ const history = this.stateManager.messages_get(persona.id);
874
+
875
+ const traitExtractionData: PersonaTraitExtractionPromptData = {
876
+ persona_name: persona.display_name,
877
+ current_traits: persona.traits,
878
+ messages_context: history.slice(0, -1),
879
+ messages_analyze: [message],
880
+ };
881
+ const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
882
+
883
+ this.stateManager.queue_enqueue({
884
+ type: LLMRequestType.JSON,
885
+ priority: LLMPriority.Low,
886
+ system: traitPrompt.system,
887
+ user: traitPrompt.user,
888
+ next_step: LLMNextStep.HandlePersonaTraitExtraction,
889
+ model: this.getModelForPersona(persona.id),
890
+ data: { personaId: persona.id, personaDisplayName: persona.display_name },
891
+ });
892
+
893
+ this.checkAndQueueHumanExtraction(persona.id, persona.display_name, history);
894
+ }
895
+
896
+ /**
897
+ * Flare Note: I've gone back and forth on this several times, and want to leave a note for myself here:
898
+ * ***This is fine.***
899
+ * The effect here is that, if a person has 5 facts already, starts a new persona and says:
900
+ * "My name is Inigo Montoya"
901
+ * Then switches away, we won't process that message or the persona response for facts (or quotes about facts) until
902
+ * the Ceremony.
903
+ * And that's ***OK***
904
+ * The ONLY reason you need the facts on the Human record is so other Personas know _some_ information about the
905
+ * Human - the persona you just told it to will have it in it's context for their conversation, and we already know 5
906
+ * things **in that category** about them.
907
+ *
908
+ * TRAIT EXTRACTION NOTE: Traits are intentionally NOT extracted here. They're stable personality patterns that:
909
+ * 1. Don't change from message to message
910
+ * 2. Need more conversational data to identify accurately
911
+ * 3. Were causing massive queue bloat with cascading updates
912
+ * Trait extraction happens during Ceremony only, where we have a full day's context.
913
+ */
914
+ private checkAndQueueHumanExtraction(personaId: string, personaDisplayName: string, history: Message[]): void {
915
+ const human = this.stateManager.getHuman();
916
+
917
+ const unextractedFacts = this.stateManager.messages_getUnextracted(personaId, "f");
918
+ if (human.facts.length < unextractedFacts.length) {
919
+ const context: ExtractionContext = {
920
+ personaId,
921
+ personaDisplayName,
922
+ messages_context: history.filter(m => m.f === true),
923
+ messages_analyze: unextractedFacts,
924
+ extraction_flag: "f",
925
+ };
926
+ queueFactScan(context, this.stateManager);
927
+ console.log(`[Processor] Human Seed extraction: facts (${human.facts.length} < ${unextractedFacts.length} unextracted)`);
928
+ }
929
+
930
+ const unextractedTopics = this.stateManager.messages_getUnextracted(personaId, "p");
931
+ if (human.topics.length < unextractedTopics.length) {
932
+ const context: ExtractionContext = {
933
+ personaId,
934
+ personaDisplayName,
935
+ messages_context: history.filter(m => m.p === true),
936
+ messages_analyze: unextractedTopics,
937
+ extraction_flag: "p",
938
+ };
939
+ queueTopicScan(context, this.stateManager);
940
+ console.log(`[Processor] Human Seed extraction: topics (${human.topics.length} < ${unextractedTopics.length} unextracted)`);
941
+ }
942
+
943
+ const unextractedPeople = this.stateManager.messages_getUnextracted(personaId, "o");
944
+ if (human.people.length < unextractedPeople.length) {
945
+ const context: ExtractionContext = {
946
+ personaId,
947
+ personaDisplayName,
948
+ messages_context: history.filter(m => m.o === true),
949
+ messages_analyze: unextractedPeople,
950
+ extraction_flag: "o",
951
+ };
952
+ queuePersonScan(context, this.stateManager);
953
+ console.log(`[Processor] Human Seed extraction: people (${human.people.length} < ${unextractedPeople.length} unextracted)`);
954
+ }
955
+ }
956
+
957
+ private async buildResponsePromptData(persona: PersonaEntity, currentMessage?: string): Promise<ResponsePromptData> {
958
+ const human = this.stateManager.getHuman();
959
+ const filteredHuman = await this.filterHumanDataByVisibility(human, persona, currentMessage);
960
+ const visiblePersonas = this.getVisiblePersonas(persona);
961
+ const messages = this.stateManager.messages_get(persona.id);
962
+ const previousMessage = messages.length >= 2 ? messages[messages.length - 2] : null;
963
+ const delayMs = previousMessage
964
+ ? Date.now() - new Date(previousMessage.timestamp).getTime()
965
+ : 0;
966
+
967
+ return {
968
+ persona: {
969
+ name: persona.display_name,
970
+ aliases: persona.aliases ?? [],
971
+ short_description: persona.short_description,
972
+ long_description: persona.long_description,
973
+ traits: persona.traits,
974
+ topics: persona.topics,
975
+ },
976
+ human: filteredHuman,
977
+ visible_personas: visiblePersonas,
978
+ delay_ms: delayMs,
979
+ isTUI: this.isTUI,
980
+ };
981
+ }
982
+
983
+ private async filterHumanDataByVisibility(
984
+ human: HumanEntity,
985
+ persona: PersonaEntity,
986
+ currentMessage?: string
987
+ ): Promise<ResponsePromptData["human"]> {
988
+ const DEFAULT_GROUP = "General";
989
+ const QUOTE_LIMIT = 10;
990
+ const SIMILARITY_THRESHOLD = 0.3;
991
+
992
+ const selectRelevantQuotes = async (quotes: Quote[]): Promise<Quote[]> => {
993
+ if (quotes.length === 0) return [];
994
+
995
+ const withEmbeddings = quotes.filter(q => q.embedding?.length);
996
+
997
+ if (currentMessage && withEmbeddings.length > 0) {
998
+ try {
999
+ const embeddingService = getEmbeddingService();
1000
+ const queryVector = await embeddingService.embed(currentMessage);
1001
+ const results = findTopK(queryVector, withEmbeddings, QUOTE_LIMIT);
1002
+ const relevant = results
1003
+ .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
1004
+ .map(({ item }) => item);
1005
+
1006
+ if (relevant.length > 0) return relevant;
1007
+ } catch (err) {
1008
+ console.warn("[filterHumanDataByVisibility] Embedding search failed:", err);
1009
+ }
1010
+ }
1011
+
1012
+ return [...quotes]
1013
+ .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
1014
+ .slice(0, QUOTE_LIMIT);
1015
+ };
1016
+
1017
+ if (persona.id === "ei") {
1018
+ const relevantQuotes = await selectRelevantQuotes(human.quotes ?? []);
1019
+ return {
1020
+ facts: human.facts,
1021
+ traits: human.traits,
1022
+ topics: human.topics,
1023
+ people: human.people,
1024
+ quotes: relevantQuotes,
1025
+ };
1026
+ }
1027
+
1028
+ const visibleGroups = new Set<string>();
1029
+ if (persona.group_primary) {
1030
+ visibleGroups.add(persona.group_primary);
1031
+ }
1032
+ (persona.groups_visible ?? []).forEach((g) => visibleGroups.add(g));
1033
+
1034
+ const filterByGroup = <T extends DataItemBase>(items: T[]): T[] => {
1035
+ return items.filter((item) => {
1036
+ const itemGroups = item.persona_groups ?? [];
1037
+ const effectiveGroups = itemGroups.length === 0 ? [DEFAULT_GROUP] : itemGroups;
1038
+ return effectiveGroups.some((g) => visibleGroups.has(g));
1039
+ });
1040
+ };
1041
+
1042
+ const groupFilteredQuotes = (human.quotes ?? []).filter((q) => {
1043
+ const effectiveGroups = q.persona_groups.length === 0 ? [DEFAULT_GROUP] : q.persona_groups;
1044
+ return effectiveGroups.some((g) => visibleGroups.has(g));
1045
+ });
1046
+
1047
+ const relevantQuotes = await selectRelevantQuotes(groupFilteredQuotes);
1048
+
1049
+ return {
1050
+ facts: filterByGroup(human.facts),
1051
+ traits: filterByGroup(human.traits),
1052
+ topics: filterByGroup(human.topics),
1053
+ people: filterByGroup(human.people),
1054
+ quotes: relevantQuotes,
1055
+ };
1056
+ }
1057
+
1058
+ private getVisiblePersonas(
1059
+ currentPersona: PersonaEntity
1060
+ ): Array<{ name: string; short_description?: string }> {
1061
+ const allPersonas = this.stateManager.persona_getAll();
1062
+
1063
+ if (currentPersona.id === "ei") {
1064
+ return allPersonas
1065
+ .filter((p) => p.id !== "ei" && !p.is_archived)
1066
+ .map((p) => ({
1067
+ name: p.display_name,
1068
+ short_description: p.short_description,
1069
+ }));
1070
+ }
1071
+
1072
+ const visibleGroups = new Set<string>();
1073
+ if (currentPersona.group_primary) {
1074
+ visibleGroups.add(currentPersona.group_primary);
1075
+ }
1076
+ (currentPersona.groups_visible ?? []).forEach((g) => visibleGroups.add(g));
1077
+
1078
+ if (visibleGroups.size === 0) {
1079
+ return [];
1080
+ }
1081
+
1082
+ return allPersonas
1083
+ .filter((p) => {
1084
+ if (p.id === currentPersona.id || p.id === "ei" || p.is_archived) {
1085
+ return false;
1086
+ }
1087
+ return p.group_primary && visibleGroups.has(p.group_primary);
1088
+ })
1089
+ .map((p) => ({
1090
+ name: p.display_name,
1091
+ short_description: p.short_description,
1092
+ }));
1093
+ }
1094
+
1095
+ async setContextBoundary(personaId: string, timestamp: string | null): Promise<void> {
1096
+ this.stateManager.persona_setContextBoundary(personaId, timestamp);
1097
+ this.interface.onContextBoundaryChanged?.(personaId);
1098
+ }
1099
+
1100
+ async setMessageContextStatus(
1101
+ personaId: string,
1102
+ messageId: string,
1103
+ status: ContextStatus
1104
+ ): Promise<void> {
1105
+ this.stateManager.messages_setContextStatus(personaId, messageId, status);
1106
+ }
1107
+
1108
+ async deleteMessages(personaId: string, messageIds: string[]): Promise<Message[]> {
1109
+ const removed = this.stateManager.messages_remove(personaId, messageIds);
1110
+ this.interface.onMessageAdded?.(personaId);
1111
+ return removed;
1112
+ }
1113
+
1114
+ async getHuman(): Promise<HumanEntity> {
1115
+ return stripHumanEmbeddings(this.stateManager.getHuman());
1116
+ }
1117
+
1118
+ async updateHuman(updates: Partial<HumanEntity>): Promise<void> {
1119
+ const current = this.stateManager.getHuman();
1120
+ this.stateManager.setHuman({ ...current, ...updates });
1121
+ this.interface.onHumanUpdated?.();
1122
+ }
1123
+
1124
+ async getStorageState(): Promise<StorageState> {
1125
+ return this.stateManager.getStorageState();
1126
+ }
1127
+
1128
+ async restoreFromState(state: StorageState): Promise<void> {
1129
+ return this.stateManager.restoreFromState(state);
1130
+ }
1131
+
1132
+ async upsertFact(fact: Fact): Promise<void> {
1133
+ const human = this.stateManager.getHuman();
1134
+ const existing = human.facts.find(f => f.id === fact.id);
1135
+
1136
+ if (needsEmbeddingUpdate(existing, fact)) {
1137
+ fact.embedding = await computeDataItemEmbedding(fact);
1138
+ } else if (existing?.embedding) {
1139
+ fact.embedding = existing.embedding;
1140
+ }
1141
+
1142
+ this.stateManager.human_fact_upsert(fact);
1143
+ this.interface.onHumanUpdated?.();
1144
+ }
1145
+
1146
+ async upsertTrait(trait: Trait): Promise<void> {
1147
+ const human = this.stateManager.getHuman();
1148
+ const existing = human.traits.find(t => t.id === trait.id);
1149
+
1150
+ if (needsEmbeddingUpdate(existing, trait)) {
1151
+ trait.embedding = await computeDataItemEmbedding(trait);
1152
+ } else if (existing?.embedding) {
1153
+ trait.embedding = existing.embedding;
1154
+ }
1155
+
1156
+ this.stateManager.human_trait_upsert(trait);
1157
+ this.interface.onHumanUpdated?.();
1158
+ }
1159
+
1160
+ async upsertTopic(topic: Topic): Promise<void> {
1161
+ const human = this.stateManager.getHuman();
1162
+ const existing = human.topics.find(t => t.id === topic.id);
1163
+
1164
+ if (needsEmbeddingUpdate(existing, topic)) {
1165
+ topic.embedding = await computeDataItemEmbedding(topic);
1166
+ } else if (existing?.embedding) {
1167
+ topic.embedding = existing.embedding;
1168
+ }
1169
+
1170
+ this.stateManager.human_topic_upsert(topic);
1171
+ this.interface.onHumanUpdated?.();
1172
+ }
1173
+
1174
+ async upsertPerson(person: Person): Promise<void> {
1175
+ const human = this.stateManager.getHuman();
1176
+ const existing = human.people.find(p => p.id === person.id);
1177
+
1178
+ if (needsEmbeddingUpdate(existing, person)) {
1179
+ person.embedding = await computeDataItemEmbedding(person);
1180
+ } else if (existing?.embedding) {
1181
+ person.embedding = existing.embedding;
1182
+ }
1183
+
1184
+ this.stateManager.human_person_upsert(person);
1185
+ this.interface.onHumanUpdated?.();
1186
+ }
1187
+
1188
+ async removeDataItem(type: "fact" | "trait" | "topic" | "person", id: string): Promise<void> {
1189
+ switch (type) {
1190
+ case "fact":
1191
+ this.stateManager.human_fact_remove(id);
1192
+ break;
1193
+ case "trait":
1194
+ this.stateManager.human_trait_remove(id);
1195
+ break;
1196
+ case "topic":
1197
+ this.stateManager.human_topic_remove(id);
1198
+ break;
1199
+ case "person":
1200
+ this.stateManager.human_person_remove(id);
1201
+ break;
1202
+ }
1203
+ this.interface.onHumanUpdated?.();
1204
+ }
1205
+
1206
+ async addQuote(quote: Quote): Promise<void> {
1207
+ if (!quote.embedding) {
1208
+ quote.embedding = await computeQuoteEmbedding(quote.text);
1209
+ }
1210
+ this.stateManager.human_quote_add(quote);
1211
+ this.interface.onQuoteAdded?.();
1212
+ }
1213
+
1214
+ async updateQuote(id: string, updates: Partial<Quote>): Promise<void> {
1215
+ if (updates.text !== undefined) {
1216
+ const human = this.stateManager.getHuman();
1217
+ const existing = human.quotes.find(q => q.id === id);
1218
+
1219
+ if (needsQuoteEmbeddingUpdate(existing, { text: updates.text })) {
1220
+ updates.embedding = await computeQuoteEmbedding(updates.text);
1221
+ }
1222
+ }
1223
+ this.stateManager.human_quote_update(id, updates);
1224
+ this.interface.onQuoteUpdated?.();
1225
+ }
1226
+
1227
+ async removeQuote(id: string): Promise<void> {
1228
+ this.stateManager.human_quote_remove(id);
1229
+ this.interface.onQuoteRemoved?.();
1230
+ }
1231
+
1232
+ async getQuotes(filter?: { message_id?: string; data_item_id?: string }): Promise<Quote[]> {
1233
+ const human = this.stateManager.getHuman();
1234
+ let quotes: Quote[];
1235
+ if (!filter) {
1236
+ quotes = human.quotes;
1237
+ } else if (filter.message_id) {
1238
+ quotes = this.stateManager.human_quote_getForMessage(filter.message_id);
1239
+ } else if (filter.data_item_id) {
1240
+ quotes = this.stateManager.human_quote_getForDataItem(filter.data_item_id);
1241
+ } else {
1242
+ quotes = human.quotes;
1243
+ }
1244
+ return quotes.map(stripQuoteEmbedding);
1245
+ }
1246
+
1247
+ async getQuotesForMessage(messageId: string): Promise<Quote[]> {
1248
+ return this.stateManager.human_quote_getForMessage(messageId).map(stripQuoteEmbedding);
1249
+ }
1250
+
1251
+ async searchHumanData(
1252
+ query: string,
1253
+ options: { types?: Array<"fact" | "trait" | "topic" | "person" | "quote">; limit?: number } = {}
1254
+ ): Promise<{
1255
+ facts: Fact[];
1256
+ traits: Trait[];
1257
+ topics: Topic[];
1258
+ people: Person[];
1259
+ quotes: Quote[];
1260
+ }> {
1261
+ const { types = ["fact", "trait", "topic", "person", "quote"], limit = 10 } = options;
1262
+ const human = this.stateManager.getHuman();
1263
+ const SIMILARITY_THRESHOLD = 0.3;
1264
+
1265
+ const result = {
1266
+ facts: [] as Fact[],
1267
+ traits: [] as Trait[],
1268
+ topics: [] as Topic[],
1269
+ people: [] as Person[],
1270
+ quotes: [] as Quote[],
1271
+ };
1272
+
1273
+ let queryVector: number[] | null = null;
1274
+ try {
1275
+ const embeddingService = getEmbeddingService();
1276
+ queryVector = await embeddingService.embed(query);
1277
+ } catch (err) {
1278
+ console.warn("[searchHumanData] Failed to generate query embedding:", err);
1279
+ }
1280
+
1281
+ const searchItems = <T extends { id: string; embedding?: number[] }>(
1282
+ items: T[],
1283
+ textExtractor: (item: T) => string
1284
+ ): T[] => {
1285
+ const withEmbeddings = items.filter(i => i.embedding?.length);
1286
+
1287
+ if (queryVector && withEmbeddings.length > 0) {
1288
+ return findTopK(queryVector, withEmbeddings, limit)
1289
+ .filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
1290
+ .map(({ item }) => item);
1291
+ }
1292
+
1293
+ const lowerQuery = query.toLowerCase();
1294
+ return items
1295
+ .filter(i => textExtractor(i).toLowerCase().includes(lowerQuery))
1296
+ .slice(0, limit);
1297
+ };
1298
+
1299
+ if (types.includes("fact")) {
1300
+ result.facts = searchItems(human.facts, f => `${f.name} ${f.description || ""}`).map(stripDataItemEmbedding);
1301
+ }
1302
+ if (types.includes("trait")) {
1303
+ result.traits = searchItems(human.traits, t => `${t.name} ${t.description || ""}`).map(stripDataItemEmbedding);
1304
+ }
1305
+ if (types.includes("topic")) {
1306
+ result.topics = searchItems(human.topics, t => `${t.name} ${t.description || ""}`).map(stripDataItemEmbedding);
1307
+ }
1308
+ if (types.includes("person")) {
1309
+ result.people = searchItems(human.people, p => `${p.name} ${p.description || ""} ${p.relationship}`).map(stripDataItemEmbedding);
1310
+ }
1311
+ if (types.includes("quote")) {
1312
+ result.quotes = searchItems(human.quotes, q => q.text).map(stripQuoteEmbedding);
1313
+ }
1314
+
1315
+ return result;
1316
+ }
1317
+
1318
+ async exportState(): Promise<string> {
1319
+ const state = this.stateManager.getStorageState();
1320
+ return JSON.stringify(state, null, 2);
1321
+ }
1322
+
1323
+ async importState(json: string): Promise<void> {
1324
+ const state = JSON.parse(json) as StorageState;
1325
+ if (!state.version || !state.human || !state.personas) {
1326
+ throw new Error("Invalid backup file format");
1327
+ }
1328
+ this.stateManager.restoreFromState(state);
1329
+ this.interface.onStateImported?.();
1330
+ }
1331
+
1332
+ async abortCurrentOperation(): Promise<void> {
1333
+ this.stateManager.queue_pause();
1334
+ this.queueProcessor.abort();
1335
+ }
1336
+
1337
+ async resumeQueue(): Promise<void> {
1338
+ this.stateManager.queue_resume();
1339
+ }
1340
+
1341
+ async getQueueStatus(): Promise<QueueStatus> {
1342
+ return {
1343
+ state: this.stateManager.queue_isPaused()
1344
+ ? "paused"
1345
+ : this.queueProcessor.getState() === "busy"
1346
+ ? "busy"
1347
+ : "idle",
1348
+ pending_count: this.stateManager.queue_length(),
1349
+ };
1350
+ }
1351
+
1352
+ async clearQueue(): Promise<number> {
1353
+ this.queueProcessor.abort();
1354
+ return this.stateManager.queue_clear();
1355
+ }
1356
+
1357
+ async submitOneShot(guid: string, systemPrompt: string, userPrompt: string): Promise<void> {
1358
+ this.stateManager.queue_enqueue({
1359
+ type: LLMRequestType.Raw,
1360
+ priority: LLMPriority.High,
1361
+ system: systemPrompt,
1362
+ user: userPrompt,
1363
+ next_step: LLMNextStep.HandleOneShot,
1364
+ model: this.getModelForPersona(),
1365
+ data: { guid },
1366
+ });
1367
+ }
1368
+
1369
+ // ============================================================================
1370
+ // DEBUG / TESTING UTILITIES
1371
+ // ============================================================================
1372
+ // These methods are for development and testing. In browser devtools:
1373
+ // 1. Set a breakpoint in App.tsx or similar
1374
+ // 2. When it hits, access: processor.triggerCeremonyNow()
1375
+ // 3. Watch console for ceremony phase logs
1376
+ // ============================================================================
1377
+
1378
+ /**
1379
+ * Manually trigger ceremony execution, bypassing time checks.
1380
+ *
1381
+ * USE FROM BROWSER DEVTOOLS:
1382
+ * processor.triggerCeremonyNow()
1383
+ *
1384
+ * This will:
1385
+ * - Run ceremony for all active personas with recent activity
1386
+ * - Apply decay to all persona topics
1387
+ * - Queue Expire → Explore → DescCheck phases (LLM calls)
1388
+ * - Run Human ceremony (decay human topics/people)
1389
+ * - Update last_ceremony timestamp
1390
+ *
1391
+ * Watch console for detailed phase logging.
1392
+ */
1393
+ triggerCeremonyNow(): void {
1394
+ console.log("[Processor] Manual ceremony trigger requested");
1395
+ startCeremony(this.stateManager);
1396
+ }
1397
+
1398
+ /**
1399
+ * Get ceremony status for debugging.
1400
+ *
1401
+ * USE FROM BROWSER DEVTOOLS:
1402
+ * processor.getCeremonyStatus()
1403
+ */
1404
+ getCeremonyStatus(): { lastRun: string | null; nextRunTime: string } {
1405
+ const human = this.stateManager.getHuman();
1406
+ const config = human.settings?.ceremony;
1407
+
1408
+ return {
1409
+ lastRun: config?.last_ceremony ?? null,
1410
+ nextRunTime: `Today at ${config?.time ?? "09:00"}`,
1411
+ };
1412
+ }
1413
+ }