ei-tui 0.1.25 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +42 -0
  2. package/package.json +2 -1
  3. package/src/README.md +4 -11
  4. package/src/cli/README.md +87 -7
  5. package/src/cli/commands/facts.ts +2 -2
  6. package/src/cli/commands/people.ts +2 -2
  7. package/src/cli/commands/quotes.ts +2 -2
  8. package/src/cli/commands/topics.ts +2 -2
  9. package/src/cli/mcp.ts +94 -0
  10. package/src/cli/retrieval.ts +67 -31
  11. package/src/cli.ts +64 -23
  12. package/src/core/AGENTS.md +1 -1
  13. package/src/core/constants/built-in-facts.ts +49 -0
  14. package/src/core/constants/index.ts +1 -0
  15. package/src/core/context-utils.ts +0 -1
  16. package/src/core/embedding-service.ts +8 -0
  17. package/src/core/handlers/dedup.ts +11 -23
  18. package/src/core/handlers/heartbeat.ts +2 -3
  19. package/src/core/handlers/human-extraction.ts +96 -30
  20. package/src/core/handlers/human-matching.ts +328 -248
  21. package/src/core/handlers/index.ts +8 -6
  22. package/src/core/handlers/persona-generation.ts +8 -8
  23. package/src/core/handlers/rewrite.ts +4 -51
  24. package/src/core/handlers/utils.ts +23 -1
  25. package/src/core/heartbeat-manager.ts +2 -4
  26. package/src/core/human-data-manager.ts +38 -36
  27. package/src/core/message-manager.ts +10 -10
  28. package/src/core/orchestrators/ceremony.ts +49 -44
  29. package/src/core/orchestrators/dedup-phase.ts +2 -4
  30. package/src/core/orchestrators/human-extraction.ts +351 -207
  31. package/src/core/orchestrators/index.ts +6 -4
  32. package/src/core/orchestrators/persona-generation.ts +3 -3
  33. package/src/core/processor.ts +167 -20
  34. package/src/core/prompt-context-builder.ts +4 -6
  35. package/src/core/state/human.ts +1 -26
  36. package/src/core/state/personas.ts +2 -2
  37. package/src/core/state-manager.ts +107 -14
  38. package/src/core/tools/builtin/read-memory.ts +13 -18
  39. package/src/core/types/data-items.ts +3 -4
  40. package/src/core/types/entities.ts +7 -4
  41. package/src/core/types/enums.ts +6 -9
  42. package/src/core/types/llm.ts +2 -2
  43. package/src/core/utils/crossFind.ts +2 -5
  44. package/src/core/utils/event-windows.ts +31 -0
  45. package/src/integrations/claude-code/importer.ts +14 -5
  46. package/src/integrations/claude-code/types.ts +3 -0
  47. package/src/integrations/cursor/importer.ts +282 -0
  48. package/src/integrations/cursor/index.ts +10 -0
  49. package/src/integrations/cursor/reader.ts +209 -0
  50. package/src/integrations/cursor/types.ts +140 -0
  51. package/src/integrations/opencode/importer.ts +14 -4
  52. package/src/prompts/AGENTS.md +73 -1
  53. package/src/prompts/ceremony/dedup.ts +0 -33
  54. package/src/prompts/ceremony/rewrite.ts +6 -41
  55. package/src/prompts/ceremony/types.ts +4 -4
  56. package/src/prompts/generation/descriptions.ts +2 -2
  57. package/src/prompts/generation/types.ts +2 -2
  58. package/src/prompts/heartbeat/types.ts +2 -2
  59. package/src/prompts/human/event-scan.ts +122 -0
  60. package/src/prompts/human/fact-find.ts +106 -0
  61. package/src/prompts/human/fact-scan.ts +0 -2
  62. package/src/prompts/human/index.ts +17 -10
  63. package/src/prompts/human/person-match.ts +65 -0
  64. package/src/prompts/human/person-scan.ts +52 -59
  65. package/src/prompts/human/person-update.ts +241 -0
  66. package/src/prompts/human/topic-match.ts +65 -0
  67. package/src/prompts/human/topic-scan.ts +51 -71
  68. package/src/prompts/human/topic-update.ts +295 -0
  69. package/src/prompts/human/types.ts +63 -40
  70. package/src/prompts/index.ts +4 -8
  71. package/src/prompts/persona/topics-update.ts +2 -2
  72. package/src/prompts/persona/traits.ts +2 -2
  73. package/src/prompts/persona/types.ts +3 -3
  74. package/src/prompts/response/index.ts +1 -1
  75. package/src/prompts/response/sections.ts +9 -12
  76. package/src/prompts/response/types.ts +2 -3
  77. package/src/storage/embeddings.ts +1 -1
  78. package/src/storage/index.ts +1 -0
  79. package/src/storage/indexed.ts +174 -0
  80. package/src/storage/merge.ts +67 -2
  81. package/tui/src/commands/me.tsx +5 -14
  82. package/tui/src/commands/settings.tsx +15 -0
  83. package/tui/src/context/ei.tsx +5 -14
  84. package/tui/src/util/yaml-serializers.ts +76 -33
  85. package/src/cli/commands/traits.ts +0 -25
  86. package/src/prompts/human/item-match.ts +0 -74
  87. package/src/prompts/human/item-update.ts +0 -364
  88. package/src/prompts/human/trait-scan.ts +0 -115
@@ -1,12 +1,14 @@
1
1
  export { orchestratePersonaGeneration, type PartialPersona } from "./persona-generation.js";
2
2
  export {
3
- queueFactScan,
4
- queueTraitScan,
3
+ queueFactFind,
5
4
  queueTopicScan,
6
5
  queuePersonScan,
7
6
  queueAllScans,
8
- queueItemMatch,
9
- queueItemUpdate,
7
+ queueTopicMatch,
8
+ queueTopicUpdate,
9
+ queuePersonMatch,
10
+ queuePersonUpdate,
11
+ queueEventSummary,
10
12
  type ExtractionContext,
11
13
  type ExtractionOptions,
12
14
  } from "./human-extraction.js";
@@ -1,4 +1,4 @@
1
- import { LLMRequestType, LLMPriority, LLMNextStep, type Trait, type PersonaTopic } from "../types.js";
1
+ import { LLMRequestType, LLMPriority, LLMNextStep, type PersonaTrait, type PersonaTopic } from "../types.js";
2
2
  import type { StateManager } from "../state-manager.js";
3
3
  import { buildPersonaGenerationPrompt } from "../../prompts/index.js";
4
4
 
@@ -11,7 +11,7 @@ export interface PartialPersona {
11
11
  description?: string;
12
12
  short_description?: string;
13
13
  long_description?: string;
14
- traits?: Partial<Trait>[];
14
+ traits?: Partial<PersonaTrait>[];
15
15
  topics?: Partial<PersonaTopic>[];
16
16
  model?: string;
17
17
  group_primary?: string;
@@ -66,7 +66,7 @@ export function orchestratePersonaGeneration(
66
66
  stateManager.persona_update(partial.id, {
67
67
  short_description: partial.short_description,
68
68
  long_description: partial.long_description,
69
- traits: partial.traits as Trait[],
69
+ traits: partial.traits as PersonaTrait[],
70
70
  topics: partial.topics as PersonaTopic[],
71
71
  last_updated: now,
72
72
  });
@@ -9,7 +9,6 @@ import {
9
9
  type MessageQueryOptions,
10
10
  type HumanEntity,
11
11
  type Fact,
12
- type Trait,
13
12
  type Topic,
14
13
  type Person,
15
14
  type Quote,
@@ -33,6 +32,7 @@ import { registerReadMemoryExecutor, registerFileReadExecutor } from "./tools/in
33
32
  import { createReadMemoryExecutor } from "./tools/builtin/read-memory.js";
34
33
  import { EI_WELCOME_MESSAGE, EI_PERSONA_DEFINITION } from "../templates/welcome.js";
35
34
  import { shouldStartCeremony, startCeremony, handleCeremonyProgress } from "./orchestrators/index.js";
35
+ import { BUILT_IN_FACTS } from "./constants/built-in-facts.js";
36
36
 
37
37
  // Static module imports
38
38
  import {
@@ -70,7 +70,6 @@ import {
70
70
  getHuman,
71
71
  updateHuman,
72
72
  upsertFact,
73
- upsertTrait,
74
73
  upsertTopic,
75
74
  upsertPerson,
76
75
  removeDataItem,
@@ -109,6 +108,7 @@ const DEFAULT_LOOP_INTERVAL_MS = 100;
109
108
  const DEFAULT_CONTEXT_WINDOW_HOURS = 8;
110
109
  const DEFAULT_OPENCODE_POLLING_MS = 1800000;
111
110
  const DEFAULT_CLAUDE_CODE_POLLING_MS = 1800000;
111
+ const DEFAULT_CURSOR_POLLING_MS = 1800000;
112
112
 
113
113
  let processorInstanceCount = 0;
114
114
 
@@ -129,6 +129,8 @@ export class Processor {
129
129
  private openCodeImportInProgress = false;
130
130
  private lastClaudeCodeSync = 0;
131
131
  private claudeCodeImportInProgress = false;
132
+ private lastCursorSync = 0;
133
+ private cursorImportInProgress = false;
132
134
  private pendingConflict: StateConflictData | null = null;
133
135
  private storage: Storage | null = null;
134
136
  private importAbortController = new AbortController();
@@ -191,10 +193,16 @@ export class Processor {
191
193
  }
192
194
  }
193
195
 
196
+ await this.completeInitialization();
197
+ }
198
+
199
+ private async completeInitialization(): Promise<void> {
194
200
  if (!this.stateManager.hasExistingData() || this.stateManager.persona_getAll().length === 0) {
195
201
  await this.bootstrapFirstRun();
196
202
  }
197
203
  this.bootstrapTools();
204
+ this.seedBuiltinFacts();
205
+ this.seedSettings();
198
206
  registerReadMemoryExecutor(createReadMemoryExecutor(this.searchHumanData.bind(this)));
199
207
  if (this.isTUI) {
200
208
  await registerFileReadExecutor();
@@ -273,19 +281,20 @@ export class Processor {
273
281
  name: "read_memory",
274
282
  display_name: "Read Memory",
275
283
  description:
276
- "Search your personal memory for relevant facts, traits, topics, people, or quotes. Use this when you need information about the user that may not be in the current conversation.",
284
+ "Search your personal memory for relevant facts, topics, people, or quotes. Use this when you need information about the user that may not be in the current conversation. Use `recent: true` to retrieve what's been discussed recently.",
277
285
  input_schema: {
278
286
  type: "object",
279
287
  properties: {
280
288
  query: { type: "string", description: "What to search for in memory" },
281
289
  types: {
282
290
  type: "array",
283
- items: { type: "string", enum: ["fact", "trait", "topic", "person", "quote"] },
291
+ items: { type: "string", enum: ["fact", "topic", "person", "quote"] },
284
292
  description: "Limit search to specific memory types (default: all types)",
285
293
  },
286
294
  limit: { type: "number", description: "Max results to return (default: 10, max: 20)" },
295
+ recent: { type: "boolean", description: "If true, return recently-mentioned results sorted by last_mentioned date instead of relevance. Combine with a query to filter recent results by topic." },
287
296
  },
288
- required: ["query"],
297
+ required: [],
289
298
  },
290
299
  runtime: "any",
291
300
  builtin: true,
@@ -571,6 +580,87 @@ export class Processor {
571
580
  }
572
581
  }
573
582
 
583
+ /**
584
+ * Seed 25 built-in facts if they don't exist yet.
585
+ * Called on every startup — safe to call repeatedly.
586
+ * New facts are created with empty descriptions and validated_date.
587
+ */
588
+ private seedBuiltinFacts(): void {
589
+ const human = this.stateManager.getHuman();
590
+ const existingFactNames = new Set(human.facts.map(f => f.name));
591
+
592
+ // BUILT_IN_FACTS imported at top of file
593
+ const now = new Date().toISOString();
594
+ let seededCount = 0;
595
+
596
+ for (const builtInFact of BUILT_IN_FACTS) {
597
+ if (existingFactNames.has(builtInFact.name)) continue;
598
+
599
+ const newFact: Fact = {
600
+ id: crypto.randomUUID(),
601
+ name: builtInFact.name,
602
+ description: '',
603
+ sentiment: 0,
604
+ validated_date: '',
605
+ last_updated: now,
606
+ };
607
+ human.facts.push(newFact);
608
+ seededCount++;
609
+ }
610
+
611
+ if (seededCount > 0) {
612
+ this.stateManager.setHuman(human);
613
+ console.log(`[Processor] Seeded ${seededCount} built-in facts`);
614
+ }
615
+ }
616
+
617
+ private seedSettings(): void {
618
+ const human = this.stateManager.getHuman();
619
+ let modified = false;
620
+
621
+ if (!human.settings) {
622
+ human.settings = {};
623
+ modified = true;
624
+ }
625
+
626
+ if (!human.settings.opencode) {
627
+ human.settings.opencode = {
628
+ integration: false,
629
+ polling_interval_ms: 1800000,
630
+ };
631
+ modified = true;
632
+ }
633
+
634
+ if (!human.settings.claudeCode) {
635
+ human.settings.claudeCode = {
636
+ integration: false,
637
+ polling_interval_ms: 1800000,
638
+ };
639
+ modified = true;
640
+ }
641
+
642
+ if (!human.settings.ceremony) {
643
+ human.settings.ceremony = {
644
+ time: "09:00",
645
+ };
646
+ modified = true;
647
+ }
648
+
649
+ if (!human.settings.backup) {
650
+ human.settings.backup = {
651
+ enabled: false,
652
+ max_backups: 24,
653
+ interval_ms: 3600000,
654
+ };
655
+ modified = true;
656
+ }
657
+
658
+ if (modified) {
659
+ this.stateManager.setHuman(human);
660
+ console.log(`[Processor] Seeded missing settings`);
661
+ }
662
+ }
663
+
574
664
  async stop(): Promise<void> {
575
665
  console.log(
576
666
  `[Processor ${this.instanceId}] stop() called, running=${this.running}, stopped=${this.stopped}`
@@ -676,13 +766,7 @@ export class Processor {
676
766
 
677
767
  this.pendingConflict = null;
678
768
  this.importAbortController = new AbortController();
679
- this.bootstrapTools();
680
- registerReadMemoryExecutor(createReadMemoryExecutor(this.searchHumanData.bind(this)));
681
- if (this.isTUI) {
682
- await registerFileReadExecutor();
683
- }
684
- this.running = true;
685
- this.runLoop();
769
+ await this.completeInitialization();
686
770
  this.interface.onStateImported?.();
687
771
  }
688
772
 
@@ -796,6 +880,14 @@ const toolNextSteps = new Set([
796
880
  await this.checkAndSyncClaudeCode(human, now);
797
881
  }
798
882
 
883
+ if (
884
+ this.isTUI &&
885
+ human.settings?.cursor?.integration &&
886
+ this.stateManager.queue_length() === 0
887
+ ) {
888
+ await this.checkAndSyncCursor(human, now);
889
+ }
890
+
799
891
  if (human.settings?.ceremony && shouldStartCeremony(human.settings.ceremony, this.stateManager)) {
800
892
  if (human.settings?.sync && remoteSync.isConfigured()) {
801
893
  const state = this.stateManager.getStorageState();
@@ -982,6 +1074,59 @@ const toolNextSteps = new Set([
982
1074
  });
983
1075
  }
984
1076
 
1077
+ private async checkAndSyncCursor(human: HumanEntity, now: number): Promise<void> {
1078
+ if (this.cursorImportInProgress) {
1079
+ return;
1080
+ }
1081
+
1082
+ const cursor = human.settings?.cursor;
1083
+ const pollingInterval = cursor?.polling_interval_ms ?? DEFAULT_CURSOR_POLLING_MS;
1084
+ const lastSync = cursor?.last_sync ? new Date(cursor.last_sync).getTime() : 0;
1085
+ const timeSinceSync = now - lastSync;
1086
+
1087
+ if (timeSinceSync < pollingInterval && this.lastCursorSync > 0) {
1088
+ return;
1089
+ }
1090
+
1091
+ this.lastCursorSync = now;
1092
+ const syncTimestamp = new Date().toISOString();
1093
+ this.stateManager.setHuman({
1094
+ ...this.stateManager.getHuman(),
1095
+ settings: {
1096
+ ...this.stateManager.getHuman().settings,
1097
+ cursor: {
1098
+ ...cursor,
1099
+ last_sync: syncTimestamp,
1100
+ },
1101
+ },
1102
+ });
1103
+
1104
+ this.cursorImportInProgress = true;
1105
+ import("../integrations/cursor/importer.js")
1106
+ .then(({ importCursorSessions }) =>
1107
+ importCursorSessions({
1108
+ stateManager: this.stateManager,
1109
+ interface: this.interface,
1110
+ signal: this.importAbortController.signal,
1111
+ })
1112
+ )
1113
+ .then((result) => {
1114
+ if (result.sessionsProcessed > 0) {
1115
+ console.log(
1116
+ `[Processor] Cursor sync complete: ${result.sessionsProcessed} sessions, ` +
1117
+ `${result.topicsCreated} topics created, ${result.messagesImported} messages imported, ` +
1118
+ `${result.extractionScansQueued} extraction scans queued`
1119
+ );
1120
+ }
1121
+ })
1122
+ .catch((err) => {
1123
+ console.warn(`[Processor] Cursor sync failed:`, err);
1124
+ })
1125
+ .finally(() => {
1126
+ this.cursorImportInProgress = false;
1127
+ });
1128
+ }
1129
+
985
1130
  private classifyLLMError(error: string): string {
986
1131
  const match = error.match(/\((\d{3})\)/);
987
1132
  if (match) {
@@ -1093,7 +1238,10 @@ const toolNextSteps = new Set([
1093
1238
  }
1094
1239
  }
1095
1240
 
1096
- if (response.request.next_step === LLMNextStep.HandleHumanItemUpdate) {
1241
+ if (
1242
+ response.request.next_step === LLMNextStep.HandleTopicUpdate ||
1243
+ response.request.next_step === LLMNextStep.HandlePersonUpdate
1244
+ ) {
1097
1245
  this.interface.onHumanUpdated?.();
1098
1246
  this.interface.onQuoteAdded?.();
1099
1247
  }
@@ -1102,6 +1250,10 @@ const toolNextSteps = new Set([
1102
1250
  this.interface.onHumanUpdated?.();
1103
1251
  }
1104
1252
 
1253
+ if (response.request.next_step === LLMNextStep.HandleFactFind) {
1254
+ this.interface.onHumanUpdated?.();
1255
+ }
1256
+
1105
1257
  if (typeof response.request.data.ceremony_progress === "number") {
1106
1258
  handleCeremonyProgress(this.stateManager, response.request.data.ceremony_progress);
1107
1259
  }
@@ -1265,10 +1417,6 @@ const toolNextSteps = new Set([
1265
1417
  this.interface.onHumanUpdated?.();
1266
1418
  }
1267
1419
 
1268
- async upsertTrait(trait: Trait): Promise<void> {
1269
- await upsertTrait(this.stateManager, trait);
1270
- this.interface.onHumanUpdated?.();
1271
- }
1272
1420
 
1273
1421
  async upsertTopic(topic: Topic): Promise<void> {
1274
1422
  await upsertTopic(this.stateManager, topic);
@@ -1281,7 +1429,7 @@ const toolNextSteps = new Set([
1281
1429
  }
1282
1430
 
1283
1431
  async removeDataItem(
1284
- type: "fact" | "trait" | "topic" | "person",
1432
+ type: "fact" | "topic" | "person",
1285
1433
  id: string
1286
1434
  ): Promise<void> {
1287
1435
  await removeDataItem(this.stateManager, type, id);
@@ -1313,10 +1461,9 @@ const toolNextSteps = new Set([
1313
1461
 
1314
1462
  async searchHumanData(
1315
1463
  query: string,
1316
- options: { types?: Array<"fact" | "trait" | "topic" | "person" | "quote">; limit?: number } = {}
1464
+ options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
1317
1465
  ): Promise<{
1318
1466
  facts: Fact[];
1319
- traits: Trait[];
1320
1467
  topics: Topic[];
1321
1468
  people: Person[];
1322
1469
  quotes: Quote[];
@@ -80,14 +80,13 @@ export async function filterHumanDataByVisibility(
80
80
  const DEFAULT_GROUP = "General";
81
81
 
82
82
  if (persona.id === "ei") {
83
- const [facts, traits, topics, people, quotes] = await Promise.all([
83
+ const [facts, topics, people, quotes] = await Promise.all([
84
84
  selectRelevantItems(human.facts, DATA_ITEM_LIMIT, currentMessage),
85
- selectRelevantItems(human.traits, DATA_ITEM_LIMIT, currentMessage),
86
85
  selectRelevantItems(human.topics, DATA_ITEM_LIMIT, currentMessage),
87
86
  selectRelevantItems(human.people, DATA_ITEM_LIMIT, currentMessage),
88
87
  selectRelevantQuotes(human.quotes ?? [], currentMessage),
89
88
  ]);
90
- return { facts, traits, topics, people, quotes };
89
+ return { facts, topics, people, quotes };
91
90
  }
92
91
 
93
92
  const visibleGroups = new Set<string>();
@@ -109,15 +108,14 @@ export async function filterHumanDataByVisibility(
109
108
  return effectiveGroups.some((g) => visibleGroups.has(g));
110
109
  });
111
110
 
112
- const [facts, traits, topics, people, quotes] = await Promise.all([
111
+ const [facts, topics, people, quotes] = await Promise.all([
113
112
  selectRelevantItems(filterByGroup(human.facts), DATA_ITEM_LIMIT, currentMessage),
114
- selectRelevantItems(filterByGroup(human.traits), DATA_ITEM_LIMIT, currentMessage),
115
113
  selectRelevantItems(filterByGroup(human.topics), DATA_ITEM_LIMIT, currentMessage),
116
114
  selectRelevantItems(filterByGroup(human.people), DATA_ITEM_LIMIT, currentMessage),
117
115
  selectRelevantQuotes(groupFilteredQuotes, currentMessage),
118
116
  ]);
119
117
 
120
- return { facts, traits, topics, people, quotes };
118
+ return { facts, topics, people, quotes };
121
119
  }
122
120
 
123
121
  // =============================================================================
@@ -1,10 +1,9 @@
1
- import type { HumanEntity, Fact, Trait, Topic, Person, Quote } from "../types.js";
1
+ import type { HumanEntity, Fact, Topic, Person, Quote } from "../types.js";
2
2
 
3
3
  export function createDefaultHumanEntity(): HumanEntity {
4
4
  return {
5
5
  entity: "human",
6
6
  facts: [],
7
- traits: [],
8
7
  topics: [],
9
8
  people: [],
10
9
  quotes: [],
@@ -63,30 +62,6 @@ export class HumanState {
63
62
  return false;
64
63
  }
65
64
 
66
- trait_upsert(trait: Trait): void {
67
- const idx = this.human.traits.findIndex((t) => t.id === trait.id);
68
- trait.last_updated = new Date().toISOString();
69
- if (idx >= 0) {
70
- this.human.traits[idx] = trait;
71
- } else {
72
- this.human.traits.push(trait);
73
- }
74
- this.human.last_updated = new Date().toISOString();
75
- }
76
-
77
- trait_remove(id: string): boolean {
78
- const idx = this.human.traits.findIndex((t) => t.id === id);
79
- if (idx >= 0) {
80
- this.human.traits.splice(idx, 1);
81
- // Clean up quote references
82
- this.human.quotes.forEach((q) => {
83
- q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== id);
84
- });
85
- this.human.last_updated = new Date().toISOString();
86
- return true;
87
- }
88
- return false;
89
- }
90
65
 
91
66
  topic_upsert(topic: Topic): void {
92
67
  const idx = this.human.topics.findIndex((t) => t.id === topic.id);
@@ -213,7 +213,7 @@ export class PersonaState {
213
213
  return removed;
214
214
  }
215
215
 
216
- messages_getUnextracted(personaId: string, flag: "f" | "r" | "p" | "o", limit?: number): Message[] {
216
+ messages_getUnextracted(personaId: string, flag: "f" | "t" | "p" | "e", limit?: number): Message[] {
217
217
  const data = this.personas.get(personaId);
218
218
  if (!data) return [];
219
219
  const unextracted = data.messages.filter(m => m[flag] !== true);
@@ -223,7 +223,7 @@ export class PersonaState {
223
223
  return unextracted.map(m => ({ ...m }));
224
224
  }
225
225
 
226
- messages_markExtracted(personaId: string, messageIds: string[], flag: "f" | "r" | "p" | "o"): number {
226
+ messages_markExtracted(personaId: string, messageIds: string[], flag: "f" | "t" | "p" | "e"): number {
227
227
  const data = this.personas.get(personaId);
228
228
  if (!data) return 0;
229
229
  const idsSet = new Set(messageIds);
@@ -3,7 +3,6 @@ import type {
3
3
  PersonaEntity,
4
4
  Message,
5
5
  Fact,
6
- Trait,
7
6
  Topic,
8
7
  Person,
9
8
  Quote,
@@ -14,6 +13,7 @@ import type {
14
13
  ToolDefinition,
15
14
  ToolProvider,
16
15
  } from "./types.js";
16
+ import { BUILT_IN_FACT_NAMES } from './constants/built-in-facts.js';
17
17
  import type { Storage } from "../storage/interface.js";
18
18
  import {
19
19
  HumanState,
@@ -43,6 +43,8 @@ export class StateManager {
43
43
  this.tools = state.tools ?? [];
44
44
  this.providers = state.providers ?? [];
45
45
  this.migrateLearnedByToIds();
46
+ this.migrateFactValidation();
47
+ this.migrateMessageFlags();
46
48
  } else {
47
49
  this.humanState.load(createDefaultHumanEntity());
48
50
  }
@@ -80,13 +82,114 @@ export class StateManager {
80
82
  dirty = true;
81
83
  }
82
84
  };
83
- [...human.facts, ...human.traits, ...human.topics, ...human.people].forEach(migrateItem);
85
+ [...human.facts, ...human.topics, ...human.people].forEach(migrateItem);
84
86
  if (dirty) {
85
87
  this.humanState.set(human);
86
88
  console.log("[StateManager] Migrated learned_by fields from display names to persona IDs");
87
89
  }
88
90
  }
89
91
 
92
+ /**
93
+ * Migration: Facts used to have a 'validated' field (now removed).
94
+ * Now, only 25 built-in facts remain; others are converted to Topics with category='Fact'.
95
+ * - Facts with 'validated' field whose name is NOT in BUILT_IN_FACT_NAMES → move to Topics
96
+ * - Facts with 'validated' field whose name IS in BUILT_IN_FACT_NAMES → strip 'validated'
97
+ * No-op for already-migrated data (no 'validated' field present).
98
+ */
99
+ private migrateFactValidation(): void {
100
+ const human = this.humanState.get();
101
+
102
+ // Check if any fact has 'validated' property (old format detection)
103
+ const hasOldFormat = human.facts.some((f) => 'validated' in f);
104
+ if (!hasOldFormat) return;
105
+
106
+ let dirty = false;
107
+ const newFacts: Fact[] = [];
108
+ let movedCount = 0;
109
+ let strippedCount = 0;
110
+ // Define legacy fact interface for type-safe migration
111
+ interface LegacyFact extends Fact {
112
+ validated?: boolean;
113
+ }
114
+
115
+ for (const fact of human.facts) {
116
+ if (!('validated' in fact)) {
117
+ // Already migrated fact, keep as-is
118
+ newFacts.push(fact);
119
+ continue;
120
+ }
121
+
122
+ if (BUILT_IN_FACT_NAMES.has(fact.name)) {
123
+ // Matching built-in: strip 'validated' field, preserve description
124
+ const { validated, ...cleanedFact } = fact as LegacyFact;
125
+ newFacts.push(cleanedFact);
126
+ strippedCount++;
127
+ dirty = true;
128
+ } else {
129
+ // Non-matching: move to Topics
130
+ const newTopic: Topic = {
131
+ id: crypto.randomUUID(),
132
+ name: fact.name,
133
+ description: fact.description,
134
+ category: 'Fact',
135
+ sentiment: fact.sentiment,
136
+ exposure_current: 0.3,
137
+ exposure_desired: 0.3,
138
+ last_updated: fact.last_updated,
139
+ learned_by: fact.learned_by,
140
+ last_changed_by: fact.last_changed_by,
141
+ persona_groups: fact.persona_groups,
142
+ embedding: fact.embedding,
143
+ };
144
+ human.topics.push(newTopic);
145
+ movedCount++;
146
+ dirty = true;
147
+ }
148
+ }
149
+
150
+ if (dirty) {
151
+ human.facts = newFacts;
152
+ this.humanState.set(human);
153
+ console.log(
154
+ `[StateManager] Migrated fact validation: moved ${movedCount} non-matching facts to Topics, stripped 'validated' from ${strippedCount} built-in facts`
155
+ );
156
+ }
157
+ }
158
+
159
+ /**
160
+ * Migration: Message extraction flags were incorrectly named.
161
+ * Old: p=Topics, o=People, r=Traits (dead)
162
+ * New: t=Topics, p=People (r and o removed)
163
+ * Detects old format by presence of 'o' flag on any message.
164
+ */
165
+ private migrateMessageFlags(): void {
166
+ const personas = this.personaState.getAll();
167
+ let migratedCount = 0;
168
+
169
+ for (const persona of personas) {
170
+ // Access raw message objects to detect and remap old flags
171
+ const rawMessages = (this.personaState as unknown as { personas: Map<string, { messages: Array<Record<string, unknown>> }> }).personas.get(persona.id)?.messages ?? [];
172
+ const hasOldFormat = rawMessages.some(m => 'o' in m || 'r' in m);
173
+ if (!hasOldFormat) continue;
174
+
175
+ for (const msg of rawMessages) {
176
+ // Remap: old p (topics) → new t; old o (people) → new p
177
+ const oldP = msg['p']; // was topics
178
+ const oldO = msg['o']; // was people
179
+ msg['t'] = oldP; // topics: old p → new t
180
+ msg['p'] = oldO; // people: old o → new p
181
+ delete msg['r']; // trait flag dead
182
+ delete msg['o']; // old people flag dead
183
+ migratedCount++;
184
+ }
185
+ }
186
+
187
+ if (migratedCount > 0) {
188
+ this.scheduleSave();
189
+ console.log(`[StateManager] Migrated message flags (p→t, o→p, removed r/o) for ${migratedCount} messages`);
190
+ }
191
+ }
192
+
90
193
  /**
91
194
  * Returns true if value looks like a persona ID (UUID or the special "ei" id).
92
195
  * Display names are free-form strings that won't match UUID format.
@@ -132,16 +235,6 @@ export class StateManager {
132
235
  return result;
133
236
  }
134
237
 
135
- human_trait_upsert(trait: Trait): void {
136
- this.humanState.trait_upsert(trait);
137
- this.scheduleSave();
138
- }
139
-
140
- human_trait_remove(id: string): boolean {
141
- const result = this.humanState.trait_remove(id);
142
- this.scheduleSave();
143
- return result;
144
- }
145
238
 
146
239
  human_topic_upsert(topic: Topic): void {
147
240
  this.humanState.topic_upsert(topic);
@@ -303,11 +396,11 @@ export class StateManager {
303
396
  return result;
304
397
  }
305
398
 
306
- messages_getUnextracted(personaId: string, flag: "f" | "r" | "p" | "o", limit?: number): Message[] {
399
+ messages_getUnextracted(personaId: string, flag: "f" | "t" | "p" | "e", limit?: number): Message[] {
307
400
  return this.personaState.messages_getUnextracted(personaId, flag, limit);
308
401
  }
309
402
 
310
- messages_markExtracted(personaId: string, messageIds: string[], flag: "f" | "r" | "p" | "o"): number {
403
+ messages_markExtracted(personaId: string, messageIds: string[], flag: "f" | "t" | "p" | "e"): number {
311
404
  const result = this.personaState.messages_markExtracted(personaId, messageIds, flag);
312
405
  this.scheduleSave();
313
406
  return result;
@@ -1,16 +1,10 @@
1
- /**
2
- * read_memory builtin tool
3
- *
4
- * Delegates to Processor.searchHumanData() — no external call, runtime: "any".
5
- * The searchHumanData function is injected at construction to avoid circular deps.
6
- */
7
1
  import type { ToolExecutor } from "../types.js";
8
- import type { Fact, Trait, Topic, Person, Quote } from "../../types.js";
2
+ import type { Fact, Topic, Person, Quote } from "../../types.js";
9
3
 
10
4
  type SearchHumanData = (
11
5
  query: string,
12
- options?: { types?: Array<"fact" | "trait" | "topic" | "person" | "quote">; limit?: number }
13
- ) => Promise<{ facts: Fact[]; traits: Trait[]; topics: Topic[]; people: Person[]; quotes: Quote[] }>;
6
+ options?: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean }
7
+ ) => Promise<{ facts: Fact[]; topics: Topic[]; people: Person[]; quotes: Quote[] }>;
14
8
 
15
9
  export function createReadMemoryExecutor(searchHumanData: SearchHumanData): ToolExecutor {
16
10
  return {
@@ -18,28 +12,29 @@ export function createReadMemoryExecutor(searchHumanData: SearchHumanData): Tool
18
12
 
19
13
  async execute(args: Record<string, unknown>): Promise<string> {
20
14
  const query = typeof args.query === "string" ? args.query.trim() : "";
21
- console.log(`[read_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}`);
22
- if (!query) {
15
+ const recent = args.recent === true;
16
+ console.log(`[read_memory] called with query="${query}", types=${JSON.stringify(args.types ?? null)}, limit=${args.limit ?? 10}, recent=${recent}`);
17
+
18
+ if (!query && !recent) {
23
19
  console.warn("[read_memory] missing query argument");
24
- return JSON.stringify({ error: "Missing required argument: query" });
20
+ return JSON.stringify({ error: "Missing required argument: query (or use recent: true)" });
25
21
  }
26
22
 
27
23
  const types = Array.isArray(args.types)
28
24
  ? (args.types.filter(
29
- t => typeof t === "string" && ["fact", "trait", "topic", "person", "quote"].includes(t)
30
- ) as Array<"fact" | "trait" | "topic" | "person" | "quote">)
25
+ t => typeof t === "string" && ["fact", "topic", "person", "quote"].includes(t)
26
+ ) as Array<"fact" | "topic" | "person" | "quote">)
31
27
  : undefined;
32
28
 
33
29
  const limit = typeof args.limit === "number" && args.limit > 0 ? Math.min(args.limit, 20) : 10;
34
30
 
35
- const results = await searchHumanData(query, { types, limit });
31
+ const results = await searchHumanData(query, { types, limit, recent });
36
32
 
37
- const total = results.facts.length + results.traits.length + results.topics.length + results.people.length + results.quotes.length;
38
- console.log(`[read_memory] query="${query}" => ${total} results (facts=${results.facts.length}, traits=${results.traits.length}, topics=${results.topics.length}, people=${results.people.length}, quotes=${results.quotes.length})`);
33
+ const total = results.facts.length + results.topics.length + results.people.length + results.quotes.length;
34
+ console.log(`[read_memory] query="${query}" => ${total} results (facts=${results.facts.length}, topics=${results.topics.length}, people=${results.people.length}, quotes=${results.quotes.length})`);
39
35
 
40
36
  const output: Record<string, unknown[]> = {};
41
37
  if (results.facts.length > 0) output.facts = results.facts.map(f => ({ name: f.name, description: f.description }));
42
- if (results.traits.length > 0) output.traits = results.traits.map(t => ({ name: t.name, description: t.description }));
43
38
  if (results.topics.length > 0) output.topics = results.topics.map(t => ({ name: t.name, description: t.description }));
44
39
  if (results.people.length > 0) output.people = results.people.map(p => ({ name: p.name, relationship: p.relationship, description: p.description }));
45
40
  if (results.quotes.length > 0) output.quotes = results.quotes.map(q => ({ text: q.text, speaker: q.speaker }));