ei-tui 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/package.json +1 -1
  2. package/src/core/constants/built-in-identifier-types.ts +24 -0
  3. package/src/core/embedding-service.ts +24 -1
  4. package/src/core/handlers/dedup.ts +34 -4
  5. package/src/core/handlers/heartbeat.ts +16 -0
  6. package/src/core/handlers/human-extraction.ts +201 -7
  7. package/src/core/handlers/human-matching.ts +71 -22
  8. package/src/core/handlers/index.ts +52 -14
  9. package/src/core/handlers/persona-generation.ts +2 -0
  10. package/src/core/handlers/persona-response.ts +37 -22
  11. package/src/core/handlers/persona-topics.ts +35 -271
  12. package/src/core/handlers/rewrite.ts +3 -0
  13. package/src/core/handlers/rooms.ts +41 -20
  14. package/src/core/handlers/utils.ts +10 -8
  15. package/src/core/heartbeat-manager.ts +60 -2
  16. package/src/core/llm-client.ts +1 -1
  17. package/src/core/message-manager.ts +3 -2
  18. package/src/core/orchestrators/ceremony.ts +54 -144
  19. package/src/core/orchestrators/dedup-phase.ts +0 -199
  20. package/src/core/orchestrators/extraction-chunker.ts +8 -3
  21. package/src/core/orchestrators/human-extraction.ts +37 -85
  22. package/src/core/orchestrators/index.ts +4 -8
  23. package/src/core/orchestrators/person-migration.ts +55 -0
  24. package/src/core/orchestrators/persona-topics.ts +64 -89
  25. package/src/core/orchestrators/room-extraction.ts +34 -0
  26. package/src/core/persona-manager.ts +21 -2
  27. package/src/core/personas/opencode-agent.ts +1 -0
  28. package/src/core/processor.ts +51 -14
  29. package/src/core/prompt-context-builder.ts +38 -5
  30. package/src/core/queue-processor.ts +4 -2
  31. package/src/core/room-manager.ts +6 -7
  32. package/src/core/state/human.ts +6 -0
  33. package/src/core/state/personas.ts +35 -10
  34. package/src/core/state/rooms.ts +21 -0
  35. package/src/core/state-manager.ts +61 -0
  36. package/src/core/types/data-items.ts +12 -0
  37. package/src/core/types/entities.ts +3 -0
  38. package/src/core/types/enums.ts +2 -7
  39. package/src/core/types/llm.ts +2 -0
  40. package/src/core/types/rooms.ts +2 -0
  41. package/src/core/utils/identifier-utils.ts +19 -0
  42. package/src/core/utils/index.ts +2 -1
  43. package/src/core/utils/levenshtein.ts +18 -0
  44. package/src/integrations/claude-code/importer.ts +1 -0
  45. package/src/integrations/cursor/importer.ts +1 -0
  46. package/src/prompts/ceremony/index.ts +1 -0
  47. package/src/prompts/ceremony/person-migration.ts +77 -0
  48. package/src/prompts/ceremony/rewrite.ts +1 -1
  49. package/src/prompts/ceremony/user-dedup.ts +15 -1
  50. package/src/prompts/heartbeat/check.ts +28 -12
  51. package/src/prompts/heartbeat/ei.ts +2 -0
  52. package/src/prompts/heartbeat/types.ts +12 -0
  53. package/src/prompts/human/index.ts +0 -2
  54. package/src/prompts/human/person-scan.ts +58 -14
  55. package/src/prompts/human/person-update.ts +171 -96
  56. package/src/prompts/human/topic-update.ts +1 -1
  57. package/src/prompts/human/types.ts +5 -1
  58. package/src/prompts/index.ts +3 -10
  59. package/src/prompts/message-utils.ts +9 -23
  60. package/src/prompts/persona/index.ts +3 -10
  61. package/src/prompts/persona/topics-rate.ts +95 -0
  62. package/src/prompts/persona/types.ts +8 -48
  63. package/src/prompts/response/index.ts +3 -7
  64. package/src/prompts/response/sections.ts +7 -57
  65. package/src/prompts/room/index.ts +1 -1
  66. package/src/prompts/room/sections.ts +8 -31
  67. package/tui/src/commands/me.tsx +14 -7
  68. package/tui/src/commands/persona.tsx +120 -83
  69. package/tui/src/components/MessageList.tsx +9 -4
  70. package/tui/src/components/RoomMessageList.tsx +10 -5
  71. package/tui/src/context/keyboard.tsx +2 -2
  72. package/tui/src/util/cyp-editor.tsx +13 -8
  73. package/tui/src/util/yaml-context.ts +66 -0
  74. package/tui/src/util/yaml-human.ts +274 -0
  75. package/tui/src/util/yaml-persona.ts +479 -0
  76. package/tui/src/util/yaml-provider.ts +215 -0
  77. package/tui/src/util/yaml-queue.ts +81 -0
  78. package/tui/src/util/yaml-quotes.ts +46 -0
  79. package/tui/src/util/yaml-serializers.ts +9 -1417
  80. package/tui/src/util/yaml-settings.ts +223 -0
  81. package/tui/src/util/yaml-shared.ts +32 -0
  82. package/tui/src/util/yaml-toolkit.ts +55 -0
  83. package/src/prompts/human/person-match.ts +0 -65
  84. package/src/prompts/persona/topics-match.ts +0 -70
  85. package/src/prompts/persona/topics-scan.ts +0 -98
  86. package/src/prompts/persona/topics-update.ts +0 -154
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ei-tui",
3
- "version": "0.5.4",
3
+ "version": "0.6.1",
4
4
  "author": "Flare576",
5
5
  "repository": {
6
6
  "type": "git",
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Built-in identifier types shipped with Ei.
3
+ *
4
+ * Title Case — these are display labels in UI dropdowns, not database columns.
5
+ * Matching is always case-insensitive, so existing lowercase records still work
6
+ * until the user edits them.
7
+ *
8
+ * Any string is valid as an identifier type — users can define their own
9
+ * (e.g. "Slack RNP", "sehimu_thinara"). This list is purely for discoverability.
10
+ */
11
+ export const BUILT_IN_IDENTIFIER_TYPES: readonly string[] = [
12
+ 'Full Name',
13
+ 'First Name',
14
+ 'Nickname',
15
+ 'Email',
16
+ 'GitHub',
17
+ 'Discord',
18
+ 'Roblox',
19
+ 'Reddit',
20
+ 'Twitter',
21
+ 'FF14',
22
+ 'Relationship',
23
+ 'Ei Persona',
24
+ ] as const;
@@ -101,6 +101,29 @@ export async function computeQuoteEmbedding(text: string): Promise<number[] | un
101
101
  }
102
102
  }
103
103
 
104
+ export function getPersonaDescriptionText(persona: {
105
+ display_name: string;
106
+ long_description?: string;
107
+ short_description?: string;
108
+ }): string {
109
+ const desc = persona.long_description ?? persona.short_description;
110
+ return [persona.display_name, desc].filter(Boolean).join(' - ');
111
+ }
112
+
113
+ export async function computePersonaDescriptionEmbedding(persona: {
114
+ display_name: string;
115
+ long_description?: string;
116
+ short_description?: string;
117
+ }): Promise<number[] | undefined> {
118
+ try {
119
+ const service = getEmbeddingService();
120
+ return await service.embed(getPersonaDescriptionText(persona));
121
+ } catch (err) {
122
+ console.warn(`[computePersonaDescriptionEmbedding] Failed for "${persona.display_name}":`, err);
123
+ return undefined;
124
+ }
125
+ }
126
+
104
127
  // =============================================================================
105
128
  // FACTORY - Lazy loading based on environment
106
129
  // =============================================================================
@@ -219,7 +242,7 @@ function createBunService(): EmbeddingService {
219
242
  const cacheDir = process.env.EI_DATA_PATH
220
243
  ? path.join(process.env.EI_DATA_PATH, 'embeddings')
221
244
  : path.join(os.homedir(), '.local', 'share', 'ei', 'embeddings');
222
- embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2, cacheDir });
245
+ embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2, cacheDir, showDownloadProgress: false });
223
246
  return embedder;
224
247
  })();
225
248
 
@@ -1,7 +1,7 @@
1
1
  import { StateManager } from "../state-manager.js";
2
2
  import { LLMResponse } from "../types.js";
3
3
  import type { DedupResult } from "../../prompts/ceremony/types.js";
4
- import type { DataItemType, Fact, Topic, Person, Quote } from "../types/data-items.js";
4
+ import type { DataItemType, Fact, Topic, Person, PersonIdentifier, Quote } from "../types/data-items.js";
5
5
  import { getEmbeddingService } from "../embedding-service.js";
6
6
 
7
7
  /**
@@ -51,7 +51,7 @@ export async function handleDedupCurate(
51
51
 
52
52
  // Pre-compute: for each survivor (replaced_by), union the removed entity's groups.
53
53
  // Must happen before any phase mutates state so we read the original values.
54
- const groupsToMerge = new Map<string, { persona_groups: string[]; interested_personas: string[] }>();
54
+ const groupsToMerge = new Map<string, { persona_groups: string[]; interested_personas: string[]; learned_on?: string; identifiers?: PersonIdentifier[] }>();
55
55
 
56
56
  // Map entity_type to pluralized state property name
57
57
  const pluralMap: Record<string, 'facts' | 'topics' | 'people'> = {
@@ -86,9 +86,25 @@ export async function handleDedupCurate(
86
86
  const removed = entities.find(e => e.id === removal.to_be_removed);
87
87
  if (!removed) continue;
88
88
  const acc = groupsToMerge.get(removal.replaced_by) ?? { persona_groups: [], interested_personas: [] };
89
+ const candidates = [acc.learned_on, removed.learned_on].filter(Boolean) as string[];
90
+
91
+ let mergedIdentifiers: PersonIdentifier[] | undefined;
92
+ if (entity_type === 'person') {
93
+ const removedPerson = removed as Person;
94
+ const accIdentifiers = acc.identifiers ?? [];
95
+ mergedIdentifiers = [...accIdentifiers];
96
+ for (const id of (removedPerson.identifiers ?? [])) {
97
+ if (!mergedIdentifiers.some(existing => existing.value === id.value)) {
98
+ mergedIdentifiers.push(id);
99
+ }
100
+ }
101
+ }
102
+
89
103
  groupsToMerge.set(removal.replaced_by, {
90
104
  persona_groups: [...new Set([...acc.persona_groups, ...(removed.persona_groups ?? [])])],
91
105
  interested_personas: [...new Set([...acc.interested_personas, ...(removed.interested_personas ?? [])])],
106
+ learned_on: candidates.length > 0 ? candidates.sort()[0] : undefined,
107
+ ...(entity_type === 'person' && { identifiers: mergedIdentifiers }),
92
108
  });
93
109
  }
94
110
 
@@ -144,12 +160,16 @@ export async function handleDedupCurate(
144
160
  }
145
161
 
146
162
  const mergedFromRemoved = groupsToMerge.get(update.id);
163
+ const minLearned = mergedFromRemoved?.learned_on
164
+ ? [entity.learned_on, mergedFromRemoved.learned_on].filter(Boolean).sort()[0]
165
+ : entity.learned_on;
147
166
  const updatedEntity = {
148
167
  ...entity,
149
168
  name: update.name ?? entity.name,
150
169
  description: update.description ?? entity.description,
151
170
  sentiment: update.sentiment ?? entity.sentiment,
152
171
  last_updated: new Date().toISOString(),
172
+ ...(minLearned !== undefined && { learned_on: minLearned }),
153
173
  embedding,
154
174
  persona_groups: mergedFromRemoved
155
175
  ? [...new Set([...(entity.persona_groups ?? []), ...mergedFromRemoved.persona_groups])]
@@ -163,6 +183,14 @@ export async function handleDedupCurate(
163
183
  ...(update.exposure_desired !== undefined && { exposure_desired: update.exposure_desired }),
164
184
  ...(update.relationship !== undefined && { relationship: update.relationship }),
165
185
  ...(update.category !== undefined && { category: update.category }),
186
+ ...(entity_type === 'person' && mergedFromRemoved?.identifiers !== undefined && (() => {
187
+ const existingIds = (entity as Person).identifiers ?? [];
188
+ const result: PersonIdentifier[] = [...existingIds];
189
+ for (const id of mergedFromRemoved.identifiers!) {
190
+ if (!result.some(e => e.value === id.value)) result.push(id);
191
+ }
192
+ return { identifiers: result };
193
+ })()),
166
194
  };
167
195
 
168
196
  // Type-safe cast based on entity_type
@@ -216,13 +244,15 @@ export async function handleDedupCurate(
216
244
  // Generate ID for new entity
217
245
  const id = crypto.randomUUID();
218
246
 
247
+ const now = new Date().toISOString();
219
248
  const newEntity = {
220
249
  id,
221
250
  type: entity_type,
222
251
  name: addition.name,
223
252
  description: addition.description,
224
253
  sentiment: addition.sentiment ?? 0.0,
225
- last_updated: new Date().toISOString(),
254
+ last_updated: now,
255
+ learned_on: now,
226
256
  learned_by: "ei",
227
257
  last_changed_by: "ei",
228
258
  embedding,
@@ -233,7 +263,7 @@ export async function handleDedupCurate(
233
263
  exposure_desired: addition.exposure_desired ?? 0.5,
234
264
  last_ei_asked: null
235
265
  }),
236
- ...(entity_type === 'person' && { relationship: addition.relationship ?? 'Unknown' }),
266
+ ...(entity_type === 'person' && { identifiers: [], validated_date: '', relationship: addition.relationship ?? 'Unknown' }),
237
267
  ...(entity_type === 'topic' && { category: addition.category ?? 'Interest' }),
238
268
  };
239
269
 
@@ -27,6 +27,11 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
27
27
  state.persona_update(personaId, { last_heartbeat: now });
28
28
  state.queue_clearPersonaResponses(personaId, LLMNextStep.HandleHeartbeatCheck);
29
29
 
30
+ if (result.mentioned_reflection === true) {
31
+ state.persona_update(personaId, { reflection_last_asked: now });
32
+ console.log(`[HeartbeatCheck ${personaDisplayName}] Persona surfaced identity drift - reflection_last_asked set`);
33
+ }
34
+
30
35
  if (!result.should_respond) {
31
36
  console.log(`[HeartbeatCheck ${personaDisplayName}] Chose not to reach out (should_respond=false)`);
32
37
  return;
@@ -107,4 +112,15 @@ export function handleEiHeartbeat(response: LLMResponse, state: StateManager): v
107
112
  default:
108
113
  console.warn(`[handleEiHeartbeat] Unexpected item type "${found.type}" for id "${result.id}"`);
109
114
  }
115
+
116
+ const newPersonIds = (response.request.data.newPersonIds ?? []) as string[];
117
+ if (newPersonIds.length > 0) {
118
+ const human = state.getHuman();
119
+ for (const personId of newPersonIds) {
120
+ const person = human.people.find(p => p.id === personId);
121
+ if (person) {
122
+ state.human_person_upsert({ ...person, validated_date: now });
123
+ }
124
+ }
125
+ }
110
126
  }
@@ -1,15 +1,89 @@
1
- import type { LLMResponse, Fact } from "../types.js";
1
+ import type { LLMResponse, Fact, Person } from "../types.js";
2
+ import type { PersonIdentifier } from "../types/data-items.js";
2
3
  import type { StateManager } from "../state-manager.js";
3
4
  import type {
4
5
  FactFindResult,
5
6
  TopicScanResult,
6
7
  PersonScanResult,
7
8
  TopicScanCandidate,
9
+ ItemMatchResult,
8
10
  } from "../../prompts/human/types.js";
9
- import { queueTopicMatch, queuePersonMatch, type ExtractionContext } from "../orchestrators/index.js";
10
- import { markMessagesExtracted } from "./utils.js";
11
+ import { queueTopicMatch, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
12
+ import { markMessagesExtracted, resolveMessageWindow } from "./utils.js";
11
13
  import { BUILT_IN_FACT_NAMES } from "../constants/built-in-facts.js";
12
- import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
14
+ import { getEmbeddingService, getItemEmbeddingText, cosineSimilarity, getPersonEmbeddingText } from "../embedding-service.js";
15
+ import { levenshtein, normalizeForMatch } from "../utils/levenshtein.js";
16
+
17
+ const MULTI_MATCH_SIMILARITY_THRESHOLD = 0.75;
18
+ const ZERO_MATCH_COSINE_THRESHOLD = 0.80;
19
+
20
+ // Relationships where a person typically has exactly one instance.
21
+ // Only these fire the "sole relationship" uniqueness shortcut when the
22
+ // existing record already has a real name (non-Unknown records in non-singleton
23
+ // relationships fall through to cosine so we don't merge David into Sisyphus).
24
+ const SINGLETON_RELATIONSHIPS = new Set([
25
+ 'self',
26
+ 'husband', 'wife', 'spouse',
27
+ 'father', 'mother',
28
+ ]);
29
+
30
+ function matchPersonCandidate(
31
+ candidateName: string,
32
+ candidateIdentifiers: PersonIdentifier[],
33
+ people: Person[]
34
+ ): Person[] {
35
+ const normName = normalizeForMatch(candidateName);
36
+ const matched = new Set<Person>();
37
+
38
+ // Step 1: Exact match on any identifier value (type-agnostic)
39
+ for (const person of people) {
40
+ const allValues = [
41
+ ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
42
+ normalizeForMatch(person.name),
43
+ ];
44
+ if (allValues.includes(normName)) matched.add(person);
45
+ }
46
+ // Also check scan-extracted identifiers against existing identifier values
47
+ for (const scanId of candidateIdentifiers) {
48
+ const normVal = normalizeForMatch(scanId.value);
49
+ for (const person of people) {
50
+ if ((person.identifiers ?? []).some(i => normalizeForMatch(i.value) === normVal)) {
51
+ matched.add(person);
52
+ }
53
+ }
54
+ }
55
+
56
+ if (matched.size > 0) return [...matched];
57
+
58
+ // Step 2: Fuzzy match — skip for short names (< 6 chars): "mike"↔"jake" = 2 edits, false positive.
59
+ if (normName.length >= 6) {
60
+ const threshold = normName.length < 10 ? 1 : 2;
61
+ for (const person of people) {
62
+ const allValues = [
63
+ ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
64
+ normalizeForMatch(person.name),
65
+ ];
66
+ if (allValues.some(v => levenshtein(normName, v) <= threshold)) matched.add(person);
67
+ }
68
+ }
69
+
70
+ if (matched.size > 0) return [...matched];
71
+
72
+ // Step 2.5: First-name match — "Lucas Jeremy Scherer" should find "Lucas".
73
+ // Only fires when first word is >= 4 chars to avoid short-name collisions.
74
+ const candidateFirstWord = normName.split(/\s+/)[0];
75
+ if (candidateFirstWord.length >= 4) {
76
+ for (const person of people) {
77
+ const allNames = [
78
+ normalizeForMatch(person.name),
79
+ ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
80
+ ];
81
+ if (allNames.some(n => n.split(/\s+/)[0] === candidateFirstWord)) matched.add(person);
82
+ }
83
+ }
84
+
85
+ return [...matched];
86
+ }
13
87
 
14
88
  export async function handleFactFind(response: LLMResponse, state: StateManager): Promise<void> {
15
89
  const result = response.parsed as FactFindResult | undefined;
@@ -117,11 +191,131 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
117
191
  const context = response.request.data as unknown as ExtractionContext;
118
192
  if (!context?.personaId) return;
119
193
 
120
- const extractionModel = (response.request.data as Record<string, unknown>).extraction_model as string | undefined;
194
+ const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
195
+ const human = state.getHuman();
196
+
121
197
  for (const candidate of result.people) {
122
- await queuePersonMatch(candidate, context, state, extractionModel);
198
+ const candidateIdentifiers: PersonIdentifier[] = (candidate.identifiers ?? []).map(i => ({
199
+ type: i.type,
200
+ value: i.value,
201
+ ...(i.is_primary ? { is_primary: i.is_primary } : {}),
202
+ }));
203
+
204
+ const matches = matchPersonCandidate(candidate.name, candidateIdentifiers, human.people);
205
+
206
+ let matchedPerson: Person | null = null;
207
+
208
+ if (matches.length === 1) {
209
+ matchedPerson = matches[0];
210
+ } else if (matches.length > 1) {
211
+ try {
212
+ const embeddingService = getEmbeddingService();
213
+ const candidateText = getPersonEmbeddingText({
214
+ name: candidate.name,
215
+ relationship: candidate.relationship,
216
+ description: candidate.description,
217
+ });
218
+ const candidateVector = await embeddingService.embed(candidateText);
219
+ let bestSimilarity = MULTI_MATCH_SIMILARITY_THRESHOLD;
220
+ for (const person of matches) {
221
+ if (person.embedding) {
222
+ const sim = cosineSimilarity(person.embedding, candidateVector);
223
+ if (sim > bestSimilarity) {
224
+ bestSimilarity = sim;
225
+ matchedPerson = person;
226
+ }
227
+ }
228
+ }
229
+ if (!matchedPerson) {
230
+ console.log(`[handleHumanPersonScan] Multi-match for "${candidate.name}" (${matches.length} hits) — no embedding above threshold, creating new record`);
231
+ }
232
+ } catch (err) {
233
+ console.warn(`[handleHumanPersonScan] Multi-match embedding failed for "${candidate.name}", using first match:`, err);
234
+ matchedPerson = matches[0];
235
+ }
236
+ } else {
237
+ // Step 3: relationship filter → uniqueness match or cosine on the relevant subset.
238
+ // Filter first (O(N)), then cosine only on the filtered set (O(K) where K <= N).
239
+ const normRel = candidate.relationship?.toLowerCase();
240
+ const sameRel = normRel && normRel !== 'unknown'
241
+ ? human.people.filter(p => p.relationship?.toLowerCase() === normRel)
242
+ : [];
243
+
244
+ if (sameRel.length === 1) {
245
+ const existing = sameRel[0];
246
+ const normExistingName = normalizeForMatch(existing.name);
247
+ const isUnknownPlaceholder = normExistingName === 'unknown' || normExistingName === normRel;
248
+ const isSingleton = SINGLETON_RELATIONSHIPS.has(normRel!);
249
+ if (isUnknownPlaceholder || isSingleton) {
250
+ matchedPerson = existing;
251
+ const reason = isUnknownPlaceholder ? 'unnamed placeholder' : 'singleton relationship';
252
+ console.log(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
253
+ }
254
+ } else {
255
+ // N>1 same relationship → cosine within that subset.
256
+ // N=0 (unknown relationship or no stored records) → cosine against all people.
257
+ const searchPool = sameRel.length > 1
258
+ ? sameRel.filter(p => p.embedding && p.embedding.length > 0)
259
+ : human.people.filter(p => p.embedding && p.embedding.length > 0);
260
+
261
+ const poolLabel = sameRel.length > 1
262
+ ? `${sameRel.length} ${candidate.relationship} records`
263
+ : `all ${human.people.length} people`;
264
+
265
+ if (searchPool.length > 0) {
266
+ console.log(`[handleHumanPersonScan] "${candidate.name}": cosine against ${searchPool.length} embedded (${poolLabel})`);
267
+ try {
268
+ const embeddingService = getEmbeddingService();
269
+ const candidateText = getPersonEmbeddingText({
270
+ name: candidate.name,
271
+ relationship: candidate.relationship,
272
+ description: candidate.description,
273
+ });
274
+ const candidateVector = await embeddingService.embed(candidateText);
275
+ const scores: Array<{ name: string; sim: number }> = [];
276
+ let bestSimilarity = ZERO_MATCH_COSINE_THRESHOLD;
277
+ for (const person of searchPool) {
278
+ const sim = cosineSimilarity(person.embedding!, candidateVector);
279
+ scores.push({ name: person.name, sim });
280
+ if (sim > bestSimilarity) {
281
+ bestSimilarity = sim;
282
+ matchedPerson = person;
283
+ }
284
+ }
285
+ const top3 = scores.sort((a, b) => b.sim - a.sim).slice(0, 3).map(s => `"${s.name}"=${s.sim.toFixed(3)}`).join(', ');
286
+ if (matchedPerson) {
287
+ console.log(`[handleHumanPersonScan] Cosine matched "${candidate.name}" → "${matchedPerson.name}" (${bestSimilarity.toFixed(3)}) | top3: ${top3}`);
288
+ } else {
289
+ console.log(`[handleHumanPersonScan] Cosine: no match above ${ZERO_MATCH_COSINE_THRESHOLD} for "${candidate.name}" | top3: ${top3}`);
290
+ }
291
+ } catch (err) {
292
+ console.warn(`[handleHumanPersonScan] Cosine failed for "${candidate.name}":`, err);
293
+ }
294
+ } else {
295
+ console.log(`[handleHumanPersonScan] "${candidate.name}": no embedded people in pool (${poolLabel}) — new person`);
296
+ }
297
+ }
298
+ }
299
+
300
+ const matchResult: ItemMatchResult = { matched_guid: matchedPerson?.id ?? null };
301
+ queuePersonUpdate(matchResult, {
302
+ ...context,
303
+ messages_context,
304
+ messages_analyze,
305
+ candidateName: candidate.name,
306
+ candidateDescription: candidate.description,
307
+ candidateRelationship: candidate.relationship,
308
+ candidateIdentifiers,
309
+ }, state);
310
+
311
+ const matched = matchedPerson
312
+ ? `matched "${matchedPerson.name}"`
313
+ : matches.length > 1
314
+ ? `multi-match ambiguous (${matches.length} hits) — new record`
315
+ : "no match (new person)";
316
+ console.log(`[handleHumanPersonScan] person "${candidate.name}": ${matched}`);
123
317
  }
124
- console.log(`[handleHumanPersonScan] Queued ${result.people.length} person(s) for matching`);
318
+ console.log(`[handleHumanPersonScan] Processed ${result.people.length} person(s)`);
125
319
  }
126
320
 
127
321
  export async function handleEventScan(response: LLMResponse, state: StateManager): Promise<void> {
@@ -5,6 +5,7 @@ import {
5
5
  type Person,
6
6
  type Quote,
7
7
  } from "../types.js";
8
+ import type { PersonIdentifier } from "../types/data-items.js";
8
9
  import type { StateManager } from "../state-manager.js";
9
10
  import type { ItemMatchResult, ExposureImpact, TopicUpdateResult, PersonUpdateResult } from "../../prompts/human/types.js";
10
11
  import { queueTopicUpdate, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
@@ -13,12 +14,12 @@ import { calculateExposureCurrent } from "../utils/exposure.js";
13
14
 
14
15
 
15
16
  import { resolveMessageWindow, getMessageText, normalizeRoomMessages } from "./utils.js";
17
+ import { sanitizeEiPersonaIdentifiers } from "../utils/identifier-utils.js";
16
18
 
17
19
  export function handleTopicMatch(response: LLMResponse, state: StateManager): void {
18
20
  const result = response.parsed as ItemMatchResult | undefined;
19
21
  if (!result) {
20
- console.error("[handleTopicMatch] No parsed result");
21
- return;
22
+ throw new Error("[handleTopicMatch] No parsed result");
22
23
  }
23
24
 
24
25
  const personaId = response.request.data.personaId as string;
@@ -64,8 +65,7 @@ export function handleTopicMatch(response: LLMResponse, state: StateManager): vo
64
65
  export function handlePersonMatch(response: LLMResponse, state: StateManager): void {
65
66
  const result = response.parsed as ItemMatchResult | undefined;
66
67
  if (!result) {
67
- console.error("[handlePersonMatch] No parsed result");
68
- return;
68
+ throw new Error("[handlePersonMatch] No parsed result");
69
69
  }
70
70
 
71
71
  const personaId = response.request.data.personaId as string;
@@ -123,11 +123,6 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
123
123
  const roomId = response.request.data.roomId as string | undefined;
124
124
  const candidateCategory = response.request.data.candidateCategory as string | undefined;
125
125
 
126
- if (!result.name || !result.description || result.sentiment === undefined) {
127
- console.error("[handleTopicUpdate] Missing required fields in result");
128
- return;
129
- }
130
-
131
126
  const personaIds = personaId.split("|").filter(Boolean);
132
127
  const primaryId = personaIds[0] ?? personaId;
133
128
 
@@ -148,14 +143,21 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
148
143
 
149
144
  const existingTopic = isNewItem ? undefined : human.topics.find(t => t.id === existingItemId);
150
145
 
146
+ const resolvedName = result.name || existingTopic?.name;
147
+ const resolvedDescription = typeof result.description === 'string' ? result.description : existingTopic?.description;
148
+
149
+ if (!resolvedName || !resolvedDescription || result.sentiment === undefined) {
150
+ throw new Error(`[handleTopicUpdate] Missing required fields: name=${resolvedName}, description=${!!resolvedDescription}, sentiment=${result.sentiment}`);
151
+ }
152
+
151
153
  let embedding: number[] | undefined;
152
154
  try {
153
155
  const embeddingService = getEmbeddingService();
154
156
  const category = result.category ?? candidateCategory ?? existingTopic?.category;
155
- const text = getTopicEmbeddingText({ name: result.name, category, description: result.description });
157
+ const text = getTopicEmbeddingText({ name: resolvedName, category, description: resolvedDescription });
156
158
  embedding = await embeddingService.embed(text);
157
159
  } catch (err) {
158
- console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${result.name}":`, err);
160
+ console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${resolvedName}":`, err);
159
161
  }
160
162
 
161
163
  const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
@@ -168,13 +170,14 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
168
170
 
169
171
  const topic: Topic = {
170
172
  id: itemId,
171
- name: result.name,
172
- description: result.description,
173
+ name: resolvedName,
174
+ description: resolvedDescription,
173
175
  sentiment: result.sentiment,
174
176
  category: result.category ?? candidateCategory ?? existingTopic?.category,
175
177
  exposure_current: calculateExposureCurrent(exposureImpact, existingTopic?.exposure_current ?? 0),
176
178
  exposure_desired: result.exposure_desired ?? 0.5,
177
179
  last_updated: now,
180
+ learned_on: isNewItem ? now : existingTopic?.learned_on,
178
181
  last_mentioned: now,
179
182
  learned_by: isNewItem ? primaryId : existingTopic?.learned_by,
180
183
  last_changed_by: primaryId,
@@ -189,11 +192,15 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
189
192
  : state.messages_get(personaId);
190
193
  await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
191
194
 
192
- console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${result.name}"`);
195
+ console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${resolvedName}"`);
193
196
  }
194
197
 
195
198
  export async function handlePersonUpdate(response: LLMResponse, state: StateManager): Promise<void> {
196
- const result = response.parsed as (PersonUpdateResult & { quotes?: Array<{ text: string; reason: string }> }) | undefined;
199
+ const result = response.parsed as (PersonUpdateResult & {
200
+ identifiers?: PersonIdentifier[];
201
+ identifiers_to_add?: PersonIdentifier[];
202
+ quotes?: Array<{ text: string; reason: string }>;
203
+ }) | undefined;
197
204
 
198
205
  if (!result || Object.keys(result).length === 0) {
199
206
  console.log("[handlePersonUpdate] No changes needed (empty result)");
@@ -206,12 +213,13 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
206
213
  const personaDisplayName = response.request.data.personaDisplayName as string;
207
214
  const roomId = response.request.data.roomId as string | undefined;
208
215
  const candidateRelationship = response.request.data.candidateRelationship as string | undefined;
216
+ const candidateIdentifiers = (response.request.data.candidateIdentifiers ?? []) as PersonIdentifier[];
209
217
 
210
- if (!result.name || !result.description || result.sentiment === undefined) {
211
- console.error("[handlePersonUpdate] Missing required fields in result");
212
- return;
218
+ if (!result.description || result.sentiment === undefined) {
219
+ throw new Error(`[handlePersonUpdate] Missing required fields: description=${!!result.description}, sentiment=${result.sentiment}`);
213
220
  }
214
221
 
222
+ const candidateName = response.request.data.candidateName as string;
215
223
  const personaIds = personaId.split("|").filter(Boolean);
216
224
  const primaryId = personaIds[0] ?? personaId;
217
225
 
@@ -236,10 +244,10 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
236
244
  try {
237
245
  const embeddingService = getEmbeddingService();
238
246
  const relationship = result.relationship ?? candidateRelationship ?? existingPerson?.relationship;
239
- const text = getPersonEmbeddingText({ name: result.name, relationship, description: result.description });
247
+ const text = getPersonEmbeddingText({ name: candidateName, relationship, description: result.description });
240
248
  embedding = await embeddingService.embed(text);
241
249
  } catch (err) {
242
- console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${result.name}":`, err);
250
+ console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${candidateName}":`, err);
243
251
  }
244
252
 
245
253
  const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
@@ -250,15 +258,51 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
250
258
  ? (allPersonaGroups.length > 0 ? allPersonaGroups : existingPerson?.persona_groups)
251
259
  : [...new Set([...(existingPerson?.persona_groups ?? []), ...allPersonaGroups])];
252
260
 
261
+ let resolvedIdentifiers: PersonIdentifier[];
262
+ if (isNewItem) {
263
+ const llmIdentifiers: PersonIdentifier[] = sanitizeEiPersonaIdentifiers(
264
+ (result.identifiers ?? []).map(i => ({
265
+ type: i.type,
266
+ value: i.value,
267
+ ...(i.is_primary ? { is_primary: i.is_primary } : {}),
268
+ })),
269
+ state
270
+ );
271
+ const allCandidateIds = [...llmIdentifiers, ...candidateIdentifiers];
272
+ if (allCandidateIds.length === 0) {
273
+ const hasSpace = candidateName.includes(' ');
274
+ allCandidateIds.push({ type: hasSpace ? "full_name" : "nickname", value: candidateName, is_primary: true });
275
+ }
276
+ const deduped: PersonIdentifier[] = [];
277
+ for (const id of allCandidateIds) {
278
+ if (!deduped.some(e => e.value === id.value)) {
279
+ deduped.push(id);
280
+ }
281
+ }
282
+ resolvedIdentifiers = deduped;
283
+ } else {
284
+ const base = [...(existingPerson?.identifiers ?? [])];
285
+ const sanitizedToAdd = sanitizeEiPersonaIdentifiers(result.identifiers_to_add ?? [], state);
286
+ for (const id of sanitizedToAdd) {
287
+ if (!base.some(e => e.value === id.value)) {
288
+ base.push({ type: id.type, value: id.value, ...(id.is_primary ? { is_primary: id.is_primary } : {}) });
289
+ }
290
+ }
291
+ resolvedIdentifiers = base;
292
+ }
293
+
253
294
  const person: Person = {
254
295
  id: itemId,
255
- name: result.name,
296
+ name: candidateName,
256
297
  description: result.description,
257
298
  sentiment: result.sentiment,
258
299
  relationship: result.relationship ?? candidateRelationship ?? existingPerson?.relationship ?? "Unknown",
259
300
  exposure_current: calculateExposureCurrent(exposureImpact, existingPerson?.exposure_current ?? 0),
260
301
  exposure_desired: result.exposure_desired ?? 0.5,
302
+ identifiers: resolvedIdentifiers,
303
+ validated_date: isNewItem ? '' : (existingPerson?.validated_date ?? ''),
261
304
  last_updated: now,
305
+ learned_on: isNewItem ? now : existingPerson?.learned_on,
262
306
  last_mentioned: now,
263
307
  learned_by: isNewItem ? primaryId : existingPerson?.learned_by,
264
308
  last_changed_by: primaryId,
@@ -273,9 +317,14 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
273
317
  : state.messages_get(personaId);
274
318
  await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
275
319
 
276
- console.log(`[handlePersonUpdate] ${isNewItem ? "Created" : "Updated"} person "${result.name}"`);
320
+ const primaryValue = resolvedIdentifiers.find(i => i.is_primary)?.value ?? candidateName;
321
+ const resolvedName = (!primaryValue || primaryValue.toLowerCase() === 'unknown')
322
+ ? (result.relationship ?? candidateRelationship ?? '(unknown)')
323
+ : primaryValue;
324
+ console.log(`[handlePersonUpdate] ${isNewItem ? "Created" : "Updated"} person "${resolvedName}"`);
277
325
  }
278
326
 
327
+
279
328
  function normalizeText(text: string): string {
280
329
  return text
281
330
  .replace(/[\u201C\u201D]/g, '"') // curly double quotes