ei-tui 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/constants/built-in-identifier-types.ts +24 -0
- package/src/core/embedding-service.ts +24 -1
- package/src/core/handlers/dedup.ts +34 -4
- package/src/core/handlers/heartbeat.ts +16 -0
- package/src/core/handlers/human-extraction.ts +201 -7
- package/src/core/handlers/human-matching.ts +71 -22
- package/src/core/handlers/index.ts +52 -14
- package/src/core/handlers/persona-generation.ts +2 -0
- package/src/core/handlers/persona-response.ts +37 -22
- package/src/core/handlers/persona-topics.ts +35 -271
- package/src/core/handlers/rewrite.ts +3 -0
- package/src/core/handlers/rooms.ts +41 -20
- package/src/core/handlers/utils.ts +10 -8
- package/src/core/heartbeat-manager.ts +60 -2
- package/src/core/llm-client.ts +1 -1
- package/src/core/message-manager.ts +3 -2
- package/src/core/orchestrators/ceremony.ts +54 -144
- package/src/core/orchestrators/dedup-phase.ts +0 -199
- package/src/core/orchestrators/extraction-chunker.ts +8 -3
- package/src/core/orchestrators/human-extraction.ts +37 -85
- package/src/core/orchestrators/index.ts +4 -8
- package/src/core/orchestrators/person-migration.ts +55 -0
- package/src/core/orchestrators/persona-topics.ts +64 -89
- package/src/core/orchestrators/room-extraction.ts +34 -0
- package/src/core/persona-manager.ts +21 -2
- package/src/core/personas/opencode-agent.ts +1 -0
- package/src/core/processor.ts +51 -14
- package/src/core/prompt-context-builder.ts +38 -5
- package/src/core/queue-processor.ts +4 -2
- package/src/core/room-manager.ts +6 -7
- package/src/core/state/human.ts +6 -0
- package/src/core/state/personas.ts +35 -10
- package/src/core/state/rooms.ts +21 -0
- package/src/core/state-manager.ts +61 -0
- package/src/core/types/data-items.ts +12 -0
- package/src/core/types/entities.ts +3 -0
- package/src/core/types/enums.ts +2 -7
- package/src/core/types/llm.ts +2 -0
- package/src/core/types/rooms.ts +2 -0
- package/src/core/utils/identifier-utils.ts +19 -0
- package/src/core/utils/index.ts +2 -1
- package/src/core/utils/levenshtein.ts +18 -0
- package/src/integrations/claude-code/importer.ts +1 -0
- package/src/integrations/cursor/importer.ts +1 -0
- package/src/prompts/ceremony/index.ts +1 -0
- package/src/prompts/ceremony/person-migration.ts +77 -0
- package/src/prompts/ceremony/rewrite.ts +1 -1
- package/src/prompts/ceremony/user-dedup.ts +15 -1
- package/src/prompts/heartbeat/check.ts +28 -12
- package/src/prompts/heartbeat/ei.ts +2 -0
- package/src/prompts/heartbeat/types.ts +12 -0
- package/src/prompts/human/index.ts +0 -2
- package/src/prompts/human/person-scan.ts +58 -14
- package/src/prompts/human/person-update.ts +171 -96
- package/src/prompts/human/topic-update.ts +1 -1
- package/src/prompts/human/types.ts +5 -1
- package/src/prompts/index.ts +3 -10
- package/src/prompts/message-utils.ts +9 -23
- package/src/prompts/persona/index.ts +3 -10
- package/src/prompts/persona/topics-rate.ts +95 -0
- package/src/prompts/persona/types.ts +8 -48
- package/src/prompts/response/index.ts +3 -7
- package/src/prompts/response/sections.ts +7 -57
- package/src/prompts/room/index.ts +1 -1
- package/src/prompts/room/sections.ts +8 -31
- package/tui/src/commands/me.tsx +14 -7
- package/tui/src/commands/persona.tsx +120 -83
- package/tui/src/components/MessageList.tsx +9 -4
- package/tui/src/components/RoomMessageList.tsx +10 -5
- package/tui/src/context/keyboard.tsx +2 -2
- package/tui/src/util/cyp-editor.tsx +13 -8
- package/tui/src/util/yaml-context.ts +66 -0
- package/tui/src/util/yaml-human.ts +274 -0
- package/tui/src/util/yaml-persona.ts +479 -0
- package/tui/src/util/yaml-provider.ts +215 -0
- package/tui/src/util/yaml-queue.ts +81 -0
- package/tui/src/util/yaml-quotes.ts +46 -0
- package/tui/src/util/yaml-serializers.ts +9 -1417
- package/tui/src/util/yaml-settings.ts +223 -0
- package/tui/src/util/yaml-shared.ts +32 -0
- package/tui/src/util/yaml-toolkit.ts +55 -0
- package/src/prompts/human/person-match.ts +0 -65
- package/src/prompts/persona/topics-match.ts +0 -70
- package/src/prompts/persona/topics-scan.ts +0 -98
- package/src/prompts/persona/topics-update.ts +0 -154
package/package.json
CHANGED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in identifier types shipped with Ei.
|
|
3
|
+
*
|
|
4
|
+
* Title Case — these are display labels in UI dropdowns, not database columns.
|
|
5
|
+
* Matching is always case-insensitive, so existing lowercase records still work
|
|
6
|
+
* until the user edits them.
|
|
7
|
+
*
|
|
8
|
+
* Any string is valid as an identifier type — users can define their own
|
|
9
|
+
* (e.g. "Slack RNP", "sehimu_thinara"). This list is purely for discoverability.
|
|
10
|
+
*/
|
|
11
|
+
export const BUILT_IN_IDENTIFIER_TYPES: readonly string[] = [
|
|
12
|
+
'Full Name',
|
|
13
|
+
'First Name',
|
|
14
|
+
'Nickname',
|
|
15
|
+
'Email',
|
|
16
|
+
'GitHub',
|
|
17
|
+
'Discord',
|
|
18
|
+
'Roblox',
|
|
19
|
+
'Reddit',
|
|
20
|
+
'Twitter',
|
|
21
|
+
'FF14',
|
|
22
|
+
'Relationship',
|
|
23
|
+
'Ei Persona',
|
|
24
|
+
] as const;
|
|
@@ -101,6 +101,29 @@ export async function computeQuoteEmbedding(text: string): Promise<number[] | un
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
export function getPersonaDescriptionText(persona: {
|
|
105
|
+
display_name: string;
|
|
106
|
+
long_description?: string;
|
|
107
|
+
short_description?: string;
|
|
108
|
+
}): string {
|
|
109
|
+
const desc = persona.long_description ?? persona.short_description;
|
|
110
|
+
return [persona.display_name, desc].filter(Boolean).join(' - ');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function computePersonaDescriptionEmbedding(persona: {
|
|
114
|
+
display_name: string;
|
|
115
|
+
long_description?: string;
|
|
116
|
+
short_description?: string;
|
|
117
|
+
}): Promise<number[] | undefined> {
|
|
118
|
+
try {
|
|
119
|
+
const service = getEmbeddingService();
|
|
120
|
+
return await service.embed(getPersonaDescriptionText(persona));
|
|
121
|
+
} catch (err) {
|
|
122
|
+
console.warn(`[computePersonaDescriptionEmbedding] Failed for "${persona.display_name}":`, err);
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
104
127
|
// =============================================================================
|
|
105
128
|
// FACTORY - Lazy loading based on environment
|
|
106
129
|
// =============================================================================
|
|
@@ -219,7 +242,7 @@ function createBunService(): EmbeddingService {
|
|
|
219
242
|
const cacheDir = process.env.EI_DATA_PATH
|
|
220
243
|
? path.join(process.env.EI_DATA_PATH, 'embeddings')
|
|
221
244
|
: path.join(os.homedir(), '.local', 'share', 'ei', 'embeddings');
|
|
222
|
-
embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2, cacheDir });
|
|
245
|
+
embedder = await mod.FlagEmbedding.init({ model: mod.EmbeddingModel.AllMiniLML6V2, cacheDir, showDownloadProgress: false });
|
|
223
246
|
return embedder;
|
|
224
247
|
})();
|
|
225
248
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { StateManager } from "../state-manager.js";
|
|
2
2
|
import { LLMResponse } from "../types.js";
|
|
3
3
|
import type { DedupResult } from "../../prompts/ceremony/types.js";
|
|
4
|
-
import type { DataItemType, Fact, Topic, Person, Quote } from "../types/data-items.js";
|
|
4
|
+
import type { DataItemType, Fact, Topic, Person, PersonIdentifier, Quote } from "../types/data-items.js";
|
|
5
5
|
import { getEmbeddingService } from "../embedding-service.js";
|
|
6
6
|
|
|
7
7
|
/**
|
|
@@ -51,7 +51,7 @@ export async function handleDedupCurate(
|
|
|
51
51
|
|
|
52
52
|
// Pre-compute: for each survivor (replaced_by), union the removed entity's groups.
|
|
53
53
|
// Must happen before any phase mutates state so we read the original values.
|
|
54
|
-
const groupsToMerge = new Map<string, { persona_groups: string[]; interested_personas: string[] }>();
|
|
54
|
+
const groupsToMerge = new Map<string, { persona_groups: string[]; interested_personas: string[]; learned_on?: string; identifiers?: PersonIdentifier[] }>();
|
|
55
55
|
|
|
56
56
|
// Map entity_type to pluralized state property name
|
|
57
57
|
const pluralMap: Record<string, 'facts' | 'topics' | 'people'> = {
|
|
@@ -86,9 +86,25 @@ export async function handleDedupCurate(
|
|
|
86
86
|
const removed = entities.find(e => e.id === removal.to_be_removed);
|
|
87
87
|
if (!removed) continue;
|
|
88
88
|
const acc = groupsToMerge.get(removal.replaced_by) ?? { persona_groups: [], interested_personas: [] };
|
|
89
|
+
const candidates = [acc.learned_on, removed.learned_on].filter(Boolean) as string[];
|
|
90
|
+
|
|
91
|
+
let mergedIdentifiers: PersonIdentifier[] | undefined;
|
|
92
|
+
if (entity_type === 'person') {
|
|
93
|
+
const removedPerson = removed as Person;
|
|
94
|
+
const accIdentifiers = acc.identifiers ?? [];
|
|
95
|
+
mergedIdentifiers = [...accIdentifiers];
|
|
96
|
+
for (const id of (removedPerson.identifiers ?? [])) {
|
|
97
|
+
if (!mergedIdentifiers.some(existing => existing.value === id.value)) {
|
|
98
|
+
mergedIdentifiers.push(id);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
89
103
|
groupsToMerge.set(removal.replaced_by, {
|
|
90
104
|
persona_groups: [...new Set([...acc.persona_groups, ...(removed.persona_groups ?? [])])],
|
|
91
105
|
interested_personas: [...new Set([...acc.interested_personas, ...(removed.interested_personas ?? [])])],
|
|
106
|
+
learned_on: candidates.length > 0 ? candidates.sort()[0] : undefined,
|
|
107
|
+
...(entity_type === 'person' && { identifiers: mergedIdentifiers }),
|
|
92
108
|
});
|
|
93
109
|
}
|
|
94
110
|
|
|
@@ -144,12 +160,16 @@ export async function handleDedupCurate(
|
|
|
144
160
|
}
|
|
145
161
|
|
|
146
162
|
const mergedFromRemoved = groupsToMerge.get(update.id);
|
|
163
|
+
const minLearned = mergedFromRemoved?.learned_on
|
|
164
|
+
? [entity.learned_on, mergedFromRemoved.learned_on].filter(Boolean).sort()[0]
|
|
165
|
+
: entity.learned_on;
|
|
147
166
|
const updatedEntity = {
|
|
148
167
|
...entity,
|
|
149
168
|
name: update.name ?? entity.name,
|
|
150
169
|
description: update.description ?? entity.description,
|
|
151
170
|
sentiment: update.sentiment ?? entity.sentiment,
|
|
152
171
|
last_updated: new Date().toISOString(),
|
|
172
|
+
...(minLearned !== undefined && { learned_on: minLearned }),
|
|
153
173
|
embedding,
|
|
154
174
|
persona_groups: mergedFromRemoved
|
|
155
175
|
? [...new Set([...(entity.persona_groups ?? []), ...mergedFromRemoved.persona_groups])]
|
|
@@ -163,6 +183,14 @@ export async function handleDedupCurate(
|
|
|
163
183
|
...(update.exposure_desired !== undefined && { exposure_desired: update.exposure_desired }),
|
|
164
184
|
...(update.relationship !== undefined && { relationship: update.relationship }),
|
|
165
185
|
...(update.category !== undefined && { category: update.category }),
|
|
186
|
+
...(entity_type === 'person' && mergedFromRemoved?.identifiers !== undefined && (() => {
|
|
187
|
+
const existingIds = (entity as Person).identifiers ?? [];
|
|
188
|
+
const result: PersonIdentifier[] = [...existingIds];
|
|
189
|
+
for (const id of mergedFromRemoved.identifiers!) {
|
|
190
|
+
if (!result.some(e => e.value === id.value)) result.push(id);
|
|
191
|
+
}
|
|
192
|
+
return { identifiers: result };
|
|
193
|
+
})()),
|
|
166
194
|
};
|
|
167
195
|
|
|
168
196
|
// Type-safe cast based on entity_type
|
|
@@ -216,13 +244,15 @@ export async function handleDedupCurate(
|
|
|
216
244
|
// Generate ID for new entity
|
|
217
245
|
const id = crypto.randomUUID();
|
|
218
246
|
|
|
247
|
+
const now = new Date().toISOString();
|
|
219
248
|
const newEntity = {
|
|
220
249
|
id,
|
|
221
250
|
type: entity_type,
|
|
222
251
|
name: addition.name,
|
|
223
252
|
description: addition.description,
|
|
224
253
|
sentiment: addition.sentiment ?? 0.0,
|
|
225
|
-
last_updated:
|
|
254
|
+
last_updated: now,
|
|
255
|
+
learned_on: now,
|
|
226
256
|
learned_by: "ei",
|
|
227
257
|
last_changed_by: "ei",
|
|
228
258
|
embedding,
|
|
@@ -233,7 +263,7 @@ export async function handleDedupCurate(
|
|
|
233
263
|
exposure_desired: addition.exposure_desired ?? 0.5,
|
|
234
264
|
last_ei_asked: null
|
|
235
265
|
}),
|
|
236
|
-
...(entity_type === 'person' && { relationship: addition.relationship ?? 'Unknown' }),
|
|
266
|
+
...(entity_type === 'person' && { identifiers: [], validated_date: '', relationship: addition.relationship ?? 'Unknown' }),
|
|
237
267
|
...(entity_type === 'topic' && { category: addition.category ?? 'Interest' }),
|
|
238
268
|
};
|
|
239
269
|
|
|
@@ -27,6 +27,11 @@ export function handleHeartbeatCheck(response: LLMResponse, state: StateManager)
|
|
|
27
27
|
state.persona_update(personaId, { last_heartbeat: now });
|
|
28
28
|
state.queue_clearPersonaResponses(personaId, LLMNextStep.HandleHeartbeatCheck);
|
|
29
29
|
|
|
30
|
+
if (result.mentioned_reflection === true) {
|
|
31
|
+
state.persona_update(personaId, { reflection_last_asked: now });
|
|
32
|
+
console.log(`[HeartbeatCheck ${personaDisplayName}] Persona surfaced identity drift - reflection_last_asked set`);
|
|
33
|
+
}
|
|
34
|
+
|
|
30
35
|
if (!result.should_respond) {
|
|
31
36
|
console.log(`[HeartbeatCheck ${personaDisplayName}] Chose not to reach out (should_respond=false)`);
|
|
32
37
|
return;
|
|
@@ -107,4 +112,15 @@ export function handleEiHeartbeat(response: LLMResponse, state: StateManager): v
|
|
|
107
112
|
default:
|
|
108
113
|
console.warn(`[handleEiHeartbeat] Unexpected item type "${found.type}" for id "${result.id}"`);
|
|
109
114
|
}
|
|
115
|
+
|
|
116
|
+
const newPersonIds = (response.request.data.newPersonIds ?? []) as string[];
|
|
117
|
+
if (newPersonIds.length > 0) {
|
|
118
|
+
const human = state.getHuman();
|
|
119
|
+
for (const personId of newPersonIds) {
|
|
120
|
+
const person = human.people.find(p => p.id === personId);
|
|
121
|
+
if (person) {
|
|
122
|
+
state.human_person_upsert({ ...person, validated_date: now });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
110
126
|
}
|
|
@@ -1,15 +1,89 @@
|
|
|
1
|
-
import type { LLMResponse, Fact } from "../types.js";
|
|
1
|
+
import type { LLMResponse, Fact, Person } from "../types.js";
|
|
2
|
+
import type { PersonIdentifier } from "../types/data-items.js";
|
|
2
3
|
import type { StateManager } from "../state-manager.js";
|
|
3
4
|
import type {
|
|
4
5
|
FactFindResult,
|
|
5
6
|
TopicScanResult,
|
|
6
7
|
PersonScanResult,
|
|
7
8
|
TopicScanCandidate,
|
|
9
|
+
ItemMatchResult,
|
|
8
10
|
} from "../../prompts/human/types.js";
|
|
9
|
-
import { queueTopicMatch,
|
|
10
|
-
import { markMessagesExtracted } from "./utils.js";
|
|
11
|
+
import { queueTopicMatch, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
|
|
12
|
+
import { markMessagesExtracted, resolveMessageWindow } from "./utils.js";
|
|
11
13
|
import { BUILT_IN_FACT_NAMES } from "../constants/built-in-facts.js";
|
|
12
|
-
import { getEmbeddingService, getItemEmbeddingText } from "../embedding-service.js";
|
|
14
|
+
import { getEmbeddingService, getItemEmbeddingText, cosineSimilarity, getPersonEmbeddingText } from "../embedding-service.js";
|
|
15
|
+
import { levenshtein, normalizeForMatch } from "../utils/levenshtein.js";
|
|
16
|
+
|
|
17
|
+
const MULTI_MATCH_SIMILARITY_THRESHOLD = 0.75;
|
|
18
|
+
const ZERO_MATCH_COSINE_THRESHOLD = 0.80;
|
|
19
|
+
|
|
20
|
+
// Relationships where a person typically has exactly one instance.
|
|
21
|
+
// Only these fire the "sole relationship" uniqueness shortcut when the
|
|
22
|
+
// existing record already has a real name (non-Unknown records in non-singleton
|
|
23
|
+
// relationships fall through to cosine so we don't merge David into Sisyphus).
|
|
24
|
+
const SINGLETON_RELATIONSHIPS = new Set([
|
|
25
|
+
'self',
|
|
26
|
+
'husband', 'wife', 'spouse',
|
|
27
|
+
'father', 'mother',
|
|
28
|
+
]);
|
|
29
|
+
|
|
30
|
+
function matchPersonCandidate(
|
|
31
|
+
candidateName: string,
|
|
32
|
+
candidateIdentifiers: PersonIdentifier[],
|
|
33
|
+
people: Person[]
|
|
34
|
+
): Person[] {
|
|
35
|
+
const normName = normalizeForMatch(candidateName);
|
|
36
|
+
const matched = new Set<Person>();
|
|
37
|
+
|
|
38
|
+
// Step 1: Exact match on any identifier value (type-agnostic)
|
|
39
|
+
for (const person of people) {
|
|
40
|
+
const allValues = [
|
|
41
|
+
...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
|
|
42
|
+
normalizeForMatch(person.name),
|
|
43
|
+
];
|
|
44
|
+
if (allValues.includes(normName)) matched.add(person);
|
|
45
|
+
}
|
|
46
|
+
// Also check scan-extracted identifiers against existing identifier values
|
|
47
|
+
for (const scanId of candidateIdentifiers) {
|
|
48
|
+
const normVal = normalizeForMatch(scanId.value);
|
|
49
|
+
for (const person of people) {
|
|
50
|
+
if ((person.identifiers ?? []).some(i => normalizeForMatch(i.value) === normVal)) {
|
|
51
|
+
matched.add(person);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (matched.size > 0) return [...matched];
|
|
57
|
+
|
|
58
|
+
// Step 2: Fuzzy match — skip for short names (< 6 chars): "mike"↔"jake" = 2 edits, false positive.
|
|
59
|
+
if (normName.length >= 6) {
|
|
60
|
+
const threshold = normName.length < 10 ? 1 : 2;
|
|
61
|
+
for (const person of people) {
|
|
62
|
+
const allValues = [
|
|
63
|
+
...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
|
|
64
|
+
normalizeForMatch(person.name),
|
|
65
|
+
];
|
|
66
|
+
if (allValues.some(v => levenshtein(normName, v) <= threshold)) matched.add(person);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (matched.size > 0) return [...matched];
|
|
71
|
+
|
|
72
|
+
// Step 2.5: First-name match — "Lucas Jeremy Scherer" should find "Lucas".
|
|
73
|
+
// Only fires when first word is >= 4 chars to avoid short-name collisions.
|
|
74
|
+
const candidateFirstWord = normName.split(/\s+/)[0];
|
|
75
|
+
if (candidateFirstWord.length >= 4) {
|
|
76
|
+
for (const person of people) {
|
|
77
|
+
const allNames = [
|
|
78
|
+
normalizeForMatch(person.name),
|
|
79
|
+
...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
|
|
80
|
+
];
|
|
81
|
+
if (allNames.some(n => n.split(/\s+/)[0] === candidateFirstWord)) matched.add(person);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return [...matched];
|
|
86
|
+
}
|
|
13
87
|
|
|
14
88
|
export async function handleFactFind(response: LLMResponse, state: StateManager): Promise<void> {
|
|
15
89
|
const result = response.parsed as FactFindResult | undefined;
|
|
@@ -117,11 +191,131 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
|
|
|
117
191
|
const context = response.request.data as unknown as ExtractionContext;
|
|
118
192
|
if (!context?.personaId) return;
|
|
119
193
|
|
|
120
|
-
const
|
|
194
|
+
const { messages_context, messages_analyze } = resolveMessageWindow(response, state);
|
|
195
|
+
const human = state.getHuman();
|
|
196
|
+
|
|
121
197
|
for (const candidate of result.people) {
|
|
122
|
-
|
|
198
|
+
const candidateIdentifiers: PersonIdentifier[] = (candidate.identifiers ?? []).map(i => ({
|
|
199
|
+
type: i.type,
|
|
200
|
+
value: i.value,
|
|
201
|
+
...(i.is_primary ? { is_primary: i.is_primary } : {}),
|
|
202
|
+
}));
|
|
203
|
+
|
|
204
|
+
const matches = matchPersonCandidate(candidate.name, candidateIdentifiers, human.people);
|
|
205
|
+
|
|
206
|
+
let matchedPerson: Person | null = null;
|
|
207
|
+
|
|
208
|
+
if (matches.length === 1) {
|
|
209
|
+
matchedPerson = matches[0];
|
|
210
|
+
} else if (matches.length > 1) {
|
|
211
|
+
try {
|
|
212
|
+
const embeddingService = getEmbeddingService();
|
|
213
|
+
const candidateText = getPersonEmbeddingText({
|
|
214
|
+
name: candidate.name,
|
|
215
|
+
relationship: candidate.relationship,
|
|
216
|
+
description: candidate.description,
|
|
217
|
+
});
|
|
218
|
+
const candidateVector = await embeddingService.embed(candidateText);
|
|
219
|
+
let bestSimilarity = MULTI_MATCH_SIMILARITY_THRESHOLD;
|
|
220
|
+
for (const person of matches) {
|
|
221
|
+
if (person.embedding) {
|
|
222
|
+
const sim = cosineSimilarity(person.embedding, candidateVector);
|
|
223
|
+
if (sim > bestSimilarity) {
|
|
224
|
+
bestSimilarity = sim;
|
|
225
|
+
matchedPerson = person;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (!matchedPerson) {
|
|
230
|
+
console.log(`[handleHumanPersonScan] Multi-match for "${candidate.name}" (${matches.length} hits) — no embedding above threshold, creating new record`);
|
|
231
|
+
}
|
|
232
|
+
} catch (err) {
|
|
233
|
+
console.warn(`[handleHumanPersonScan] Multi-match embedding failed for "${candidate.name}", using first match:`, err);
|
|
234
|
+
matchedPerson = matches[0];
|
|
235
|
+
}
|
|
236
|
+
} else {
|
|
237
|
+
// Step 3: relationship filter → uniqueness match or cosine on the relevant subset.
|
|
238
|
+
// Filter first (O(N)), then cosine only on the filtered set (O(K) where K <= N).
|
|
239
|
+
const normRel = candidate.relationship?.toLowerCase();
|
|
240
|
+
const sameRel = normRel && normRel !== 'unknown'
|
|
241
|
+
? human.people.filter(p => p.relationship?.toLowerCase() === normRel)
|
|
242
|
+
: [];
|
|
243
|
+
|
|
244
|
+
if (sameRel.length === 1) {
|
|
245
|
+
const existing = sameRel[0];
|
|
246
|
+
const normExistingName = normalizeForMatch(existing.name);
|
|
247
|
+
const isUnknownPlaceholder = normExistingName === 'unknown' || normExistingName === normRel;
|
|
248
|
+
const isSingleton = SINGLETON_RELATIONSHIPS.has(normRel!);
|
|
249
|
+
if (isUnknownPlaceholder || isSingleton) {
|
|
250
|
+
matchedPerson = existing;
|
|
251
|
+
const reason = isUnknownPlaceholder ? 'unnamed placeholder' : 'singleton relationship';
|
|
252
|
+
console.log(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
|
|
253
|
+
}
|
|
254
|
+
} else {
|
|
255
|
+
// N>1 same relationship → cosine within that subset.
|
|
256
|
+
// N=0 (unknown relationship or no stored records) → cosine against all people.
|
|
257
|
+
const searchPool = sameRel.length > 1
|
|
258
|
+
? sameRel.filter(p => p.embedding && p.embedding.length > 0)
|
|
259
|
+
: human.people.filter(p => p.embedding && p.embedding.length > 0);
|
|
260
|
+
|
|
261
|
+
const poolLabel = sameRel.length > 1
|
|
262
|
+
? `${sameRel.length} ${candidate.relationship} records`
|
|
263
|
+
: `all ${human.people.length} people`;
|
|
264
|
+
|
|
265
|
+
if (searchPool.length > 0) {
|
|
266
|
+
console.log(`[handleHumanPersonScan] "${candidate.name}": cosine against ${searchPool.length} embedded (${poolLabel})`);
|
|
267
|
+
try {
|
|
268
|
+
const embeddingService = getEmbeddingService();
|
|
269
|
+
const candidateText = getPersonEmbeddingText({
|
|
270
|
+
name: candidate.name,
|
|
271
|
+
relationship: candidate.relationship,
|
|
272
|
+
description: candidate.description,
|
|
273
|
+
});
|
|
274
|
+
const candidateVector = await embeddingService.embed(candidateText);
|
|
275
|
+
const scores: Array<{ name: string; sim: number }> = [];
|
|
276
|
+
let bestSimilarity = ZERO_MATCH_COSINE_THRESHOLD;
|
|
277
|
+
for (const person of searchPool) {
|
|
278
|
+
const sim = cosineSimilarity(person.embedding!, candidateVector);
|
|
279
|
+
scores.push({ name: person.name, sim });
|
|
280
|
+
if (sim > bestSimilarity) {
|
|
281
|
+
bestSimilarity = sim;
|
|
282
|
+
matchedPerson = person;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
const top3 = scores.sort((a, b) => b.sim - a.sim).slice(0, 3).map(s => `"${s.name}"=${s.sim.toFixed(3)}`).join(', ');
|
|
286
|
+
if (matchedPerson) {
|
|
287
|
+
console.log(`[handleHumanPersonScan] Cosine matched "${candidate.name}" → "${matchedPerson.name}" (${bestSimilarity.toFixed(3)}) | top3: ${top3}`);
|
|
288
|
+
} else {
|
|
289
|
+
console.log(`[handleHumanPersonScan] Cosine: no match above ${ZERO_MATCH_COSINE_THRESHOLD} for "${candidate.name}" | top3: ${top3}`);
|
|
290
|
+
}
|
|
291
|
+
} catch (err) {
|
|
292
|
+
console.warn(`[handleHumanPersonScan] Cosine failed for "${candidate.name}":`, err);
|
|
293
|
+
}
|
|
294
|
+
} else {
|
|
295
|
+
console.log(`[handleHumanPersonScan] "${candidate.name}": no embedded people in pool (${poolLabel}) — new person`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const matchResult: ItemMatchResult = { matched_guid: matchedPerson?.id ?? null };
|
|
301
|
+
queuePersonUpdate(matchResult, {
|
|
302
|
+
...context,
|
|
303
|
+
messages_context,
|
|
304
|
+
messages_analyze,
|
|
305
|
+
candidateName: candidate.name,
|
|
306
|
+
candidateDescription: candidate.description,
|
|
307
|
+
candidateRelationship: candidate.relationship,
|
|
308
|
+
candidateIdentifiers,
|
|
309
|
+
}, state);
|
|
310
|
+
|
|
311
|
+
const matched = matchedPerson
|
|
312
|
+
? `matched "${matchedPerson.name}"`
|
|
313
|
+
: matches.length > 1
|
|
314
|
+
? `multi-match ambiguous (${matches.length} hits) — new record`
|
|
315
|
+
: "no match (new person)";
|
|
316
|
+
console.log(`[handleHumanPersonScan] person "${candidate.name}": ${matched}`);
|
|
123
317
|
}
|
|
124
|
-
console.log(`[handleHumanPersonScan]
|
|
318
|
+
console.log(`[handleHumanPersonScan] Processed ${result.people.length} person(s)`);
|
|
125
319
|
}
|
|
126
320
|
|
|
127
321
|
export async function handleEventScan(response: LLMResponse, state: StateManager): Promise<void> {
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
type Person,
|
|
6
6
|
type Quote,
|
|
7
7
|
} from "../types.js";
|
|
8
|
+
import type { PersonIdentifier } from "../types/data-items.js";
|
|
8
9
|
import type { StateManager } from "../state-manager.js";
|
|
9
10
|
import type { ItemMatchResult, ExposureImpact, TopicUpdateResult, PersonUpdateResult } from "../../prompts/human/types.js";
|
|
10
11
|
import { queueTopicUpdate, queuePersonUpdate, type ExtractionContext } from "../orchestrators/index.js";
|
|
@@ -13,12 +14,12 @@ import { calculateExposureCurrent } from "../utils/exposure.js";
|
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
import { resolveMessageWindow, getMessageText, normalizeRoomMessages } from "./utils.js";
|
|
17
|
+
import { sanitizeEiPersonaIdentifiers } from "../utils/identifier-utils.js";
|
|
16
18
|
|
|
17
19
|
export function handleTopicMatch(response: LLMResponse, state: StateManager): void {
|
|
18
20
|
const result = response.parsed as ItemMatchResult | undefined;
|
|
19
21
|
if (!result) {
|
|
20
|
-
|
|
21
|
-
return;
|
|
22
|
+
throw new Error("[handleTopicMatch] No parsed result");
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
const personaId = response.request.data.personaId as string;
|
|
@@ -64,8 +65,7 @@ export function handleTopicMatch(response: LLMResponse, state: StateManager): vo
|
|
|
64
65
|
export function handlePersonMatch(response: LLMResponse, state: StateManager): void {
|
|
65
66
|
const result = response.parsed as ItemMatchResult | undefined;
|
|
66
67
|
if (!result) {
|
|
67
|
-
|
|
68
|
-
return;
|
|
68
|
+
throw new Error("[handlePersonMatch] No parsed result");
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
const personaId = response.request.data.personaId as string;
|
|
@@ -123,11 +123,6 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
|
|
|
123
123
|
const roomId = response.request.data.roomId as string | undefined;
|
|
124
124
|
const candidateCategory = response.request.data.candidateCategory as string | undefined;
|
|
125
125
|
|
|
126
|
-
if (!result.name || !result.description || result.sentiment === undefined) {
|
|
127
|
-
console.error("[handleTopicUpdate] Missing required fields in result");
|
|
128
|
-
return;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
126
|
const personaIds = personaId.split("|").filter(Boolean);
|
|
132
127
|
const primaryId = personaIds[0] ?? personaId;
|
|
133
128
|
|
|
@@ -148,14 +143,21 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
|
|
|
148
143
|
|
|
149
144
|
const existingTopic = isNewItem ? undefined : human.topics.find(t => t.id === existingItemId);
|
|
150
145
|
|
|
146
|
+
const resolvedName = result.name || existingTopic?.name;
|
|
147
|
+
const resolvedDescription = typeof result.description === 'string' ? result.description : existingTopic?.description;
|
|
148
|
+
|
|
149
|
+
if (!resolvedName || !resolvedDescription || result.sentiment === undefined) {
|
|
150
|
+
throw new Error(`[handleTopicUpdate] Missing required fields: name=${resolvedName}, description=${!!resolvedDescription}, sentiment=${result.sentiment}`);
|
|
151
|
+
}
|
|
152
|
+
|
|
151
153
|
let embedding: number[] | undefined;
|
|
152
154
|
try {
|
|
153
155
|
const embeddingService = getEmbeddingService();
|
|
154
156
|
const category = result.category ?? candidateCategory ?? existingTopic?.category;
|
|
155
|
-
const text = getTopicEmbeddingText({ name:
|
|
157
|
+
const text = getTopicEmbeddingText({ name: resolvedName, category, description: resolvedDescription });
|
|
156
158
|
embedding = await embeddingService.embed(text);
|
|
157
159
|
} catch (err) {
|
|
158
|
-
console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${
|
|
160
|
+
console.warn(`[handleTopicUpdate] Failed to compute embedding for topic "${resolvedName}":`, err);
|
|
159
161
|
}
|
|
160
162
|
|
|
161
163
|
const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
|
|
@@ -168,13 +170,14 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
|
|
|
168
170
|
|
|
169
171
|
const topic: Topic = {
|
|
170
172
|
id: itemId,
|
|
171
|
-
name:
|
|
172
|
-
description:
|
|
173
|
+
name: resolvedName,
|
|
174
|
+
description: resolvedDescription,
|
|
173
175
|
sentiment: result.sentiment,
|
|
174
176
|
category: result.category ?? candidateCategory ?? existingTopic?.category,
|
|
175
177
|
exposure_current: calculateExposureCurrent(exposureImpact, existingTopic?.exposure_current ?? 0),
|
|
176
178
|
exposure_desired: result.exposure_desired ?? 0.5,
|
|
177
179
|
last_updated: now,
|
|
180
|
+
learned_on: isNewItem ? now : existingTopic?.learned_on,
|
|
178
181
|
last_mentioned: now,
|
|
179
182
|
learned_by: isNewItem ? primaryId : existingTopic?.learned_by,
|
|
180
183
|
last_changed_by: primaryId,
|
|
@@ -189,11 +192,15 @@ export async function handleTopicUpdate(response: LLMResponse, state: StateManag
|
|
|
189
192
|
: state.messages_get(personaId);
|
|
190
193
|
await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
|
|
191
194
|
|
|
192
|
-
console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${
|
|
195
|
+
console.log(`[handleTopicUpdate] ${isNewItem ? "Created" : "Updated"} topic "${resolvedName}"`);
|
|
193
196
|
}
|
|
194
197
|
|
|
195
198
|
export async function handlePersonUpdate(response: LLMResponse, state: StateManager): Promise<void> {
|
|
196
|
-
const result = response.parsed as (PersonUpdateResult & {
|
|
199
|
+
const result = response.parsed as (PersonUpdateResult & {
|
|
200
|
+
identifiers?: PersonIdentifier[];
|
|
201
|
+
identifiers_to_add?: PersonIdentifier[];
|
|
202
|
+
quotes?: Array<{ text: string; reason: string }>;
|
|
203
|
+
}) | undefined;
|
|
197
204
|
|
|
198
205
|
if (!result || Object.keys(result).length === 0) {
|
|
199
206
|
console.log("[handlePersonUpdate] No changes needed (empty result)");
|
|
@@ -206,12 +213,13 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
|
|
|
206
213
|
const personaDisplayName = response.request.data.personaDisplayName as string;
|
|
207
214
|
const roomId = response.request.data.roomId as string | undefined;
|
|
208
215
|
const candidateRelationship = response.request.data.candidateRelationship as string | undefined;
|
|
216
|
+
const candidateIdentifiers = (response.request.data.candidateIdentifiers ?? []) as PersonIdentifier[];
|
|
209
217
|
|
|
210
|
-
if (!result.
|
|
211
|
-
|
|
212
|
-
return;
|
|
218
|
+
if (!result.description || result.sentiment === undefined) {
|
|
219
|
+
throw new Error(`[handlePersonUpdate] Missing required fields: description=${!!result.description}, sentiment=${result.sentiment}`);
|
|
213
220
|
}
|
|
214
221
|
|
|
222
|
+
const candidateName = response.request.data.candidateName as string;
|
|
215
223
|
const personaIds = personaId.split("|").filter(Boolean);
|
|
216
224
|
const primaryId = personaIds[0] ?? personaId;
|
|
217
225
|
|
|
@@ -236,10 +244,10 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
|
|
|
236
244
|
try {
|
|
237
245
|
const embeddingService = getEmbeddingService();
|
|
238
246
|
const relationship = result.relationship ?? candidateRelationship ?? existingPerson?.relationship;
|
|
239
|
-
const text = getPersonEmbeddingText({ name:
|
|
247
|
+
const text = getPersonEmbeddingText({ name: candidateName, relationship, description: result.description });
|
|
240
248
|
embedding = await embeddingService.embed(text);
|
|
241
249
|
} catch (err) {
|
|
242
|
-
console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${
|
|
250
|
+
console.warn(`[handlePersonUpdate] Failed to compute embedding for person "${candidateName}":`, err);
|
|
243
251
|
}
|
|
244
252
|
|
|
245
253
|
const exposureImpact = result.exposure_impact as ExposureImpact | undefined;
|
|
@@ -250,15 +258,51 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
|
|
|
250
258
|
? (allPersonaGroups.length > 0 ? allPersonaGroups : existingPerson?.persona_groups)
|
|
251
259
|
: [...new Set([...(existingPerson?.persona_groups ?? []), ...allPersonaGroups])];
|
|
252
260
|
|
|
261
|
+
let resolvedIdentifiers: PersonIdentifier[];
|
|
262
|
+
if (isNewItem) {
|
|
263
|
+
const llmIdentifiers: PersonIdentifier[] = sanitizeEiPersonaIdentifiers(
|
|
264
|
+
(result.identifiers ?? []).map(i => ({
|
|
265
|
+
type: i.type,
|
|
266
|
+
value: i.value,
|
|
267
|
+
...(i.is_primary ? { is_primary: i.is_primary } : {}),
|
|
268
|
+
})),
|
|
269
|
+
state
|
|
270
|
+
);
|
|
271
|
+
const allCandidateIds = [...llmIdentifiers, ...candidateIdentifiers];
|
|
272
|
+
if (allCandidateIds.length === 0) {
|
|
273
|
+
const hasSpace = candidateName.includes(' ');
|
|
274
|
+
allCandidateIds.push({ type: hasSpace ? "full_name" : "nickname", value: candidateName, is_primary: true });
|
|
275
|
+
}
|
|
276
|
+
const deduped: PersonIdentifier[] = [];
|
|
277
|
+
for (const id of allCandidateIds) {
|
|
278
|
+
if (!deduped.some(e => e.value === id.value)) {
|
|
279
|
+
deduped.push(id);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
resolvedIdentifiers = deduped;
|
|
283
|
+
} else {
|
|
284
|
+
const base = [...(existingPerson?.identifiers ?? [])];
|
|
285
|
+
const sanitizedToAdd = sanitizeEiPersonaIdentifiers(result.identifiers_to_add ?? [], state);
|
|
286
|
+
for (const id of sanitizedToAdd) {
|
|
287
|
+
if (!base.some(e => e.value === id.value)) {
|
|
288
|
+
base.push({ type: id.type, value: id.value, ...(id.is_primary ? { is_primary: id.is_primary } : {}) });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
resolvedIdentifiers = base;
|
|
292
|
+
}
|
|
293
|
+
|
|
253
294
|
const person: Person = {
|
|
254
295
|
id: itemId,
|
|
255
|
-
name:
|
|
296
|
+
name: candidateName,
|
|
256
297
|
description: result.description,
|
|
257
298
|
sentiment: result.sentiment,
|
|
258
299
|
relationship: result.relationship ?? candidateRelationship ?? existingPerson?.relationship ?? "Unknown",
|
|
259
300
|
exposure_current: calculateExposureCurrent(exposureImpact, existingPerson?.exposure_current ?? 0),
|
|
260
301
|
exposure_desired: result.exposure_desired ?? 0.5,
|
|
302
|
+
identifiers: resolvedIdentifiers,
|
|
303
|
+
validated_date: isNewItem ? '' : (existingPerson?.validated_date ?? ''),
|
|
261
304
|
last_updated: now,
|
|
305
|
+
learned_on: isNewItem ? now : existingPerson?.learned_on,
|
|
262
306
|
last_mentioned: now,
|
|
263
307
|
learned_by: isNewItem ? primaryId : existingPerson?.learned_by,
|
|
264
308
|
last_changed_by: primaryId,
|
|
@@ -273,9 +317,14 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
|
|
|
273
317
|
: state.messages_get(personaId);
|
|
274
318
|
await validateAndStoreQuotes(result.quotes, allMessages, itemId, personaDisplayName, personaGroup, state);
|
|
275
319
|
|
|
276
|
-
|
|
320
|
+
const primaryValue = resolvedIdentifiers.find(i => i.is_primary)?.value ?? candidateName;
|
|
321
|
+
const resolvedName = (!primaryValue || primaryValue.toLowerCase() === 'unknown')
|
|
322
|
+
? (result.relationship ?? candidateRelationship ?? '(unknown)')
|
|
323
|
+
: primaryValue;
|
|
324
|
+
console.log(`[handlePersonUpdate] ${isNewItem ? "Created" : "Updated"} person "${resolvedName}"`);
|
|
277
325
|
}
|
|
278
326
|
|
|
327
|
+
|
|
279
328
|
function normalizeText(text: string): string {
|
|
280
329
|
return text
|
|
281
330
|
.replace(/[\u201C\u201D]/g, '"') // curly double quotes
|