ei-tui 1.6.9 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +4 -0
  2. package/package.json +2 -1
  3. package/skills/coding-harness-reflect/SKILL.md +230 -0
  4. package/skills/ei-curate/SKILL.md +174 -0
  5. package/skills/ei-curate/references/cli.md +160 -0
  6. package/skills/ei-curate/references/provenance.md +88 -0
  7. package/skills/ei-curate/references/recipes.md +144 -0
  8. package/skills/ei-curate/references/talking-to-the-user.md +71 -0
  9. package/skills/ei-persona/SKILL.md +238 -0
  10. package/skills/ei-persona/references/cli.md +265 -0
  11. package/skills/ei-persona/references/recipes.md +247 -0
  12. package/skills/ei-persona/references/talking-to-the-user.md +107 -0
  13. package/src/cli/README.md +72 -2
  14. package/src/cli/corrections-endpoints.ts +297 -0
  15. package/src/cli/corrections-writer.ts +138 -0
  16. package/src/cli/install.ts +117 -25
  17. package/src/cli/mcp.ts +80 -1
  18. package/src/cli/persona-corrections.ts +442 -0
  19. package/src/cli/retrieval.ts +46 -2
  20. package/src/cli.ts +131 -1
  21. package/src/core/corrections.ts +233 -0
  22. package/src/core/handlers/human-extraction.ts +64 -15
  23. package/src/core/handlers/human-matching.ts +2 -2
  24. package/src/core/orchestrators/human-extraction.ts +1 -11
  25. package/src/core/persona-manager.ts +18 -7
  26. package/src/core/persona-tools.ts +92 -0
  27. package/src/core/processor.ts +113 -1
  28. package/src/core/state/human.ts +10 -0
  29. package/src/core/state/personas.ts +8 -0
  30. package/src/core/state-manager.ts +11 -0
  31. package/src/core/utils/identifier-utils.ts +3 -2
  32. package/src/prompts/human/person-scan.ts +2 -0
  33. package/src/prompts/human/person-update.ts +14 -3
  34. package/src/storage/file-lock.ts +120 -0
  35. package/tui/README.md +2 -0
  36. package/tui/src/commands/provider.tsx +1 -1
  37. package/tui/src/components/WelcomeOverlay.tsx +3 -3
  38. package/tui/src/context/ei.tsx +14 -0
  39. package/tui/src/index.tsx +13 -1
  40. package/tui/src/storage/file.ts +15 -83
  41. package/tui/src/util/instance-lock.ts +3 -2
  42. package/tui/src/util/yaml-persona.ts +7 -38
package/src/cli.ts CHANGED
@@ -17,6 +17,9 @@ import type { StorageState } from "./core/types";
17
17
  import { resolvePersonaId, filterByPersona, filterTypeSpecificByPersona, filterBySource, filterTypeSpecificBySource } from "./cli/persona-filter.js";
18
18
  import { installMcpClients } from "./cli/install.js";
19
19
  import { getRecentSessionMessages } from "./cli/session-context.js";
20
+ import { createEntity, updateEntity, removeEntity, CorrectionValidationError, CORRECTABLE_TYPES, UPDATABLE_TYPES } from "./cli/corrections-endpoints.js";
21
+ import { createPersonaEntity, updatePersonaEntity, removePersonaEntity } from "./cli/persona-corrections.js";
22
+ import type { CorrectableType } from "./core/corrections.js";
20
23
  import pkg from "../package.json" assert { type: "json" };
21
24
 
22
25
  const rawArgs = process.argv.slice(2);
@@ -38,6 +41,39 @@ const TYPE_ALIASES: Record<string, string> = {
38
41
  personas: "personas",
39
42
  };
40
43
 
44
+ // Wider set accepted by `ei create/update/remove persona` — corrections-endpoints.ts's
45
+ // CORRECTABLE_TYPES/UPDATABLE_TYPES intentionally stay persona-free (personas bypass
46
+ // that module's shared SCHEMAS dispatch entirely, see persona-corrections.ts), so this
47
+ // CLI-only layer is what actually widens the accepted type set.
48
+ const CLI_CORRECTABLE_TYPES = [...CORRECTABLE_TYPES, "persona"] as const;
49
+ const CLI_UPDATABLE_TYPES = [...UPDATABLE_TYPES, "persona"] as const;
50
+
51
+ // Singular CorrectableType resolution for `ei create/update/remove` — derived
52
+ // from TYPE_ALIASES + CLI_CORRECTABLE_TYPES so the two alias systems can never
53
+ // silently diverge on which strings are accepted (e.g. "person"/"people"
54
+ // stay synonymous here too, since both funnel through TYPE_ALIASES first).
55
+ const PLURAL_TO_CORRECTABLE: Record<string, CorrectableType> = Object.fromEntries(
56
+ CLI_CORRECTABLE_TYPES.map((t) => [TYPE_ALIASES[t], t])
57
+ );
58
+
59
+ function resolveCorrectableType(raw: string): CorrectableType | null {
60
+ const plural = TYPE_ALIASES[raw];
61
+ return plural ? PLURAL_TO_CORRECTABLE[plural] ?? null : null;
62
+ }
63
+
64
+ // Plural CorrectableType resolution for `ei update` — same TYPE_ALIASES
65
+ // lookup as resolveCorrectableType above, but sourced from CLI_UPDATABLE_TYPES
66
+ // instead of CLI_CORRECTABLE_TYPES since quotes are correctable via update
67
+ // (repointing data_item_ids after a split/merge, fixing mistranscribed
68
+ // text) but never created or removed.
69
+ const PLURAL_TO_UPDATABLE: Record<string, CorrectableType> = Object.fromEntries(
70
+ CLI_UPDATABLE_TYPES.map((t) => [TYPE_ALIASES[t], t])
71
+ );
72
+ function resolveUpdatableType(raw: string): CorrectableType | null {
73
+ const plural = TYPE_ALIASES[raw];
74
+ return plural ? PLURAL_TO_UPDATABLE[plural] ?? null : null;
75
+ }
76
+
41
77
  function printHelp(): void {
42
78
  console.log(`
43
79
  Ei
@@ -55,6 +91,9 @@ Usage:
55
91
  ei --id <id> Look up a specific entity by ID
56
92
  echo <id> | ei --id Look up entity by ID from stdin
57
93
  ei mcp Start the Ei MCP stdio server (for Claude Code/Cursor/Codex)
94
+ ei create <type> --json '<json>' Create a new entity (fact/topic/person/persona)
95
+ ei update <type> <id> --json '<json>' Replace an entity by ID (full record, not a patch; fact/topic/person/quote/persona)
96
+ ei remove <type> <id> Remove an entity by ID (fact/topic/person/persona; not quotes)
58
97
 
59
98
  Types:
60
99
  quote / quotes Quotes from conversation history
@@ -69,12 +108,13 @@ Options:
69
108
  --persona, -p Filter to entities a specific persona has learned about
70
109
  --source, -s Filter to entities from a specific source (prefix match, e.g. "cursor", "codex:my-machine", "opencode:my-machine:ses_abc123")
71
110
  --id Look up entity by ID (accepts value or stdin)
72
- --install Register Ei with Claude Code, Cursor, Codex, and OpenCode (MCP + context hooks where supported)
111
+ --install Register Ei with Claude Code, Cursor, Codex, and OpenCode (MCP + context hooks + skills where supported)
73
112
  --sync Pull latest state from remote sync server into state.backup.json (no TUI required)
74
113
  --session <id> Session ID to enrich the query with recent context (use with --hook-source)
75
114
  --hook-source <src> Source of the hook: "opencode-plugin" (OpenCode SQLite), "cursor", or "codex"
76
115
  --transcript <path> Path to a Claude Code JSONL transcript file for context enrichment
77
116
  --help, -h Show this help message
117
+ --json <json> JSON body for create/update (full record for update, not a patch)
78
118
 
79
119
  Examples:
80
120
  ei "debugging" # Search everything
@@ -86,6 +126,13 @@ Examples:
86
126
  ei topics --source cursor "X" # Topics learned from Cursor sessions
87
127
  ei --id abc-123 # Look up entity by ID
88
128
  ei "memory leak" | jq .[0].id | ei --id # Pipe ID from search
129
+ ei create fact --json '{"name":"Field of Study","description":"CS","sentiment":0,"validated_date":""}'
130
+ ei update fact abc-123 --json '{"name":"Field of Study","description":"Updated","sentiment":0,"validated_date":""}'
131
+ ei update quote <id> --json '{"data_item_ids":["person-b-id"], ...}' # Repoint a quote after splitting a bad merge (fetch the full record via 'ei --id <id>' first)
132
+ ei create persona --json '{"display_name":"Yoda","long_description":"Speaks in inverted syntax, wise and patient.","traits":[{"name":"Inverted speech","description":"Talks like Yoda","sentiment":0.7}],"topics":[]}'
133
+ ei update persona <id> --json '<full persona record from ei --id <id>, edited>'
134
+ ei remove persona abc-123 # Remove a persona (reserved personas like "ei"/"emmet" must be archived instead)
135
+ ei remove fact abc-123 # Remove a fact by ID
89
136
  `);
90
137
  }
91
138
 
@@ -146,6 +193,89 @@ async function main(): Promise<void> {
146
193
  process.exit(0);
147
194
  }
148
195
 
196
+ if (args[0] === "create") {
197
+ const rawType = args[1];
198
+ const entityType = rawType ? resolveCorrectableType(rawType) : null;
199
+ if (!entityType) {
200
+ console.error(`ei create requires a valid type (${CLI_CORRECTABLE_TYPES.join(", ")}). Got: ${rawType ?? "(none)"}`);
201
+ process.exit(1);
202
+ }
203
+ const jsonIdx = args.indexOf("--json");
204
+ const jsonStr = jsonIdx !== -1 ? args[jsonIdx + 1] : undefined;
205
+ if (!jsonStr) {
206
+ console.error("ei create requires --json '<json>'");
207
+ process.exit(1);
208
+ }
209
+ let body: unknown;
210
+ try {
211
+ body = JSON.parse(jsonStr);
212
+ } catch (e) {
213
+ console.error(`Invalid JSON: ${(e as Error).message}`);
214
+ process.exit(1);
215
+ }
216
+ try {
217
+ const result = entityType === "persona" ? await createPersonaEntity(body) : await createEntity(entityType, body);
218
+ console.log(JSON.stringify(result, null, 2));
219
+ process.exit(0);
220
+ } catch (e) {
221
+ console.error(e instanceof CorrectionValidationError ? e.message : (e as Error).message);
222
+ process.exit(1);
223
+ }
224
+ }
225
+
226
+ if (args[0] === "update") {
227
+ const rawType = args[1];
228
+ const id = args[2];
229
+ const entityType = rawType ? resolveUpdatableType(rawType) : null;
230
+ if (!entityType || !id) {
231
+ console.error(`Usage: ei update <type> <id> --json '<json>' (types: ${CLI_UPDATABLE_TYPES.join(", ")})`);
232
+ process.exit(1);
233
+ }
234
+ const jsonIdx = args.indexOf("--json");
235
+ const jsonStr = jsonIdx !== -1 ? args[jsonIdx + 1] : undefined;
236
+ if (!jsonStr) {
237
+ console.error("ei update requires --json '<json>'");
238
+ process.exit(1);
239
+ }
240
+ let body: unknown;
241
+ try {
242
+ body = JSON.parse(jsonStr);
243
+ } catch (e) {
244
+ console.error(`Invalid JSON: ${(e as Error).message}`);
245
+ process.exit(1);
246
+ }
247
+ try {
248
+ const record = entityType === "persona" ? await updatePersonaEntity(id, body) : await updateEntity(entityType, id, body);
249
+ console.log(JSON.stringify(record, null, 2));
250
+ process.exit(0);
251
+ } catch (e) {
252
+ console.error(e instanceof CorrectionValidationError ? e.message : (e as Error).message);
253
+ process.exit(1);
254
+ }
255
+ }
256
+
257
+ if (args[0] === "remove") {
258
+ const rawType = args[1];
259
+ const id = args[2];
260
+ const entityType = rawType ? resolveCorrectableType(rawType) : null;
261
+ if (!entityType || !id) {
262
+ console.error(`Usage: ei remove <type> <id> (types: ${CLI_CORRECTABLE_TYPES.join(", ")})`);
263
+ process.exit(1);
264
+ }
265
+ try {
266
+ if (entityType === "persona") {
267
+ await removePersonaEntity(id);
268
+ } else {
269
+ await removeEntity(entityType, id);
270
+ }
271
+ console.log(JSON.stringify({ removed: true, id }, null, 2));
272
+ process.exit(0);
273
+ } catch (e) {
274
+ console.error((e as Error).message);
275
+ process.exit(1);
276
+ }
277
+ }
278
+
149
279
  if (args[0] === "--sync") {
150
280
  const { getDataPath } = await import("./cli/retrieval.js");
151
281
  const { RemoteSync } = await import("./storage/remote.js");
@@ -0,0 +1,233 @@
1
+ /**
2
+ * Corrections Queue — external write path for CLI/MCP tools.
3
+ *
4
+ * Ei's CLI and MCP server are read-only against a snapshot of state.json —
5
+ * they never hold the live StateManager the running TUI/daemon owns, so
6
+ * they can't safely mutate state.json directly (the running instance's
7
+ * next debounced save would silently clobber the write with its stale
8
+ * in-memory copy).
9
+ *
10
+ * corrections.json is the queue that closes this gap. External writers
11
+ * (ei_update / ei_create / ei_remove) append fully-formed CorrectionRecords
12
+ * here under an advisory lock. Two consumers drain it:
13
+ * - The running Processor, once per runLoop tick (near-instant, ~100ms).
14
+ * - The CLI itself, self-draining directly into state.json when it
15
+ * detects no live instance is running (see src/cli/corrections-io.ts).
16
+ *
17
+ * Every CorrectionRecord is a complete, ready-to-apply entity — including
18
+ * a pre-computed embedding and (for creates) a pre-assigned id — so
19
+ * draining is a pure apply, never a fetch-then-merge.
20
+ */
21
+
22
+ import type { HumanEntity, Fact, Topic, Person, Quote, PersonaEntity, StorageState } from "./types.js";
23
+ import { isReservedPersonaId } from "./types.js";
24
+ import { withLock, atomicWrite } from "../storage/file-lock.js";
25
+
26
+ export type CorrectableType = "fact" | "topic" | "person" | "quote" | "persona";
27
+ export type CorrectableEntity = Fact | Topic | Person | Quote | PersonaEntity;
28
+ export const CORRECTABLE_TYPES: CorrectableType[] = ["fact", "topic", "person", "quote", "persona"];
29
+
30
+ export interface CorrectionUpsert {
31
+ op: "upsert";
32
+ entity_type: CorrectableType;
33
+ id: string;
34
+ record: CorrectableEntity;
35
+ timestamp: string;
36
+ }
37
+
38
+ export interface CorrectionRemove {
39
+ op: "remove";
40
+ entity_type: CorrectableType;
41
+ id: string;
42
+ timestamp: string;
43
+ }
44
+
45
+ export type CorrectionRecord = CorrectionUpsert | CorrectionRemove;
46
+
47
+ /**
48
+ * Runtime shape validation for a CorrectionRecord read back from
49
+ * corrections.json. The TypeScript union is compile-time only — every
50
+ * record actually enters the system via `JSON.parse(...) as CorrectionRecord`
51
+ * in readCorrections(), so a malformed-but-valid-JSON record (bad `op`,
52
+ * mismatched `record.id`, missing `record` on an upsert) is otherwise
53
+ * silently trusted. Both consumers (CLI read-merge/self-drain via
54
+ * applyCorrectionToHuman, and Processor.applyCorrectionRecord) call this
55
+ * before mutating anything — it throws, never coerces, so a malformed `op`
56
+ * can never be silently treated as its sibling operation.
57
+ */
58
+ export function assertValidCorrection(value: unknown): asserts value is CorrectionRecord {
59
+ if (!value || typeof value !== "object") {
60
+ throw new Error(`Malformed correction record: expected an object, got ${JSON.stringify(value)}`);
61
+ }
62
+ if (!("op" in value) || (value.op !== "upsert" && value.op !== "remove")) {
63
+ throw new Error(`Malformed correction record: op must be "upsert" or "remove", got ${JSON.stringify("op" in value ? value.op : undefined)}`);
64
+ }
65
+ if (!("entity_type" in value) || !CORRECTABLE_TYPES.includes(value.entity_type as CorrectableType)) {
66
+ throw new Error(`Malformed correction record: entity_type must be one of ${CORRECTABLE_TYPES.join(", ")}, got ${JSON.stringify("entity_type" in value ? value.entity_type : undefined)}`);
67
+ }
68
+ if (!("id" in value) || typeof value.id !== "string" || value.id.length === 0) {
69
+ throw new Error(`Malformed correction record: id must be a non-empty string, got ${JSON.stringify("id" in value ? value.id : undefined)}`);
70
+ }
71
+ if (value.op === "upsert") {
72
+ if (!("record" in value) || !value.record || typeof value.record !== "object") {
73
+ throw new Error(`Malformed correction record: upsert requires a record object`);
74
+ }
75
+ if (!("id" in value.record) || value.record.id !== value.id) {
76
+ throw new Error(`Malformed correction record: record.id (${JSON.stringify("id" in value.record ? value.record.id : undefined)}) must equal wrapper id (${JSON.stringify(value.id)})`);
77
+ }
78
+ }
79
+ }
80
+
81
+ /**
82
+ * Read pending corrections without a lock. Returns [] if the file doesn't
83
+ * exist or is empty. fs/promises is imported dynamically, not statically —
84
+ * this module is transitively bundled into Web's browser build via
85
+ * src/core/processor.ts, and a static `import { readFile } from "fs/promises"`
86
+ * fails Vite's rollup build (no named exports on the browser-externalized
87
+ * stub), not just at runtime.
88
+ */
89
+ export async function readCorrections(correctionsPath: string): Promise<CorrectionRecord[]> {
90
+ const { readFile } = await import(/* @vite-ignore */ "fs/promises");
91
+ let text: string;
92
+ try {
93
+ text = await readFile(correctionsPath, "utf-8");
94
+ } catch {
95
+ return [];
96
+ }
97
+ if (!text) return [];
98
+ return JSON.parse(text) as CorrectionRecord[];
99
+ }
100
+
101
+ /**
102
+ * Append one correction under lock (read-modify-write — corrections.json
103
+ * is a JSON array, not an append-only log, so concurrent writers must
104
+ * serialize through the same lock rather than racing on a blind append).
105
+ */
106
+ export async function appendCorrection(
107
+ correctionsPath: string,
108
+ record: CorrectionRecord
109
+ ): Promise<void> {
110
+ await withLock(correctionsPath, async () => {
111
+ const existing = await readCorrections(correctionsPath);
112
+ existing.push(record);
113
+ await atomicWrite(correctionsPath, JSON.stringify(existing, null, 2));
114
+ });
115
+ }
116
+
117
+ /**
118
+ * Merge a person's `name` from its primary identifier, replicating the
119
+ * invariant HumanState.person_upsert enforces on the live-write path
120
+ * (CONTRACTS.md "Person Identifiers — name Sync Rule"). Both consumers of
121
+ * this module (Live's StateManager and the CLI's self-drain) must produce
122
+ * an identical HumanEntity for a given corrections.json, so the formula
123
+ * is centralized here rather than re-derived at each call site.
124
+ */
125
+ function syncPersonName(person: Person): Person {
126
+ const identifiers = person.identifiers ?? [];
127
+ const primary = identifiers.find((i) => i.is_primary) ?? identifiers[0];
128
+ return primary ? { ...person, name: primary.value } : person;
129
+ }
130
+
131
+ /**
132
+ * Resolve the target array for a CorrectableType. Throws on anything
133
+ * other than the 4 known types — corrections.json is external input from
134
+ * CLI/MCP tools (potentially LLM-driven), and a malformed entity_type must
135
+ * never silently fall through to the people array. Live's Processor
136
+ * already enforces this (applyCorrectionRecord); this is the equivalent
137
+ * guard for the CLI read-merge and self-drain paths.
138
+ */
139
+ function getCorrectableArray(human: HumanEntity, entityType: string): Array<{ id: string }> {
140
+ if (entityType === "fact") return human.facts;
141
+ if (entityType === "topic") return human.topics;
142
+ if (entityType === "person") return human.people;
143
+ if (entityType === "quote") return human.quotes;
144
+ throw new Error(`Unrecognized correction entity_type: ${entityType}`);
145
+ }
146
+
147
+ /**
148
+ * Apply one correction to a HumanEntity in place, mirroring the exact
149
+ * upsert/remove semantics of HumanState (src/core/state/human.ts) —
150
+ * replace-by-id for upsert, splice + orphaned quote-reference cleanup for
151
+ * remove (for all 3 types, matching fact_remove/topic_remove/person_remove
152
+ * in HumanState — not just person). Used by both the CLI's read-merge
153
+ * (materializing a corrected view without a StateManager) and its
154
+ * self-drain (writing corrections straight into state.json when no live
155
+ * instance is running).
156
+ */
157
+ export function applyCorrectionToHuman(human: HumanEntity, correction: CorrectionRecord): void {
158
+ assertValidCorrection(correction);
159
+ const array = getCorrectableArray(human, correction.entity_type);
160
+
161
+ if (correction.op === "remove") {
162
+ const idx = array.findIndex((item) => item.id === correction.id);
163
+ if (idx < 0) return;
164
+ array.splice(idx, 1);
165
+ human.quotes.forEach((q) => {
166
+ q.data_item_ids = q.data_item_ids.filter((itemId) => itemId !== correction.id);
167
+ });
168
+ human.last_updated = new Date().toISOString();
169
+ return;
170
+ }
171
+
172
+ const record = correction.entity_type === "person"
173
+ ? syncPersonName(correction.record as Person)
174
+ : correction.record;
175
+ const idx = array.findIndex((item) => item.id === correction.id);
176
+ if (idx >= 0) {
177
+ array[idx] = record;
178
+ } else {
179
+ array.push(record);
180
+ }
181
+ human.last_updated = new Date().toISOString();
182
+ }
183
+
184
+ /** Apply every pending correction to a HumanEntity, in file order (later records for the same id win). */
185
+ export function applyCorrectionsToHuman(human: HumanEntity, corrections: CorrectionRecord[]): void {
186
+ for (const correction of corrections) {
187
+ applyCorrectionToHuman(human, correction);
188
+ }
189
+ }
190
+
191
+ /**
192
+ * Apply one correction to a StorageState's personas map in place. Personas
193
+ * live outside HumanEntity — src/core/types/integrations.ts's StorageState.personas
194
+ * is a top-level `Record<id, {entity, messages}>` — so they need their own
195
+ * apply function rather than routing through getCorrectableArray/
196
+ * applyCorrectionToHuman, which only ever resolve arrays on HumanEntity.
197
+ *
198
+ * The reserved-persona delete guard here is defense-in-depth for a
199
+ * hand-edited corrections.json: the primary guard is the SYNCHRONOUS check
200
+ * in src/cli/persona-corrections.ts's removePersonaEntity, which runs
201
+ * before a correction is ever queued (so a live-drained rejection here can
202
+ * never surface as a silent no-op after an apparent CLI success).
203
+ */
204
+ export function applyCorrectionToPersonas(personas: StorageState["personas"], correction: CorrectionRecord): void {
205
+ assertValidCorrection(correction);
206
+ if (correction.op === "remove") {
207
+ if (isReservedPersonaId(correction.id)) {
208
+ throw new Error(`Cannot delete reserved persona "${correction.id}". Use archive instead.`);
209
+ }
210
+ delete personas[correction.id];
211
+ return;
212
+ }
213
+ personas[correction.id] = {
214
+ entity: correction.record as PersonaEntity,
215
+ messages: personas[correction.id]?.messages ?? [],
216
+ };
217
+ }
218
+
219
+ /** Route one correction to its target: the personas map for entity_type "persona", the HumanEntity for everything else. */
220
+ export function applyCorrectionToState(state: StorageState, correction: CorrectionRecord): void {
221
+ if (correction.entity_type === "persona") {
222
+ applyCorrectionToPersonas(state.personas, correction);
223
+ } else {
224
+ applyCorrectionToHuman(state.human, correction);
225
+ }
226
+ }
227
+
228
+ /** Apply every pending correction to a StorageState, in file order (later records for the same id win). */
229
+ export function applyCorrectionsToState(state: StorageState, corrections: CorrectionRecord[]): void {
230
+ for (const correction of corrections) {
231
+ applyCorrectionToState(state, correction);
232
+ }
233
+ }
@@ -14,6 +14,9 @@ import { BUILT_IN_FACT_NAMES } from "../constants/built-in-facts.js";
14
14
  import { getEmbeddingService, getItemEmbeddingText, cosineSimilarity, getPersonEmbeddingText } from "../embedding-service.js";
15
15
  import { levenshtein, normalizeForMatch } from "../utils/levenshtein.js";
16
16
 
17
+ export type PersonMatchStrength = 'strong' | 'weak';
18
+ export interface PersonMatch { person: Person; strength: PersonMatchStrength; }
19
+
17
20
  const MULTI_MATCH_SIMILARITY_THRESHOLD = 0.75;
18
21
  const ZERO_MATCH_COSINE_THRESHOLD = 0.80;
19
22
 
@@ -27,13 +30,25 @@ const SINGLETON_RELATIONSHIPS = new Set([
27
30
  'father', 'mother',
28
31
  ]);
29
32
 
33
+ function sharesNameToken(normalizedCandidateName: string, person: Person): boolean {
34
+ const candidateTokens = new Set(normalizedCandidateName.split(/\s+/).filter(t => t.length >= 3));
35
+ if (candidateTokens.size === 0) return false;
36
+ const personStrings = [normalizeForMatch(person.name), ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value))];
37
+ for (const s of personStrings) {
38
+ for (const tok of s.split(/\s+/)) {
39
+ if (tok.length >= 3 && candidateTokens.has(tok)) return true;
40
+ }
41
+ }
42
+ return false;
43
+ }
44
+
30
45
  function matchPersonCandidate(
31
46
  candidateName: string,
32
47
  candidateIdentifiers: PersonIdentifier[],
33
48
  people: Person[]
34
- ): Person[] {
49
+ ): PersonMatch[] {
35
50
  const normName = normalizeForMatch(candidateName);
36
- const matched = new Set<Person>();
51
+ const matched = new Map<Person, PersonMatchStrength>();
37
52
 
38
53
  // Step 1: Exact match on any identifier value (type-agnostic)
39
54
  for (const person of people) {
@@ -41,19 +56,24 @@ function matchPersonCandidate(
41
56
  ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
42
57
  normalizeForMatch(person.name),
43
58
  ];
44
- if (allValues.includes(normName)) matched.add(person);
59
+ if (allValues.includes(normName)) matched.set(person, 'strong');
45
60
  }
46
61
  // Also check scan-extracted identifiers against existing identifier values
47
62
  for (const scanId of candidateIdentifiers) {
48
63
  const normVal = normalizeForMatch(scanId.value);
49
64
  for (const person of people) {
50
65
  if ((person.identifiers ?? []).some(i => normalizeForMatch(i.value) === normVal)) {
51
- matched.add(person);
66
+ // Corroboration gate (#78 C1): a scan-extracted identifier only STRONG-binds when the
67
+ // candidate's name shares a token with the match. A bare identifier hit with zero name
68
+ // overlap is the cross-attribution signature (one person's handle on another's record),
69
+ // so it drops to WEAK and must clear the cosine gate — or become a new record.
70
+ const strength: PersonMatchStrength = sharesNameToken(normName, person) ? 'strong' : 'weak';
71
+ if (matched.get(person) !== 'strong') matched.set(person, strength);
52
72
  }
53
73
  }
54
74
  }
55
75
 
56
- if (matched.size > 0) return [...matched];
76
+ if (matched.size > 0) return [...matched].map(([person, strength]) => ({ person, strength }));
57
77
 
58
78
  // Step 2: Fuzzy match — skip for short names (< 6 chars): "mike"↔"jake" = 2 edits, false positive.
59
79
  if (normName.length >= 6) {
@@ -63,11 +83,11 @@ function matchPersonCandidate(
63
83
  ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
64
84
  normalizeForMatch(person.name),
65
85
  ];
66
- if (allValues.some(v => levenshtein(normName, v) <= threshold)) matched.add(person);
86
+ if (allValues.some(v => levenshtein(normName, v) <= threshold)) matched.set(person, 'weak');
67
87
  }
68
88
  }
69
89
 
70
- if (matched.size > 0) return [...matched];
90
+ if (matched.size > 0) return [...matched].map(([person, strength]) => ({ person, strength }));
71
91
 
72
92
  // Step 2.5: First-name match — "Lucas Jeremy Scherer" should find "Lucas".
73
93
  // Only fires when first word is >= 4 chars to avoid short-name collisions.
@@ -78,11 +98,11 @@ function matchPersonCandidate(
78
98
  normalizeForMatch(person.name),
79
99
  ...(person.identifiers ?? []).map(i => normalizeForMatch(i.value)),
80
100
  ];
81
- if (allNames.some(n => n.split(/\s+/)[0] === candidateFirstWord)) matched.add(person);
101
+ if (allNames.some(n => n.split(/\s+/)[0] === candidateFirstWord)) matched.set(person, 'weak');
82
102
  }
83
103
  }
84
104
 
85
- return [...matched];
105
+ return [...matched].map(([person, strength]) => ({ person, strength }));
86
106
  }
87
107
 
88
108
  export async function handleFactFind(response: LLMResponse, state: StateManager): Promise<void> {
@@ -182,6 +202,27 @@ export async function handleHumanTopicScan(response: LLMResponse, state: StateMa
182
202
  console.log(`[handleHumanTopicScan] Queued ${result.topics.length} topic(s) for matching`);
183
203
  }
184
204
 
205
+ async function confirmMatchByCosine(
206
+ person: Person,
207
+ candidate: { name: string; relationship?: string; description?: string },
208
+ threshold: number
209
+ ): Promise<Person | null> {
210
+ if (!person.embedding || person.embedding.length === 0) return null;
211
+ try {
212
+ const embeddingService = getEmbeddingService();
213
+ const candidateVector = await embeddingService.embed(getPersonEmbeddingText({
214
+ name: candidate.name, relationship: candidate.relationship, description: candidate.description,
215
+ }));
216
+ const sim = cosineSimilarity(person.embedding, candidateVector);
217
+ if (sim >= threshold) return person;
218
+ console.debug(`[handleHumanPersonScan] Weak single-match "${candidate.name}" → "${person.name}" rejected (cosine ${sim.toFixed(3)} < ${threshold}) — new record`);
219
+ return null;
220
+ } catch (err) {
221
+ console.warn(`[handleHumanPersonScan] Weak-match cosine failed for "${candidate.name}":`, err);
222
+ return null;
223
+ }
224
+ }
225
+
185
226
  export async function handleHumanPersonScan(response: LLMResponse, state: StateManager): Promise<void> {
186
227
  const result = response.parsed as PersonScanResult | undefined;
187
228
 
@@ -213,7 +254,12 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
213
254
  let matchedPerson: Person | null = null;
214
255
 
215
256
  if (matches.length === 1) {
216
- matchedPerson = matches[0];
257
+ const { person, strength } = matches[0];
258
+ if (strength === 'strong') {
259
+ matchedPerson = person;
260
+ } else {
261
+ matchedPerson = await confirmMatchByCosine(person, candidate, MULTI_MATCH_SIMILARITY_THRESHOLD);
262
+ }
217
263
  } else if (matches.length > 1) {
218
264
  try {
219
265
  const embeddingService = getEmbeddingService();
@@ -224,7 +270,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
224
270
  });
225
271
  const candidateVector = await embeddingService.embed(candidateText);
226
272
  let bestSimilarity = MULTI_MATCH_SIMILARITY_THRESHOLD;
227
- for (const person of matches) {
273
+ for (const { person } of matches) {
228
274
  if (person.embedding) {
229
275
  const sim = cosineSimilarity(person.embedding, candidateVector);
230
276
  if (sim > bestSimilarity) {
@@ -238,7 +284,7 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
238
284
  }
239
285
  } catch (err) {
240
286
  console.warn(`[handleHumanPersonScan] Multi-match embedding failed for "${candidate.name}", using first match:`, err);
241
- matchedPerson = matches[0];
287
+ matchedPerson = matches[0].person;
242
288
  }
243
289
  } else {
244
290
  // Step 3: relationship filter → uniqueness match or cosine on the relevant subset.
@@ -253,10 +299,13 @@ export async function handleHumanPersonScan(response: LLMResponse, state: StateM
253
299
  const normExistingName = normalizeForMatch(existing.name);
254
300
  const isUnknownPlaceholder = normExistingName === 'unknown' || normExistingName === normRel;
255
301
  const isSingleton = SINGLETON_RELATIONSHIPS.has(normRel!);
256
- if (isUnknownPlaceholder || isSingleton) {
302
+ if (isSingleton) {
257
303
  matchedPerson = existing;
258
- const reason = isUnknownPlaceholder ? 'unnamed placeholder' : 'singleton relationship';
259
- console.debug(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" "${existing.name}" (sole ${candidate.relationship}, ${reason})`);
304
+ console.debug(`[handleHumanPersonScan] Relationship unique match: "${candidate.name}" "${existing.name}" (sole ${candidate.relationship}, singleton relationship)`);
305
+ } else if (isUnknownPlaceholder) {
306
+ // M1 (deferred, #78): a placeholder with no embedding cannot be confirmed here and will fork a new record instead of promoting. Acceptable under the dupe-tolerant policy; revisit with embedding backfill.
307
+ matchedPerson = await confirmMatchByCosine(existing, candidate, ZERO_MATCH_COSINE_THRESHOLD);
308
+ console.debug(`[handleHumanPersonScan] Relationship unique match gated by cosine: "${candidate.name}" → "${existing.name}" (sole ${candidate.relationship}, unnamed placeholder) — ${matchedPerson ? 'confirmed' : 'rejected, new record'}`);
260
309
  }
261
310
  } else {
262
311
  // N>1 same relationship → cosine within that subset.
@@ -282,7 +282,7 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
282
282
  value: i.value,
283
283
  ...(i.is_primary ? { is_primary: i.is_primary } : {}),
284
284
  })),
285
- state
285
+ state.persona_getAll()
286
286
  );
287
287
  const allCandidateIds = [...llmIdentifiers, ...candidateIdentifiers];
288
288
  if (allCandidateIds.length === 0) {
@@ -303,7 +303,7 @@ export async function handlePersonUpdate(response: LLMResponse, state: StateMana
303
303
  ...i,
304
304
  type: normalizeIdentifierType(i.type, state),
305
305
  })),
306
- state
306
+ state.persona_getAll()
307
307
  );
308
308
  for (const id of sanitizedToAdd) {
309
309
  if (!base.some(e => e.value === id.value)) {
@@ -607,17 +607,6 @@ export function queuePersonUpdate(
607
607
 
608
608
  const candidateIdentifiers = context.candidateIdentifiers ?? [];
609
609
 
610
- if (!isNewItem && existingItem && candidateIdentifiers.length > 0) {
611
- const merged = [...(existingItem.identifiers ?? [])];
612
- for (const ci of candidateIdentifiers) {
613
- if (!merged.some(ei => ei.value === ci.value)) {
614
- merged.push(ci);
615
- }
616
- }
617
- existingItem = { ...existingItem, identifiers: merged };
618
- state.human_person_upsert(existingItem);
619
- }
620
-
621
610
  const userIdentifierTypes = [...new Set(
622
611
  state.getHuman().people
623
612
  .flatMap(p => (p.identifiers ?? []).map(i => i.type))
@@ -642,6 +631,7 @@ export function queuePersonUpdate(
642
631
  persona_name: chunk.channelDisplayName,
643
632
  participant_context: buildParticipantContext(primaryPersonaIdForUpdate, state),
644
633
  known_identifier_types: userIdentifierTypes,
634
+ suggested_identifiers: !isNewItem ? candidateIdentifiers : undefined,
645
635
  });
646
636
 
647
637
  state.queue_enqueue({
@@ -128,15 +128,26 @@ export async function updatePersona(
128
128
  const persona = sm.persona_getById(personaId);
129
129
  if (!persona) return false;
130
130
 
131
- if ('long_description' in updates) {
132
- const merged = { ...persona, ...updates };
133
- const embedding = await computePersonaDescriptionEmbedding(merged);
134
- if (embedding) {
135
- updates = { ...updates, description_embedding: embedding };
136
- }
137
- }
131
+ const shouldRefreshDescriptionEmbedding = 'long_description' in updates;
132
+ const mergedForEmbedding = shouldRefreshDescriptionEmbedding
133
+ ? { ...persona, ...updates }
134
+ : null;
138
135
 
139
136
  sm.persona_update(personaId, updates);
137
+
138
+ if (shouldRefreshDescriptionEmbedding && mergedForEmbedding) {
139
+ void computePersonaDescriptionEmbedding(mergedForEmbedding).then((embedding) => {
140
+ if (!embedding) return;
141
+ const current = sm.persona_getById(personaId);
142
+ if (!current) return;
143
+ if (
144
+ current.long_description !== mergedForEmbedding.long_description ||
145
+ current.short_description !== mergedForEmbedding.short_description
146
+ ) return;
147
+ sm.persona_update(personaId, { description_embedding: embedding });
148
+ });
149
+ }
150
+
140
151
  return true;
141
152
  }
142
153