@modusensus/dsh-mneme 0.2.11 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/store.js CHANGED
@@ -118,6 +118,52 @@ CREATE TABLE IF NOT EXISTS conflict_pending (
118
118
  resolved_winner TEXT
119
119
  );
120
120
  CREATE INDEX IF NOT EXISTS idx_conflict_pending_unresolved ON conflict_pending(resolved_at);
121
+
122
+ -- entity gene (v0.3.0): named entities mentioned across memories, with
123
+ -- time-boxed attributes (valid_from → valid_until) and typed relations.
124
+ -- Attributes follow the snapshot style: saveAttr invalidates the previous
125
+ -- value for the same entity+key before inserting a new row, so the current
126
+ -- value is always the row with valid_until IS NULL.
127
+ CREATE TABLE IF NOT EXISTS entities (
128
+ id TEXT PRIMARY KEY,
129
+ name TEXT NOT NULL,
130
+ type TEXT,
131
+ first_seen TEXT NOT NULL,
132
+ last_seen TEXT NOT NULL,
133
+ mention_count INTEGER DEFAULT 1,
134
+ canonical_memory_id TEXT
135
+ );
136
+ CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
137
+ CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type);
138
+
139
+ CREATE TABLE IF NOT EXISTS entity_attrs (
140
+ id TEXT PRIMARY KEY,
141
+ entity_id TEXT NOT NULL,
142
+ attr_key TEXT NOT NULL,
143
+ attr_value TEXT NOT NULL,
144
+ memory_id TEXT,
145
+ valid_from TEXT NOT NULL,
146
+ valid_until TEXT,
147
+ confidence REAL DEFAULT 1.0,
148
+ source TEXT
149
+ );
150
+ CREATE INDEX IF NOT EXISTS idx_attrs_entity ON entity_attrs(entity_id);
151
+ CREATE INDEX IF NOT EXISTS idx_attrs_key ON entity_attrs(attr_key);
152
+ CREATE INDEX IF NOT EXISTS idx_attrs_valid ON entity_attrs(valid_from, valid_until);
153
+ CREATE INDEX IF NOT EXISTS idx_attrs_memory ON entity_attrs(memory_id);
154
+
155
+ CREATE TABLE IF NOT EXISTS entity_relations (
156
+ id TEXT PRIMARY KEY,
157
+ from_entity TEXT NOT NULL,
158
+ to_entity TEXT NOT NULL,
159
+ relation_type TEXT NOT NULL,
160
+ memory_id TEXT,
161
+ created_at TEXT NOT NULL,
162
+ metadata TEXT
163
+ );
164
+ CREATE INDEX IF NOT EXISTS idx_relations_from ON entity_relations(from_entity);
165
+ CREATE INDEX IF NOT EXISTS idx_relations_to ON entity_relations(to_entity);
166
+ CREATE INDEX IF NOT EXISTS idx_relations_type ON entity_relations(relation_type);
121
167
  `;
122
168
 
123
169
  const TYPES = new Set(["preference", "project", "decision", "history", "summary"]);
@@ -228,6 +274,55 @@ function toRecallRun(row) {
228
274
  };
229
275
  }
230
276
 
277
+ function toEntity(row) {
278
+ if (!row) return undefined;
279
+ return {
280
+ id: row.id,
281
+ name: row.name,
282
+ type: row.type ?? undefined,
283
+ first_seen: row.first_seen,
284
+ last_seen: row.last_seen,
285
+ mention_count: row.mention_count,
286
+ canonical_memory_id: row.canonical_memory_id ?? undefined
287
+ };
288
+ }
289
+
290
+ function toAttr(row) {
291
+ if (!row) return undefined;
292
+ return {
293
+ id: row.id,
294
+ entity_id: row.entity_id,
295
+ attr_key: row.attr_key,
296
+ attr_value: row.attr_value,
297
+ memory_id: row.memory_id ?? undefined,
298
+ valid_from: row.valid_from,
299
+ valid_until: row.valid_until ?? undefined,
300
+ confidence: row.confidence,
301
+ source: row.source ?? undefined
302
+ };
303
+ }
304
+
305
+ function toRelation(row) {
306
+ if (!row) return undefined;
307
+ let metadata;
308
+ if (row.metadata != null) {
309
+ try {
310
+ metadata = JSON.parse(row.metadata);
311
+ } catch {
312
+ metadata = row.metadata;
313
+ }
314
+ }
315
+ return {
316
+ id: row.id,
317
+ from_entity: row.from_entity,
318
+ to_entity: row.to_entity,
319
+ relation_type: row.relation_type,
320
+ memory_id: row.memory_id ?? undefined,
321
+ created_at: row.created_at,
322
+ metadata
323
+ };
324
+ }
325
+
231
326
  function parseJsonArray(raw) {
232
327
  try {
233
328
  const arr = JSON.parse(raw);
@@ -805,6 +900,190 @@ export function createStore(path) {
805
900
  return stats;
806
901
  }
807
902
 
903
+ // --- entity gene: named entities + time-boxed attrs + relations (v0.3.0) --
904
+
905
+ /**
906
+ * Create a named entity. A fresh mention always records first_seen = now;
907
+ * repeated sightings should call updateEntity (which bumps mention_count and
908
+ * refreshes last_seen) rather than creating duplicate rows.
909
+ */
910
+ function createEntity({ name, type }) {
911
+ const id = randomUUID();
912
+ const now = nowIso();
913
+ db.prepare(
914
+ `INSERT INTO entities (id, name, type, first_seen, last_seen, mention_count, canonical_memory_id)
915
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
916
+ ).run(id, name, type ?? null, now, now, 1, null);
917
+ return toEntity(db.prepare("SELECT * FROM entities WHERE id = ?").get(id));
918
+ }
919
+
920
+ function findEntityByName(name) {
921
+ return toEntity(db.prepare("SELECT * FROM entities WHERE name = ?").get(name));
922
+ }
923
+
924
+ function findEntityById(id) {
925
+ return toEntity(db.prepare("SELECT * FROM entities WHERE id = ?").get(id));
926
+ }
927
+
928
+ /**
929
+ * Apply a partial update to an entity, always refreshing last_seen. The
930
+ * mention counter increments on every sighting unless the caller overrides
931
+ * it explicitly via patch.mention_count (e.g. to correct a count).
932
+ */
933
+ function updateEntity(id, patch) {
934
+ const old = findEntityById(id);
935
+ if (!old) return undefined;
936
+ const has = (k) => Object.prototype.hasOwnProperty.call(patch, k);
937
+ const name = has("name") ? patch.name : old.name;
938
+ const type = has("type") ? patch.type : old.type;
939
+ const canonical_memory_id = has("canonical_memory_id")
940
+ ? patch.canonical_memory_id
941
+ : old.canonical_memory_id;
942
+ const mention_count = has("mention_count")
943
+ ? patch.mention_count
944
+ : (old.mention_count ?? 1) + 1;
945
+ const now = nowIso();
946
+ db.prepare(
947
+ `UPDATE entities SET name = ?, type = ?, last_seen = ?, mention_count = ?, canonical_memory_id = ? WHERE id = ?`
948
+ ).run(name, type ?? null, now, mention_count, canonical_memory_id ?? null, id);
949
+ return findEntityById(id);
950
+ }
951
+
952
+ /**
953
+ * Record an attribute value for an entity. The previous value for the same
954
+ * entity+key is invalidated (valid_until = now) before the new row is
955
+ * inserted, so exactly one row per entity+key is current (valid_until IS NULL).
956
+ */
957
+ function saveAttr({ entity_id, attr_key, attr_value, memory_id, confidence, source }) {
958
+ const now = nowIso();
959
+ invalidateOldAttr(entity_id, attr_key, now);
960
+ const id = randomUUID();
961
+ db.prepare(
962
+ `INSERT INTO entity_attrs (id, entity_id, attr_key, attr_value, memory_id, valid_from, valid_until, confidence, source)
963
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
964
+ ).run(id, entity_id, attr_key, attr_value, memory_id ?? null, now, null, confidence ?? 1.0, source ?? null);
965
+ return toAttr(db.prepare("SELECT * FROM entity_attrs WHERE id = ?").get(id));
966
+ }
967
+
968
+ /** Mark every currently-valid attr row for entityId+attrKey as expired. Returns rows changed. */
969
+ function invalidateOldAttr(entityId, attrKey, now) {
970
+ return db.prepare(
971
+ `UPDATE entity_attrs SET valid_until = ? WHERE entity_id = ? AND attr_key = ? AND valid_until IS NULL`
972
+ ).run(now, entityId, attrKey).changes;
973
+ }
974
+
975
+ /** Only the live value per attr_key (valid_until IS NULL). */
976
+ function getCurrentAttrs(entityId) {
977
+ return db.prepare(
978
+ "SELECT * FROM entity_attrs WHERE entity_id = ? AND valid_until IS NULL"
979
+ ).all(entityId).map(toAttr);
980
+ }
981
+
982
+ /** Full history per attr_key, oldest first. */
983
+ function getAttrHistory(entityId) {
984
+ return db.prepare(
985
+ "SELECT * FROM entity_attrs WHERE entity_id = ? ORDER BY valid_from"
986
+ ).all(entityId).map(toAttr);
987
+ }
988
+
989
+ /**
990
+ * All attr rows carrying a reference to the given memory (any valid state),
991
+ * oldest first. Used by autoDream's update path to record what an update
992
+ * superseded (v0.3.0 Phase 4 / 4.3.1).
993
+ */
994
+ function getAttrsByMemory(memoryId) {
995
+ return db.prepare(
996
+ "SELECT * FROM entity_attrs WHERE memory_id = ? ORDER BY valid_from ASC"
997
+ ).all(memoryId).map(toAttr);
998
+ }
999
+
1000
+ /**
1001
+ * Memories carrying a currently-valid attr matching key=value (deduped).
1002
+ * When value is empty/undefined, the attr_value filter is dropped and every
1003
+ * currently-valid memory for that attr_key is returned — the "attr:key"
1004
+ * (no =value) contract, v0.3.0. Only live rows (valid_until IS NULL) with a
1005
+ * memory reference participate, and each memory appears at most once.
1006
+ */
1007
+ function findMemoriesByAttr(key, value) {
1008
+ const empty = value === undefined || value === null || value === "";
1009
+ const sql = empty
1010
+ ? `SELECT DISTINCT memory_id FROM entity_attrs
1011
+ WHERE attr_key = ? AND valid_until IS NULL
1012
+ AND memory_id IS NOT NULL AND memory_id != ''`
1013
+ : `SELECT DISTINCT memory_id FROM entity_attrs
1014
+ WHERE attr_key = ? AND attr_value = ? AND valid_until IS NULL
1015
+ AND memory_id IS NOT NULL AND memory_id != ''`;
1016
+ const params = empty ? [key] : [key, value];
1017
+ const rows = db.prepare(sql).all(...params);
1018
+ const memories = [];
1019
+ const stmt = db.prepare("SELECT * FROM memories WHERE id = ?");
1020
+ for (const { memory_id } of rows) {
1021
+ const row = stmt.get(memory_id);
1022
+ if (row) memories.push(toRow(row));
1023
+ }
1024
+ return memories;
1025
+ }
1026
+
1027
+ /**
1028
+ * Record a typed relation between two entities. metadata (optional) is a
1029
+ * free-form JSON blob describing the relation. Relations are append-only.
1030
+ */
1031
+ function saveRelation({ from_entity, to_entity, relation_type, memory_id, metadata }) {
1032
+ const id = randomUUID();
1033
+ const now = nowIso();
1034
+ const metaStr = metadata === undefined
1035
+ ? null
1036
+ : typeof metadata === "string"
1037
+ ? metadata
1038
+ : JSON.stringify(metadata);
1039
+ db.prepare(
1040
+ `INSERT INTO entity_relations (id, from_entity, to_entity, relation_type, memory_id, created_at, metadata)
1041
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
1042
+ ).run(id, from_entity, to_entity, relation_type, memory_id ?? null, now, metaStr);
1043
+ return toRelation(db.prepare("SELECT * FROM entity_relations WHERE id = ?").get(id));
1044
+ }
1045
+
1046
+ /**
1047
+ * Re-point every attr row whose memory_id is fromMemoryId to toMemoryId
1048
+ * (autoDream merge migration, v0.3.0 Phase 4 / 4.3.2). When the keeper
1049
+ * already carries a live attr for the same entity+key, the source row is
1050
+ * superseded and invalidated instead (the keeper's value wins). Returns
1051
+ * { migrated, invalidated }.
1052
+ */
1053
+ function migrateAttrsToMemory(fromMemoryId, toMemoryId, now) {
1054
+ let migrated = 0;
1055
+ let invalidated = 0;
1056
+ const attrs = db.prepare(
1057
+ "SELECT * FROM entity_attrs WHERE memory_id = ?"
1058
+ ).all(fromMemoryId);
1059
+ for (const attr of attrs) {
1060
+ // 仅当 keeper 已有同 entity+key 的当前有效属性才视为被替代(限定 memory_id,
1061
+ // 避免把 loser 自身的 live 行误判为 keeper 行)。
1062
+ const keeperLive = db.prepare(
1063
+ "SELECT id FROM entity_attrs WHERE entity_id = ? AND attr_key = ? AND valid_until IS NULL AND memory_id = ?"
1064
+ ).get(attr.entity_id, attr.attr_key, toMemoryId);
1065
+ if (keeperLive) {
1066
+ db.prepare(
1067
+ "UPDATE entity_attrs SET valid_until = ? WHERE id = ?"
1068
+ ).run(now, attr.id);
1069
+ invalidated++;
1070
+ } else {
1071
+ db.prepare(
1072
+ "UPDATE entity_attrs SET memory_id = ? WHERE id = ?"
1073
+ ).run(toMemoryId, attr.id);
1074
+ migrated++;
1075
+ }
1076
+ }
1077
+ return { migrated, invalidated };
1078
+ }
1079
+
1080
+ /** Relations where the entity appears on either side (from or to). */
1081
+ function getRelations(entityId) {
1082
+ return db.prepare(
1083
+ "SELECT * FROM entity_relations WHERE from_entity = ? OR to_entity = ?"
1084
+ ).all(entityId, entityId).map(toRelation);
1085
+ }
1086
+
808
1087
  return {
809
1088
  db,
810
1089
  count,
@@ -840,6 +1119,19 @@ export function createStore(path) {
840
1119
  listConflictPending,
841
1120
  resolveConflictPending,
842
1121
  countConflictPending,
1122
+ createEntity,
1123
+ findEntityByName,
1124
+ findEntityById,
1125
+ updateEntity,
1126
+ saveAttr,
1127
+ invalidateOldAttr,
1128
+ getCurrentAttrs,
1129
+ getAttrHistory,
1130
+ getAttrsByMemory,
1131
+ findMemoriesByAttr,
1132
+ saveRelation,
1133
+ migrateAttrsToMemory,
1134
+ getRelations,
843
1135
  close() {
844
1136
  db.close();
845
1137
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@modusensus/dsh-mneme",
3
3
  "description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
4
- "version": "0.2.11",
4
+ "version": "0.3.1",
5
5
  "license": "MIT",
6
6
  "type": "module",
7
7
  "main": "lib/index.js",
package/src/config.js CHANGED
@@ -75,4 +75,18 @@ export const Config = z.object({
75
75
  conflictFreezeEnabled: z.boolean().default(false),
76
76
  // Maximum number of frozen conflicts to keep pending for manual review.
77
77
  conflictFreezeMaxPending: z.natural().min(1).max(1000).default(100),
78
+
79
+ // --- entity gene (v0.3.0) -----------------------------------------------
80
+ // Opt-in: when false (default) nothing in the pipeline extracts entities.
81
+ // The storage layer (entities/entity_attrs/entity_relations tables + CRUD)
82
+ // is always available regardless of this flag.
83
+ entityExtractionEnabled: z.boolean().default(false),
84
+ // Optional model override for entity extraction; empty = use the caller's
85
+ // default provider/model.
86
+ entityExtractionModel: z.string().default(""),
87
+ // Cap on entities per extraction pass and attributes per entity.
88
+ entityExtractionMaxEntities: z.natural().min(1).max(20).default(10),
89
+ entityExtractionMaxAttrs: z.natural().min(1).max(50).default(20),
90
+ // Prefix/semantic search over entity names (used by recall).
91
+ entitySearchEnabled: z.boolean().default(true),
78
92
  });
@@ -167,7 +167,7 @@ function casGuard(service, snapshot, ids) {
167
167
  * committed - the decisions that actually landed, for outcome/receipt based
168
168
  * on real committed sub-steps rather than the raw LLM list.
169
169
  */
170
- export function applyDecisions(decisions, service, logger = null, snapshot = null) {
170
+ export function applyDecisions(decisions, service, logger = null, snapshot = null, config = {}) {
171
171
  let applied = 0;
172
172
  const conflicts = [];
173
173
  const failures = [];
@@ -180,7 +180,7 @@ export function applyDecisions(decisions, service, logger = null, snapshot = nul
180
180
  committed.push({ action: "keep", ids: d.ids });
181
181
  continue;
182
182
  }
183
- const outcome = applyOne(d, service, snapshot);
183
+ const outcome = applyOne(d, service, snapshot, config);
184
184
  if (outcome === "skipped") continue;
185
185
  applied += outcome.applied;
186
186
  committed.push(outcome.committed);
@@ -197,12 +197,12 @@ export function applyDecisions(decisions, service, logger = null, snapshot = nul
197
197
  return { applied, conflicts, failures, committed };
198
198
  }
199
199
 
200
- function applyOne(d, service, snapshot) {
200
+ function applyOne(d, service, snapshot, config = {}) {
201
201
  switch (d.action) {
202
202
  case "archive": return applyArchive(d, service, snapshot);
203
- case "merge": return applyMerge(d, service, snapshot);
203
+ case "merge": return applyMerge(d, service, snapshot, config);
204
204
  case "conflict": return applyConflict(d, service, snapshot);
205
- default: return applyUpdate(d, service, snapshot);
205
+ default: return applyUpdate(d, service, snapshot, config);
206
206
  }
207
207
  }
208
208
 
@@ -222,7 +222,7 @@ function applyArchive(d, service, snapshot) {
222
222
  return { applied: targets.length, committed: { action: "archive", ids: targets } };
223
223
  }
224
224
 
225
- function applyMerge(d, service, snapshot) {
225
+ function applyMerge(d, service, snapshot, config = {}) {
226
226
  const sources = d.ids.filter((id) => id !== d.keepSource);
227
227
  // Idempotent replay: if every other source is already archived, this merge
228
228
  // already landed — skip so a replayed/concurrent decision never double-counts
@@ -241,6 +241,18 @@ function applyMerge(d, service, snapshot) {
241
241
  const mem = service.getById(id);
242
242
  if (mem && !mem.archived) service.setArchived(id, true);
243
243
  }
244
+ // 4.3.2 迁移实体关联(opt-in):将 loser(source)记忆关联的 entity_attrs 的
245
+ // memory_id 迁移到 keeper;keeper 已有同 entity+key 的当前属性时 loser 行被
246
+ // 失效。单个 source 迁移失败只告警,绝不能导致整个 merge 事务回滚(fail-safe)。
247
+ if (config.entityExtractionEnabled && typeof service.migrateAttrsToMemory === "function") {
248
+ for (const id of sources) {
249
+ try {
250
+ service.migrateAttrsToMemory(id, d.keepSource, new Date().toISOString());
251
+ } catch (error) {
252
+ logger?.warn?.(`dsh-mneme dream: failed to migrate attrs from ${id} to ${d.keepSource}: ${error.message}`);
253
+ }
254
+ }
255
+ }
244
256
  });
245
257
  return {
246
258
  applied: 1,
@@ -269,7 +281,7 @@ function applyConflict(d, service, snapshot) {
269
281
  return { applied: 1, committed: { action: "conflict", winner: d.winner, loser: d.loser, count_before: 2, count_after: 1 } };
270
282
  }
271
283
 
272
- function applyUpdate(d, service, snapshot) {
284
+ function applyUpdate(d, service, snapshot, config = {}) {
273
285
  const id = d.ids[0];
274
286
  const mem = service.getById(id);
275
287
  if (!mem || mem.archived) return "skipped";
@@ -288,5 +300,25 @@ function applyUpdate(d, service, snapshot) {
288
300
  importance: d.importance ?? cur.importance
289
301
  });
290
302
  });
303
+ // 4.3.1 supersedes 关系(opt-in):事务提交成功后,为该记忆关联的每条实体属性
304
+ // 建立自引用 supersedes 关系,表示"此属性版本已被替代"。仅记录、绝不阻断主流程
305
+ // (fail-safe):记录失败只告警,update 本身照常生效。
306
+ if (config.entityExtractionEnabled && typeof service.saveRelation === "function" && typeof service.getAttrsByMemory === "function") {
307
+ try {
308
+ const oldAttrs = service.getAttrsByMemory(id);
309
+ for (const attr of oldAttrs) {
310
+ if (!attr.entity_id) continue;
311
+ service.saveRelation({
312
+ from_entity: attr.entity_id,
313
+ to_entity: attr.entity_id,
314
+ relation_type: "supersedes",
315
+ memory_id: id,
316
+ metadata: JSON.stringify({ attr_key: attr.attr_key, old_value: attr.attr_value })
317
+ });
318
+ }
319
+ } catch (error) {
320
+ logger?.warn?.(`dsh-mneme dream: failed to record supersedes relations for ${id}: ${error.message}`);
321
+ }
322
+ }
291
323
  return { applied: 1, committed: { action: "update", ids: [id], title: d.title, content: d.content, importance: d.importance, count_before: 1, count_after: 1 } };
292
324
  }
package/src/dream.js CHANGED
@@ -541,7 +541,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
541
541
  // a target changed during the LLM call is skipped and reported as a
542
542
  // conflict instead of being overwritten (item ①). Frozen conflicts are
543
543
  // excluded from this list (they are parked, not applied).
544
- const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot);
544
+ const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot, config);
545
545
  // Per-record receipt chain: one row per actually-committed merge/conflict/
546
546
  // update verdict, stamped with the decision-basis digest + idempotency
547
547
  // counters (count_before → count_after). Written here, before the run audit
@@ -0,0 +1,242 @@
1
+ const VALID_TYPES = new Set(["person", "project", "concept", "technology", "organization"]);
2
+ const VALID_RELATIONS = new Set(["uses", "depends_on", "part_of", "related_to"]);
3
+ const MAX_MEMORY_CHARS = 4000;
4
+
5
+ function buildSystemPrompt(config) {
6
+ const maxEntities = config.entityExtractionMaxEntities ?? 10;
7
+ const maxAttrs = config.entityExtractionMaxAttrs ?? 20;
8
+
9
+ return `You are an entity extraction engine. Extract named entities, their attributes, and relations from the given text.
10
+
11
+ RULES:
12
+ - Entities are concrete people, projects, technologies, concepts, or organizations.
13
+ - Attributes are characteristic states of an entity. Only extract attributes explicitly mentioned in the text.
14
+ - Do NOT infer, guess, or hallucinate. Only extract what is clearly stated.
15
+ - Use canonical names (full name / primary name). Do NOT merge synonyms — different spellings of the same person are different entities.
16
+ - Output MUST be strict JSON with this exact structure:
17
+ {
18
+ "entities": [
19
+ {"name": "string", "type": "person|project|concept|technology|organization", "attrs": [{"key": "string", "value": "string", "confidence": 0.9}]}
20
+ ],
21
+ "relations": [
22
+ {"from": "entityName", "to": "entityName", "type": "uses|depends_on|part_of|related_to"}
23
+ ]
24
+ }
25
+
26
+ CONSTRAINTS:
27
+ - "type" must be one of: person, project, concept, technology, organization. If unsure, use "concept".
28
+ - "relations" from/to must reference entity names from the "entities" list.
29
+ - Maximum ${maxEntities} entities. Maximum ${maxAttrs} attributes per entity. Truncate if exceeded.
30
+ - Only include entity-related attributes. Ignore irrelevant miscellaneous details.
31
+ - Return ONLY the JSON object. No explanations, no markdown, no code fences.`;
32
+ }
33
+
34
+ function buildUserMessage(memoryText) {
35
+ const truncated = memoryText.length > MAX_MEMORY_CHARS
36
+ ? memoryText.slice(0, MAX_MEMORY_CHARS) + "..."
37
+ : memoryText;
38
+ return truncated;
39
+ }
40
+
41
+ function extractJsonFromText(text) {
42
+ if (!text || typeof text !== "string") return null;
43
+
44
+ // Try direct parse first
45
+ try {
46
+ return JSON.parse(text);
47
+ } catch {
48
+ // fall through
49
+ }
50
+
51
+ // Try to find first {...} block
52
+ const match = text.match(/\{[\s\S]*\}/);
53
+ if (match) {
54
+ try {
55
+ return JSON.parse(match[0]);
56
+ } catch {
57
+ return null;
58
+ }
59
+ }
60
+
61
+ return null;
62
+ }
63
+
64
+ function sanitizeType(type) {
65
+ if (VALID_TYPES.has(type)) return type;
66
+ return "concept";
67
+ }
68
+
69
+ function sanitizeConfidence(conf) {
70
+ const num = Number(conf);
71
+ if (Number.isNaN(num)) return 0.9;
72
+ return Math.min(1, Math.max(0, num));
73
+ }
74
+
75
+ function sanitizeRelationType(type) {
76
+ if (VALID_RELATIONS.has(type)) return type;
77
+ return "related_to";
78
+ }
79
+
80
+ function sanitizeExtractedData(data, config) {
81
+ const maxEntities = config.entityExtractionMaxEntities ?? 10;
82
+ const maxAttrs = config.entityExtractionMaxAttrs ?? 20;
83
+
84
+ if (!data || !Array.isArray(data.entities)) {
85
+ throw new Error("Invalid extraction: missing entities array");
86
+ }
87
+
88
+ const entities = [];
89
+ const seenNames = new Set();
90
+
91
+ for (const rawEntity of data.entities.slice(0, maxEntities)) {
92
+ if (!rawEntity || typeof rawEntity.name !== "string" || !rawEntity.name.trim()) continue;
93
+
94
+ const name = rawEntity.name.trim();
95
+ if (seenNames.has(name)) continue;
96
+ seenNames.add(name);
97
+
98
+ const attrs = [];
99
+ if (Array.isArray(rawEntity.attrs)) {
100
+ for (const rawAttr of rawEntity.attrs.slice(0, maxAttrs)) {
101
+ if (!rawAttr || typeof rawAttr.key !== "string" || typeof rawAttr.value !== "string") continue;
102
+ attrs.push({
103
+ key: rawAttr.key.trim(),
104
+ value: rawAttr.value.trim(),
105
+ confidence: sanitizeConfidence(rawAttr.confidence)
106
+ });
107
+ }
108
+ }
109
+
110
+ entities.push({
111
+ name,
112
+ type: sanitizeType(rawEntity.type),
113
+ attrs
114
+ });
115
+ }
116
+
117
+ const relations = [];
118
+ if (Array.isArray(data.relations)) {
119
+ for (const rawRel of data.relations) {
120
+ if (!rawRel || typeof rawRel.from !== "string" || typeof rawRel.to !== "string") continue;
121
+ if (!seenNames.has(rawRel.from) || !seenNames.has(rawRel.to)) continue;
122
+ relations.push({
123
+ from: rawRel.from,
124
+ to: rawRel.to,
125
+ type: sanitizeRelationType(rawRel.type)
126
+ });
127
+ }
128
+ }
129
+
130
+ return { entities, relations };
131
+ }
132
+
133
+ async function resolveEntity(entity, store) {
134
+ const existing = store.findEntityByName(entity.name);
135
+ if (existing) {
136
+ store.updateEntity(existing.id, {});
137
+ return existing.id;
138
+ }
139
+ const created = store.createEntity({ name: entity.name, type: entity.type });
140
+ return created.id;
141
+ }
142
+
143
+ export async function extractEntities(memory, { store, config, callLLM, logger }) {
144
+ try {
145
+ if (!memory || !memory.content) {
146
+ return { ok: false, error: "Invalid memory: missing content" };
147
+ }
148
+
149
+ const model = config.entityExtractionModel || null;
150
+ const systemPrompt = buildSystemPrompt(config);
151
+ const userText = buildUserMessage(memory.content);
152
+
153
+ const messages = [
154
+ { role: "system", content: [{ type: "text", text: systemPrompt }] },
155
+ { role: "user", content: [{ type: "text", text: userText }] }
156
+ ];
157
+
158
+ const options = model ? { model } : {};
159
+ const llmResponse = await callLLM(messages, options);
160
+
161
+ if (!llmResponse) {
162
+ return { ok: false, error: "LLM returned empty response" };
163
+ }
164
+
165
+ const rawData = extractJsonFromText(llmResponse);
166
+ if (!rawData) {
167
+ return { ok: false, error: "Failed to parse JSON from LLM response" };
168
+ }
169
+
170
+ const { entities, relations } = sanitizeExtractedData(rawData, config);
171
+
172
+ const resolvedEntities = [];
173
+ const entityIdMap = new Map();
174
+ let skipCount = 0;
175
+
176
+ // Resolve entities
177
+ for (const entity of entities) {
178
+ try {
179
+ const entityId = await resolveEntity(entity, store);
180
+ entityIdMap.set(entity.name, entityId);
181
+ resolvedEntities.push({ ...entity, entity_id: entityId });
182
+ } catch (err) {
183
+ skipCount++;
184
+ logger?.warn?.(`[extractor] Failed to resolve entity "${entity.name}":`, err.message);
185
+ }
186
+ }
187
+
188
+ // Record attributes
189
+ const attrs = [];
190
+ for (const entity of resolvedEntities) {
191
+ for (const attr of entity.attrs) {
192
+ try {
193
+ const saved = store.saveAttr({
194
+ entity_id: entity.entity_id,
195
+ attr_key: attr.key,
196
+ attr_value: String(attr.value),
197
+ memory_id: memory.id,
198
+ confidence: attr.confidence,
199
+ source: "llm_extract"
200
+ });
201
+ attrs.push(saved);
202
+ } catch (err) {
203
+ skipCount++;
204
+ logger?.warn?.(`[extractor] Failed to save attr "${attr.key}" for entity "${entity.name}":`, err.message);
205
+ }
206
+ }
207
+ }
208
+
209
+ // Record relations
210
+ const savedRelations = [];
211
+ for (const rel of relations) {
212
+ const fromId = entityIdMap.get(rel.from);
213
+ const toId = entityIdMap.get(rel.to);
214
+ if (!fromId || !toId) continue;
215
+
216
+ try {
217
+ const saved = store.saveRelation({
218
+ from_entity: fromId,
219
+ to_entity: toId,
220
+ relation_type: rel.type,
221
+ memory_id: memory.id,
222
+ metadata: { model: model || "default" }
223
+ });
224
+ savedRelations.push(saved);
225
+ } catch (err) {
226
+ skipCount++;
227
+ logger?.warn?.(`[extractor] Failed to save relation "${rel.from} -> ${rel.to}":`, err.message);
228
+ }
229
+ }
230
+
231
+ return {
232
+ ok: true,
233
+ entities: resolvedEntities,
234
+ attrs,
235
+ relations: savedRelations,
236
+ skipped: skipCount
237
+ };
238
+
239
+ } catch (err) {
240
+ return { ok: false, error: String(err) };
241
+ }
242
+ }