@199-bio/engram 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/graph/index.d.ts.map +1 -1
- package/dist/graph/knowledge-graph.d.ts.map +1 -1
- package/dist/index.js +54 -14
- package/dist/retrieval/hybrid.d.ts.map +1 -1
- package/dist/web/server.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/graph/index.ts +0 -1
- package/src/graph/knowledge-graph.ts +0 -50
- package/src/index.ts +62 -14
- package/src/retrieval/hybrid.ts +8 -5
- package/src/web/server.ts +1 -2
- package/src/graph/extractor.ts +0 -489
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-graph.d.ts","sourceRoot":"","sources":["../../src/graph/knowledge-graph.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,cAAc,EACd,MAAM,EACN,WAAW,EACX,QAAQ,EACT,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-graph.d.ts","sourceRoot":"","sources":["../../src/graph/knowledge-graph.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,cAAc,EACd,MAAM,EACN,WAAW,EACX,QAAQ,EACT,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,iBAAkB,SAAQ,MAAM;IAC/C,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,aAAa,EAAE,KAAK,CAAC,QAAQ,GAAG;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1D,WAAW,EAAE,KAAK,CAAC,QAAQ,GAAG;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,YAAY,EAAE,WAAW,EAAE,CAAC;CAC7B;AAED,qBAAa,cAAc;IACb,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,cAAc;IAItC;;OAEG;IACH,iBAAiB,CACf,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,MAAM,CAAC,MAAM,CAAY,GAC9B,MAAM;IAMT;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI;IAoC5D;;OAEG;IACH,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE;IAI5D;;OAEG;IACH,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAE,MAAY,GAAG,MAAM,EAAE;IAMlE;;OAEG;IACH,cAAc,CACZ,cAAc,EAAE,MAAM,EACtB,OAAO,EAAE,MAAM,EACf,cAAc,CAAC,EAAE,MAAM,EACvB,UAAU,GAAE,MAAY,GACvB,WAAW;IAWd;;OAEG;IACH,eAAe,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,EAAE;IAWtD;;OAEG;IACH,MAAM,CACJ,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACnC,QAAQ;IAgBX;;OAEG;IACH,YAAY,CACV,cAAc,EAAE,MAAM,EACtB,SAAS,GAAE,MAAM,GAAG,IAAI,GAAG,MAAe,GACzC,QAAQ,EAAE;IAWb;;OAEG;IACH,QAAQ,CACN,cAAc,EAAE,MAAM,EACtB,KAAK,GAAE,MAAU,EACjB,aAAa,CAAC,EAAE,MAAM,EAAE,GACvB,cAAc,GAAG,IAAI;IAcxB;;OAEG;IACH,oBAAoB,CAClB,cAAc,EAAE,MAAM,EACtB,KAAK,GAAE,MAAU,GAChB,MAAM,EAAE;CAcZ"}
|
package/dist/index.js
CHANGED
|
@@ -42,7 +42,7 @@ async function initialize() {
|
|
|
42
42
|
// ============ MCP Server ============
|
|
43
43
|
const server = new Server({
|
|
44
44
|
name: "engram",
|
|
45
|
-
version: "0.4.
|
|
45
|
+
version: "0.4.1",
|
|
46
46
|
}, {
|
|
47
47
|
capabilities: {
|
|
48
48
|
tools: {},
|
|
@@ -53,26 +53,46 @@ const server = new Server({
|
|
|
53
53
|
const TOOLS = [
|
|
54
54
|
{
|
|
55
55
|
name: "remember",
|
|
56
|
-
description: "
|
|
56
|
+
description: "Store information with entities and relationships. Extract key people, organizations, and places from the content and pass them as entities. Include relationships between entities when mentioned (e.g., 'works_at', 'lives_in', 'knows').",
|
|
57
57
|
inputSchema: {
|
|
58
58
|
type: "object",
|
|
59
59
|
properties: {
|
|
60
60
|
content: {
|
|
61
61
|
type: "string",
|
|
62
|
-
description: "The information to store
|
|
63
|
-
},
|
|
64
|
-
source: {
|
|
65
|
-
type: "string",
|
|
66
|
-
description: "Source of the memory (e.g., 'conversation', 'note', 'import')",
|
|
67
|
-
default: "conversation",
|
|
62
|
+
description: "The information to store",
|
|
68
63
|
},
|
|
69
64
|
importance: {
|
|
70
65
|
type: "number",
|
|
71
|
-
description: "
|
|
66
|
+
description: "0-1 score. Use 0.8+ for key facts (names, preferences, important events), 0.5 for general info, 0.3- for trivial mentions",
|
|
72
67
|
minimum: 0,
|
|
73
68
|
maximum: 1,
|
|
74
69
|
default: 0.5,
|
|
75
70
|
},
|
|
71
|
+
entities: {
|
|
72
|
+
type: "array",
|
|
73
|
+
description: "Key entities mentioned (people, organizations, places). Only include clear, specific named entities.",
|
|
74
|
+
items: {
|
|
75
|
+
type: "object",
|
|
76
|
+
properties: {
|
|
77
|
+
name: { type: "string", description: "Entity name (e.g., 'Boris Djordjevic', 'Google', 'Paris')" },
|
|
78
|
+
type: { type: "string", enum: ["person", "organization", "place"], description: "Entity type" },
|
|
79
|
+
},
|
|
80
|
+
required: ["name", "type"],
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
relationships: {
|
|
84
|
+
type: "array",
|
|
85
|
+
description: "Relationships between entities mentioned in the content",
|
|
86
|
+
items: {
|
|
87
|
+
type: "object",
|
|
88
|
+
properties: {
|
|
89
|
+
from: { type: "string", description: "Source entity name" },
|
|
90
|
+
to: { type: "string", description: "Target entity name" },
|
|
91
|
+
type: { type: "string", description: "Relationship type (e.g., 'works_at', 'lives_in', 'sibling_of', 'knows')" },
|
|
92
|
+
},
|
|
93
|
+
required: ["from", "to", "type"],
|
|
94
|
+
},
|
|
95
|
+
},
|
|
76
96
|
},
|
|
77
97
|
required: ["content"],
|
|
78
98
|
},
|
|
@@ -168,13 +188,33 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
168
188
|
try {
|
|
169
189
|
switch (name) {
|
|
170
190
|
case "remember": {
|
|
171
|
-
const { content, source = "conversation", importance = 0.5 } = args;
|
|
191
|
+
const { content, source = "conversation", importance = 0.5, entities: providedEntities = [], relationships: providedRelationships = [], } = args;
|
|
172
192
|
// Create memory
|
|
173
193
|
const memory = db.createMemory(content, source, importance);
|
|
174
194
|
// Index for semantic search
|
|
175
195
|
await search.indexMemory(memory);
|
|
176
|
-
//
|
|
177
|
-
const
|
|
196
|
+
// Store Claude-provided entities and link to memory
|
|
197
|
+
const storedEntities = [];
|
|
198
|
+
for (const ent of providedEntities) {
|
|
199
|
+
const entity = graph.getOrCreateEntity(ent.name, ent.type);
|
|
200
|
+
storedEntities.push(entity.name);
|
|
201
|
+
// Create observation linking entity to this memory
|
|
202
|
+
db.addObservation(entity.id, content, memory.id, 1.0);
|
|
203
|
+
}
|
|
204
|
+
// Store Claude-provided relationships
|
|
205
|
+
const storedRelations = [];
|
|
206
|
+
for (const rel of providedRelationships) {
|
|
207
|
+
try {
|
|
208
|
+
// Ensure both entities exist (create if not provided explicitly)
|
|
209
|
+
const fromEntity = graph.getOrCreateEntity(rel.from, "person");
|
|
210
|
+
const toEntity = graph.getOrCreateEntity(rel.to, "person");
|
|
211
|
+
graph.relate(fromEntity.name, toEntity.name, rel.type);
|
|
212
|
+
storedRelations.push(`${rel.from} -[${rel.type}]-> ${rel.to}`);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// Skip invalid relationships
|
|
216
|
+
}
|
|
217
|
+
}
|
|
178
218
|
return {
|
|
179
219
|
content: [
|
|
180
220
|
{
|
|
@@ -182,8 +222,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
182
222
|
text: JSON.stringify({
|
|
183
223
|
success: true,
|
|
184
224
|
memory_id: memory.id,
|
|
185
|
-
|
|
186
|
-
|
|
225
|
+
entities_stored: storedEntities,
|
|
226
|
+
relationships_stored: storedRelations,
|
|
187
227
|
}, null, 2),
|
|
188
228
|
},
|
|
189
229
|
],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/retrieval/hybrid.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAA0B,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/retrieval/hybrid.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAA0B,MAAM,cAAc,CAAC;AAEzF,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,qBAAa,YAAY;IAErB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,SAAS;gBAFT,EAAE,EAAE,cAAc,EAClB,KAAK,EAAE,cAAc,EACrB,SAAS,EAAE,gBAAgB,GAAG,eAAe;IAGvD;;;;;OAKG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,OAAO,CAAC;KACnB,GACL,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAqHhC;;OAEG;YACW,UAAU;IASxB;;OAEG;YACW,cAAc;IAS5B;;OAEG;YACW,WAAW;IAmBzB;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOhD;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAYhD;;OAEG;IACG,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAGvD"}
|
package/dist/web/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/web/server.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAiBtD,UAAU,gBAAgB;IACxB,EAAE,EAAE,cAAc,CAAC;IACnB,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,IAAI,CAAS;gBAET,OAAO,EAAE,gBAAgB;IAO/B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAkB9B,IAAI,IAAI,IAAI;YAOE,aAAa;YA+Bb,SAAS;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/web/server.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAiBtD,UAAU,gBAAgB;IACxB,EAAE,EAAE,cAAc,CAAC;IACnB,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,IAAI,CAAS;gBAET,OAAO,EAAE,gBAAgB;IAO/B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAkB9B,IAAI,IAAI,IAAI;YAOE,aAAa;YA+Bb,SAAS;YAwKT,WAAW;IAgCzB,OAAO,CAAC,SAAS;CAclB"}
|
package/package.json
CHANGED
package/src/graph/index.ts
CHANGED
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
Observation,
|
|
10
10
|
Relation,
|
|
11
11
|
} from "../storage/database.js";
|
|
12
|
-
import { entityExtractor, ExtractedEntity } from "./extractor.js";
|
|
13
12
|
|
|
14
13
|
export interface EntityWithDetails extends Entity {
|
|
15
14
|
observations: Observation[];
|
|
@@ -191,55 +190,6 @@ export class KnowledgeGraph {
|
|
|
191
190
|
};
|
|
192
191
|
}
|
|
193
192
|
|
|
194
|
-
// ============ Auto-extraction ============
|
|
195
|
-
|
|
196
|
-
/**
|
|
197
|
-
* Extract entities from text and add them to the graph
|
|
198
|
-
*/
|
|
199
|
-
extractAndStore(
|
|
200
|
-
text: string,
|
|
201
|
-
sourceMemoryId?: string
|
|
202
|
-
): { entities: Entity[]; observations: Observation[] } {
|
|
203
|
-
const extracted = entityExtractor.extractAll(text);
|
|
204
|
-
const entities: Entity[] = [];
|
|
205
|
-
const observations: Observation[] = [];
|
|
206
|
-
|
|
207
|
-
for (const ext of extracted) {
|
|
208
|
-
// Only store high-confidence extractions
|
|
209
|
-
if (ext.confidence < 0.5) continue;
|
|
210
|
-
|
|
211
|
-
// Get or create entity
|
|
212
|
-
const entity = this.getOrCreateEntity(ext.name, ext.type);
|
|
213
|
-
entities.push(entity);
|
|
214
|
-
|
|
215
|
-
// Create observation linking this entity to the memory content
|
|
216
|
-
const obs = this.db.addObservation(
|
|
217
|
-
entity.id,
|
|
218
|
-
text,
|
|
219
|
-
sourceMemoryId || null,
|
|
220
|
-
ext.confidence
|
|
221
|
-
);
|
|
222
|
-
observations.push(obs);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
// Extract and create relationships
|
|
226
|
-
const relationships = entityExtractor.extractRelationships(text);
|
|
227
|
-
for (const rel of relationships) {
|
|
228
|
-
try {
|
|
229
|
-
// Ensure both entities exist
|
|
230
|
-
const fromEntity = this.getOrCreateEntity(rel.subject, "person");
|
|
231
|
-
const toEntity = this.getOrCreateEntity(rel.object, "person");
|
|
232
|
-
|
|
233
|
-
// Create the relation
|
|
234
|
-
this.db.createRelation(fromEntity.id, toEntity.id, rel.relation);
|
|
235
|
-
} catch {
|
|
236
|
-
// Silently ignore relation creation failures
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
return { entities, observations };
|
|
241
|
-
}
|
|
242
|
-
|
|
243
193
|
/**
|
|
244
194
|
* Find memories related to an entity by traversing the graph
|
|
245
195
|
*/
|
package/src/index.ts
CHANGED
|
@@ -60,7 +60,7 @@ async function initialize(): Promise<void> {
|
|
|
60
60
|
const server = new Server(
|
|
61
61
|
{
|
|
62
62
|
name: "engram",
|
|
63
|
-
version: "0.4.
|
|
63
|
+
version: "0.4.1",
|
|
64
64
|
},
|
|
65
65
|
{
|
|
66
66
|
capabilities: {
|
|
@@ -75,26 +75,46 @@ const TOOLS = [
|
|
|
75
75
|
{
|
|
76
76
|
name: "remember",
|
|
77
77
|
description:
|
|
78
|
-
"
|
|
78
|
+
"Store information with entities and relationships. Extract key people, organizations, and places from the content and pass them as entities. Include relationships between entities when mentioned (e.g., 'works_at', 'lives_in', 'knows').",
|
|
79
79
|
inputSchema: {
|
|
80
80
|
type: "object" as const,
|
|
81
81
|
properties: {
|
|
82
82
|
content: {
|
|
83
83
|
type: "string",
|
|
84
|
-
description: "The information to store
|
|
85
|
-
},
|
|
86
|
-
source: {
|
|
87
|
-
type: "string",
|
|
88
|
-
description: "Source of the memory (e.g., 'conversation', 'note', 'import')",
|
|
89
|
-
default: "conversation",
|
|
84
|
+
description: "The information to store",
|
|
90
85
|
},
|
|
91
86
|
importance: {
|
|
92
87
|
type: "number",
|
|
93
|
-
description: "
|
|
88
|
+
description: "0-1 score. Use 0.8+ for key facts (names, preferences, important events), 0.5 for general info, 0.3- for trivial mentions",
|
|
94
89
|
minimum: 0,
|
|
95
90
|
maximum: 1,
|
|
96
91
|
default: 0.5,
|
|
97
92
|
},
|
|
93
|
+
entities: {
|
|
94
|
+
type: "array",
|
|
95
|
+
description: "Key entities mentioned (people, organizations, places). Only include clear, specific named entities.",
|
|
96
|
+
items: {
|
|
97
|
+
type: "object",
|
|
98
|
+
properties: {
|
|
99
|
+
name: { type: "string", description: "Entity name (e.g., 'Boris Djordjevic', 'Google', 'Paris')" },
|
|
100
|
+
type: { type: "string", enum: ["person", "organization", "place"], description: "Entity type" },
|
|
101
|
+
},
|
|
102
|
+
required: ["name", "type"],
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
relationships: {
|
|
106
|
+
type: "array",
|
|
107
|
+
description: "Relationships between entities mentioned in the content",
|
|
108
|
+
items: {
|
|
109
|
+
type: "object",
|
|
110
|
+
properties: {
|
|
111
|
+
from: { type: "string", description: "Source entity name" },
|
|
112
|
+
to: { type: "string", description: "Target entity name" },
|
|
113
|
+
type: { type: "string", description: "Relationship type (e.g., 'works_at', 'lives_in', 'sibling_of', 'knows')" },
|
|
114
|
+
},
|
|
115
|
+
required: ["from", "to", "type"],
|
|
116
|
+
},
|
|
117
|
+
},
|
|
98
118
|
},
|
|
99
119
|
required: ["content"],
|
|
100
120
|
},
|
|
@@ -194,10 +214,18 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
194
214
|
try {
|
|
195
215
|
switch (name) {
|
|
196
216
|
case "remember": {
|
|
197
|
-
const {
|
|
217
|
+
const {
|
|
218
|
+
content,
|
|
219
|
+
source = "conversation",
|
|
220
|
+
importance = 0.5,
|
|
221
|
+
entities: providedEntities = [],
|
|
222
|
+
relationships: providedRelationships = [],
|
|
223
|
+
} = args as {
|
|
198
224
|
content: string;
|
|
199
225
|
source?: string;
|
|
200
226
|
importance?: number;
|
|
227
|
+
entities?: Array<{ name: string; type: "person" | "organization" | "place" }>;
|
|
228
|
+
relationships?: Array<{ from: string; to: string; type: string }>;
|
|
201
229
|
};
|
|
202
230
|
|
|
203
231
|
// Create memory
|
|
@@ -206,8 +234,28 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
206
234
|
// Index for semantic search
|
|
207
235
|
await search.indexMemory(memory);
|
|
208
236
|
|
|
209
|
-
//
|
|
210
|
-
const
|
|
237
|
+
// Store Claude-provided entities and link to memory
|
|
238
|
+
const storedEntities: string[] = [];
|
|
239
|
+
for (const ent of providedEntities) {
|
|
240
|
+
const entity = graph.getOrCreateEntity(ent.name, ent.type);
|
|
241
|
+
storedEntities.push(entity.name);
|
|
242
|
+
// Create observation linking entity to this memory
|
|
243
|
+
db.addObservation(entity.id, content, memory.id, 1.0);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Store Claude-provided relationships
|
|
247
|
+
const storedRelations: string[] = [];
|
|
248
|
+
for (const rel of providedRelationships) {
|
|
249
|
+
try {
|
|
250
|
+
// Ensure both entities exist (create if not provided explicitly)
|
|
251
|
+
const fromEntity = graph.getOrCreateEntity(rel.from, "person");
|
|
252
|
+
const toEntity = graph.getOrCreateEntity(rel.to, "person");
|
|
253
|
+
graph.relate(fromEntity.name, toEntity.name, rel.type);
|
|
254
|
+
storedRelations.push(`${rel.from} -[${rel.type}]-> ${rel.to}`);
|
|
255
|
+
} catch {
|
|
256
|
+
// Skip invalid relationships
|
|
257
|
+
}
|
|
258
|
+
}
|
|
211
259
|
|
|
212
260
|
return {
|
|
213
261
|
content: [
|
|
@@ -216,8 +264,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
216
264
|
text: JSON.stringify({
|
|
217
265
|
success: true,
|
|
218
266
|
memory_id: memory.id,
|
|
219
|
-
|
|
220
|
-
|
|
267
|
+
entities_stored: storedEntities,
|
|
268
|
+
relationships_stored: storedRelations,
|
|
221
269
|
}, null, 2),
|
|
222
270
|
},
|
|
223
271
|
],
|
package/src/retrieval/hybrid.ts
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
import { EngramDatabase, Memory } from "../storage/database.js";
|
|
7
7
|
import { KnowledgeGraph } from "../graph/knowledge-graph.js";
|
|
8
8
|
import { ColBERTRetriever, SimpleRetriever, SearchResult, Document } from "./colbert.js";
|
|
9
|
-
import { entityExtractor } from "../graph/extractor.js";
|
|
10
9
|
|
|
11
10
|
export interface HybridSearchResult {
|
|
12
11
|
memory: Memory;
|
|
@@ -183,15 +182,19 @@ export class HybridSearch {
|
|
|
183
182
|
}
|
|
184
183
|
|
|
185
184
|
/**
|
|
186
|
-
* Graph-based search: find entities in query, traverse graph
|
|
185
|
+
* Graph-based search: find known entities in query, traverse graph
|
|
187
186
|
*/
|
|
188
187
|
private async searchGraph(query: string): Promise<string[]> {
|
|
189
|
-
//
|
|
190
|
-
const
|
|
188
|
+
// Find known entities whose names appear in the query
|
|
189
|
+
const queryLower = query.toLowerCase();
|
|
190
|
+
const allEntities = this.graph.listEntities(undefined, 500);
|
|
191
|
+
const matchedEntities = allEntities.filter(e =>
|
|
192
|
+
queryLower.includes(e.name.toLowerCase())
|
|
193
|
+
);
|
|
191
194
|
|
|
192
195
|
const memoryIds = new Set<string>();
|
|
193
196
|
|
|
194
|
-
for (const entity of
|
|
197
|
+
for (const entity of matchedEntities) {
|
|
195
198
|
// Find related memory IDs through graph traversal
|
|
196
199
|
const relatedIds = this.graph.findRelatedMemoryIds(entity.name, 2);
|
|
197
200
|
relatedIds.forEach(id => memoryIds.add(id));
|
package/src/web/server.ts
CHANGED
|
@@ -147,9 +147,8 @@ export class EngramWebServer {
|
|
|
147
147
|
const { content, source, importance } = body as any;
|
|
148
148
|
const memory = this.db.createMemory(content, source || "web", importance || 0.5);
|
|
149
149
|
await this.search.indexMemory(memory);
|
|
150
|
-
const { entities, observations } = this.graph.extractAndStore(content, memory.id);
|
|
151
150
|
res.writeHead(201);
|
|
152
|
-
res.end(JSON.stringify({ memory
|
|
151
|
+
res.end(JSON.stringify({ memory }));
|
|
153
152
|
return;
|
|
154
153
|
}
|
|
155
154
|
|
package/src/graph/extractor.ts
DELETED
|
@@ -1,489 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Entity extraction from text using heuristics
|
|
3
|
-
* No external APIs - pure local processing
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
export interface ExtractedEntity {
|
|
7
|
-
name: string;
|
|
8
|
-
type: "person" | "place" | "concept" | "event" | "organization";
|
|
9
|
-
confidence: number;
|
|
10
|
-
span: { start: number; end: number };
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
// Common words that look like names but aren't (including verbs, adjectives, common nouns)
|
|
14
|
-
const STOPWORDS = new Set([
|
|
15
|
-
// Articles, conjunctions, prepositions
|
|
16
|
-
"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for",
|
|
17
|
-
"of", "with", "by", "from", "as", "is", "was", "are", "were", "been",
|
|
18
|
-
"be", "have", "has", "had", "do", "does", "did", "will", "would",
|
|
19
|
-
"could", "should", "may", "might", "must", "shall", "can", "need",
|
|
20
|
-
// Pronouns
|
|
21
|
-
"this", "that", "these", "those", "i", "you", "he", "she", "it",
|
|
22
|
-
"we", "they", "what", "which", "who", "whom", "whose", "where",
|
|
23
|
-
"when", "why", "how", "all", "each", "every", "both", "few", "more",
|
|
24
|
-
"most", "other", "some", "such", "no", "not", "only", "same", "so",
|
|
25
|
-
"than", "too", "very", "just", "also", "now", "here", "there", "then",
|
|
26
|
-
// Conjunctions
|
|
27
|
-
"if", "because", "while", "although", "though", "after", "before",
|
|
28
|
-
"since", "until", "unless", "however", "therefore", "thus", "hence",
|
|
29
|
-
// Days and months
|
|
30
|
-
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
|
|
31
|
-
"january", "february", "march", "april", "may", "june", "july",
|
|
32
|
-
"august", "september", "october", "november", "december",
|
|
33
|
-
// Time words
|
|
34
|
-
"today", "tomorrow", "yesterday", "morning", "afternoon", "evening", "night",
|
|
35
|
-
// Common verbs (often capitalized at sentence start)
|
|
36
|
-
"said", "says", "told", "asked", "replied", "answered", "mentioned",
|
|
37
|
-
"think", "know", "believe", "feel", "want", "need", "like", "love",
|
|
38
|
-
"crashed", "gets", "getting", "got", "planning", "planned", "plans",
|
|
39
|
-
"working", "worked", "works", "going", "went", "gone", "coming", "came",
|
|
40
|
-
"looking", "looked", "looks", "trying", "tried", "tries", "using", "used",
|
|
41
|
-
"making", "made", "makes", "taking", "took", "takes", "giving", "gave",
|
|
42
|
-
"seeing", "saw", "seen", "being", "having", "doing", "saying", "getting",
|
|
43
|
-
"finding", "found", "keeping", "kept", "letting", "let", "putting", "put",
|
|
44
|
-
"running", "ran", "calling", "called", "moving", "moved", "living", "lived",
|
|
45
|
-
"starting", "started", "seems", "seemed", "showing", "showed", "hearing",
|
|
46
|
-
"playing", "played", "standing", "stood", "understanding", "understood",
|
|
47
|
-
"turning", "turned", "following", "followed", "watching", "watched",
|
|
48
|
-
"adding", "added", "changing", "changed", "writing", "wrote", "reading",
|
|
49
|
-
"learning", "learned", "growing", "grew", "opening", "opened", "walking",
|
|
50
|
-
"winning", "won", "offering", "offered", "remembering", "remembered",
|
|
51
|
-
"considering", "considered", "appearing", "appeared", "buying", "bought",
|
|
52
|
-
"waiting", "waited", "serving", "served", "dying", "died", "sending", "sent",
|
|
53
|
-
"building", "built", "staying", "stayed", "falling", "fell", "cutting", "cut",
|
|
54
|
-
"reaching", "reached", "killing", "killed", "raising", "raised", "passing",
|
|
55
|
-
"selling", "sold", "deciding", "decided", "returning", "returned",
|
|
56
|
-
// Common adjectives (often capitalized at sentence start)
|
|
57
|
-
"good", "bad", "great", "small", "large", "big", "little", "old", "young",
|
|
58
|
-
"new", "first", "last", "long", "short", "high", "low", "right", "wrong",
|
|
59
|
-
"next", "early", "late", "hard", "easy", "clear", "full", "empty", "ready",
|
|
60
|
-
"sure", "open", "closed", "free", "busy", "hot", "cold", "warm", "cool",
|
|
61
|
-
"fast", "slow", "strong", "weak", "deep", "wide", "near", "far", "dark",
|
|
62
|
-
"light", "heavy", "simple", "complex", "real", "true", "false", "best",
|
|
63
|
-
"worst", "happy", "sad", "angry", "afraid", "sorry", "glad", "nice",
|
|
64
|
-
"fine", "okay", "different", "similar", "same", "special", "important",
|
|
65
|
-
"interesting", "beautiful", "wonderful", "terrible", "amazing", "awesome",
|
|
66
|
-
// Common nouns (sentence starters)
|
|
67
|
-
"people", "time", "year", "years", "way", "day", "days", "man", "woman",
|
|
68
|
-
"child", "children", "world", "life", "hand", "part", "place", "case",
|
|
69
|
-
"week", "weeks", "company", "system", "program", "question", "work",
|
|
70
|
-
"government", "number", "point", "home", "water", "room", "mother",
|
|
71
|
-
"area", "money", "story", "fact", "month", "months", "lot", "right",
|
|
72
|
-
"study", "book", "books", "eye", "eyes", "job", "word", "words",
|
|
73
|
-
"business", "issue", "issues", "side", "kind", "head", "house", "service",
|
|
74
|
-
"friend", "friends", "power", "hour", "hours", "game", "line", "end",
|
|
75
|
-
"member", "members", "law", "car", "city", "community", "name", "names",
|
|
76
|
-
"team", "minute", "minutes", "idea", "ideas", "body", "information",
|
|
77
|
-
"back", "parent", "parents", "face", "others", "level", "office", "door",
|
|
78
|
-
"health", "person", "art", "war", "history", "party", "result", "change",
|
|
79
|
-
"reason", "research", "girl", "guy", "moment", "air", "teacher", "force",
|
|
80
|
-
]);
|
|
81
|
-
|
|
82
|
-
// Common titles that precede names
|
|
83
|
-
const TITLES = ["mr", "mrs", "ms", "miss", "dr", "prof", "sir", "lady", "lord"];
|
|
84
|
-
|
|
85
|
-
// Organization suffixes and keywords
|
|
86
|
-
const ORG_SUFFIXES = [
|
|
87
|
-
"inc", "inc.", "corp", "corp.", "corporation", "llc", "llp", "ltd", "ltd.",
|
|
88
|
-
"limited", "co", "co.", "company", "companies", "group", "holdings",
|
|
89
|
-
"partners", "partnership", "associates", "foundation", "institute",
|
|
90
|
-
"university", "college", "school", "hospital", "clinic", "bank",
|
|
91
|
-
"capital", "ventures", "labs", "laboratory", "laboratories",
|
|
92
|
-
"technologies", "tech", "software", "systems", "solutions", "services",
|
|
93
|
-
"industries", "international", "global", "worldwide", "enterprises",
|
|
94
|
-
];
|
|
95
|
-
|
|
96
|
-
// Well-known organizations (case-insensitive matching)
|
|
97
|
-
// Note: Avoid short words that could match common English words (e.g., "WHO")
|
|
98
|
-
const KNOWN_ORGANIZATIONS = new Set([
|
|
99
|
-
"goldman sachs", "morgan stanley", "jp morgan", "jpmorgan", "citibank",
|
|
100
|
-
"bank of america", "wells fargo", "barclays", "deutsche bank", "hsbc",
|
|
101
|
-
"credit suisse", "ubs", "blackrock", "blackstone", "kkr", "carlyle",
|
|
102
|
-
"apollo global", "bridgewater", "citadel", "two sigma", "renaissance technologies",
|
|
103
|
-
"google", "alphabet", "microsoft", "apple", "amazon", "meta", "facebook",
|
|
104
|
-
"netflix", "tesla", "nvidia", "intel", "amd", "ibm", "oracle", "salesforce",
|
|
105
|
-
"adobe", "spotify", "uber", "lyft", "airbnb", "stripe", "square", "paypal",
|
|
106
|
-
"twitter", "x corp", "linkedin", "snapchat", "tiktok", "bytedance",
|
|
107
|
-
"openai", "anthropic", "deepmind", "cohere", "stability ai", "midjourney",
|
|
108
|
-
"199 biotechnologies", "199 bio",
|
|
109
|
-
"harvard university", "stanford university", "yale university", "princeton university",
|
|
110
|
-
"columbia university", "oxford university", "cambridge university",
|
|
111
|
-
"mit", "caltech", "nyu", "ucla", "usc", "berkeley",
|
|
112
|
-
"fbi", "cia", "nsa", "nasa", "fda", "sec", "fcc", "epa", "doj",
|
|
113
|
-
"united nations", "world bank", "imf", "nato", "european union",
|
|
114
|
-
"red cross", "unicef", "greenpeace", "amnesty international",
|
|
115
|
-
"new york times", "washington post", "wall street journal", "bbc", "cnn",
|
|
116
|
-
"nbc", "abc news", "cbs news", "fox news", "reuters", "associated press", "bloomberg",
|
|
117
|
-
]);
|
|
118
|
-
|
|
119
|
-
// Words that look like names but aren't (nationalities, religions, etc.)
|
|
120
|
-
const NOT_PERSON_NAMES = new Set([
|
|
121
|
-
"russian", "american", "british", "chinese", "japanese", "german", "french",
|
|
122
|
-
"italian", "spanish", "indian", "brazilian", "mexican", "canadian", "australian",
|
|
123
|
-
"muslim", "christian", "jewish", "hindu", "buddhist", "atheist", "catholic",
|
|
124
|
-
"protestant", "orthodox", "sunni", "shia", "sikh", "jain",
|
|
125
|
-
"asian", "european", "african", "latin", "caucasian", "middle eastern",
|
|
126
|
-
]);
|
|
127
|
-
|
|
128
|
-
// Common places (US states, major cities, countries)
|
|
129
|
-
const KNOWN_PLACES = new Set([
|
|
130
|
-
"california", "new york", "texas", "florida", "washington", "massachusetts",
|
|
131
|
-
"colorado", "illinois", "pennsylvania", "ohio", "georgia", "michigan",
|
|
132
|
-
"san francisco", "los angeles", "seattle", "boston", "chicago", "miami",
|
|
133
|
-
"london", "paris", "tokyo", "singapore", "hong kong", "dubai", "berlin",
|
|
134
|
-
"sydney", "toronto", "vancouver", "amsterdam", "zurich", "geneva",
|
|
135
|
-
"usa", "uk", "china", "japan", "germany", "france", "india", "canada",
|
|
136
|
-
"australia", "brazil", "mexico", "russia", "spain", "italy", "switzerland",
|
|
137
|
-
]);
|
|
138
|
-
|
|
139
|
-
// Relationship words that often precede person mentions
|
|
140
|
-
const RELATION_WORDS = [
|
|
141
|
-
"brother", "sister", "mother", "father", "mom", "dad", "mum",
|
|
142
|
-
"son", "daughter", "wife", "husband", "partner", "boyfriend", "girlfriend",
|
|
143
|
-
"uncle", "aunt", "cousin", "nephew", "niece", "grandmother", "grandfather",
|
|
144
|
-
"grandma", "grandpa", "friend", "colleague", "boss", "ex", "fiancé", "fiancée",
|
|
145
|
-
];
|
|
146
|
-
|
|
147
|
-
export class EntityExtractor {
|
|
148
|
-
/**
|
|
149
|
-
* Extract all entities from text
|
|
150
|
-
*/
|
|
151
|
-
extractAll(text: string): ExtractedEntity[] {
|
|
152
|
-
const entities: ExtractedEntity[] = [];
|
|
153
|
-
|
|
154
|
-
// Extract organizations FIRST (higher priority)
|
|
155
|
-
const orgs = this.extractOrganizations(text);
|
|
156
|
-
entities.push(...orgs);
|
|
157
|
-
|
|
158
|
-
// Track organization names to avoid re-extracting as persons
|
|
159
|
-
const orgNames = new Set(orgs.map((o) => o.name.toLowerCase()));
|
|
160
|
-
|
|
161
|
-
// Extract persons (excluding already-found orgs)
|
|
162
|
-
const persons = this.extractPersons(text).filter(
|
|
163
|
-
(p) => !orgNames.has(p.name.toLowerCase())
|
|
164
|
-
);
|
|
165
|
-
entities.push(...persons);
|
|
166
|
-
|
|
167
|
-
// First: filter out entities with bad prefixes/suffixes
|
|
168
|
-
const badSuffixes = ["managing", "as", "last", "and", "or", "the", "a", "an", "for", "with"];
|
|
169
|
-
const badPrefixes = ["he", "she", "they", "my", "his", "her", "the", "a", "an", "joined"];
|
|
170
|
-
|
|
171
|
-
const cleanEntities = entities.filter((entity) => {
|
|
172
|
-
const words = entity.name.toLowerCase().split(/\s+/);
|
|
173
|
-
const lastWord = words[words.length - 1];
|
|
174
|
-
const firstWord = words[0];
|
|
175
|
-
if (badSuffixes.includes(lastWord)) return false;
|
|
176
|
-
if (badPrefixes.includes(firstWord)) return false;
|
|
177
|
-
return true;
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
// Deduplicate by name, preferring higher confidence and orgs over persons
|
|
181
|
-
const seen = new Map<string, ExtractedEntity>();
|
|
182
|
-
for (const entity of cleanEntities) {
|
|
183
|
-
const key = entity.name.toLowerCase();
|
|
184
|
-
const existing = seen.get(key);
|
|
185
|
-
if (!existing) {
|
|
186
|
-
seen.set(key, entity);
|
|
187
|
-
} else if (entity.type === "organization" && existing.type === "person") {
|
|
188
|
-
// Prefer org over person
|
|
189
|
-
seen.set(key, entity);
|
|
190
|
-
} else if (entity.confidence > existing.confidence && entity.type === existing.type) {
|
|
191
|
-
seen.set(key, entity);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// Remove entities that are proper substrings of other entities with same type
|
|
196
|
-
const result = Array.from(seen.values());
|
|
197
|
-
return result.filter((entity) => {
|
|
198
|
-
const key = entity.name.toLowerCase();
|
|
199
|
-
for (const other of result) {
|
|
200
|
-
const otherKey = other.name.toLowerCase();
|
|
201
|
-
if (otherKey !== key && other.type === entity.type) {
|
|
202
|
-
// If this entity is a prefix of another (longer) entity, keep the shorter one
|
|
203
|
-
// unless the longer one has much higher confidence
|
|
204
|
-
if (otherKey.startsWith(key + " ") && other.confidence > entity.confidence + 0.1) {
|
|
205
|
-
return false;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
return true;
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Extract organizations from text
|
|
215
|
-
*/
|
|
216
|
-
extractOrganizations(text: string): ExtractedEntity[] {
|
|
217
|
-
const results: ExtractedEntity[] = [];
|
|
218
|
-
const foundNames = new Set<string>();
|
|
219
|
-
|
|
220
|
-
// Pattern 1: Check for known organizations
|
|
221
|
-
for (const orgName of KNOWN_ORGANIZATIONS) {
|
|
222
|
-
const pattern = new RegExp(`\\b${this.escapeRegex(orgName)}\\b`, "gi");
|
|
223
|
-
let match;
|
|
224
|
-
while ((match = pattern.exec(text)) !== null) {
|
|
225
|
-
const name = match[0];
|
|
226
|
-
const key = name.toLowerCase();
|
|
227
|
-
if (!foundNames.has(key)) {
|
|
228
|
-
foundNames.add(key);
|
|
229
|
-
results.push({
|
|
230
|
-
name,
|
|
231
|
-
type: "organization",
|
|
232
|
-
confidence: 0.95,
|
|
233
|
-
span: { start: match.index, end: match.index + name.length },
|
|
234
|
-
});
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
// Pattern 2: Capitalized word(s) followed by org suffixes
|
|
240
|
-
// Allow single word + suffix (e.g., "Acme Corporation")
|
|
241
|
-
// Use case-sensitive matching for proper nouns, handle suffix case separately
|
|
242
|
-
const suffixPatternStr = ORG_SUFFIXES.map(s =>
|
|
243
|
-
`${s.charAt(0).toUpperCase()}${s.slice(1)}|${s.toLowerCase()}`
|
|
244
|
-
).join("|");
|
|
245
|
-
const suffixPattern = new RegExp(
|
|
246
|
-
`(?:^|[^A-Za-z])([A-Z][a-z]+(?:\\s+[A-Z][a-z]+)*)\\s+(${suffixPatternStr})(?=\\s|,|\\.|\\)|$)`,
|
|
247
|
-
"g"
|
|
248
|
-
);
|
|
249
|
-
let match;
|
|
250
|
-
while ((match = suffixPattern.exec(text)) !== null) {
|
|
251
|
-
const baseName = match[1].trim();
|
|
252
|
-
const suffix = match[2].trim();
|
|
253
|
-
const fullName = `${baseName} ${suffix}`;
|
|
254
|
-
const key = fullName.toLowerCase();
|
|
255
|
-
|
|
256
|
-
// Skip common adjective+suffix combos
|
|
257
|
-
const firstWord = baseName.split(/\s+/)[0].toLowerCase();
|
|
258
|
-
if (NOT_PERSON_NAMES.has(firstWord)) continue;
|
|
259
|
-
// Skip single words that are not proper nouns
|
|
260
|
-
if (STOPWORDS.has(firstWord)) continue;
|
|
261
|
-
|
|
262
|
-
if (!foundNames.has(key)) {
|
|
263
|
-
foundNames.add(key);
|
|
264
|
-
results.push({
|
|
265
|
-
name: fullName,
|
|
266
|
-
type: "organization",
|
|
267
|
-
confidence: 0.85,
|
|
268
|
-
span: { start: match.index, end: match.index + fullName.length },
|
|
269
|
-
});
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
// Pattern 3: "works at/for X", "joined X" - only extract multi-word org names
|
|
274
|
-
// Single-word orgs should be in KNOWN_ORGANIZATIONS
|
|
275
|
-
// Use case-sensitive matching for proper nouns (no 'i' flag)
|
|
276
|
-
const workPattern = /(?:works?\s+(?:at|for)|joined|employed\s+(?:at|by)|hired\s+by)\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2})(?=\s+[a-z]|\s*[,.]|\s*$)/g;
|
|
277
|
-
|
|
278
|
-
while ((match = workPattern.exec(text)) !== null) {
|
|
279
|
-
const name = match[1].trim();
|
|
280
|
-
const key = name.toLowerCase();
|
|
281
|
-
const words = name.split(/\s+/);
|
|
282
|
-
|
|
283
|
-
// Skip if first word is a stopword or nationality/religion
|
|
284
|
-
if (STOPWORDS.has(words[0].toLowerCase()) ||
|
|
285
|
-
NOT_PERSON_NAMES.has(words[0].toLowerCase())) {
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
if (!foundNames.has(key)) {
|
|
290
|
-
foundNames.add(key);
|
|
291
|
-
results.push({
|
|
292
|
-
name,
|
|
293
|
-
type: "organization",
|
|
294
|
-
confidence: 0.7,
|
|
295
|
-
span: { start: match.index, end: match.index + match[0].length },
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
return results;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
/**
|
|
304
|
-
* Escape special regex characters
|
|
305
|
-
*/
|
|
306
|
-
private escapeRegex(str: string): string {
|
|
307
|
-
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/**
|
|
311
|
-
* Extract person names from text using heuristics
|
|
312
|
-
*/
|
|
313
|
-
extractPersons(text: string): ExtractedEntity[] {
|
|
314
|
-
const persons: ExtractedEntity[] = [];
|
|
315
|
-
|
|
316
|
-
// Pattern 1: Capitalized words (potential names)
|
|
317
|
-
persons.push(...this.extractCapitalizedNames(text));
|
|
318
|
-
|
|
319
|
-
// Pattern 2: Possessive patterns ("X's brother", "my friend X")
|
|
320
|
-
persons.push(...this.extractFromPossessives(text));
|
|
321
|
-
|
|
322
|
-
// Pattern 3: Relation patterns ("her brother", "my mom")
|
|
323
|
-
persons.push(...this.extractFromRelations(text));
|
|
324
|
-
|
|
325
|
-
return persons;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
/**
|
|
329
|
-
* Extract capitalized words that look like names
|
|
330
|
-
*/
|
|
331
|
-
private extractCapitalizedNames(text: string): ExtractedEntity[] {
|
|
332
|
-
const results: ExtractedEntity[] = [];
|
|
333
|
-
|
|
334
|
-
// Match capitalized words not at sentence start
|
|
335
|
-
// This regex finds sequences of capitalized words
|
|
336
|
-
const pattern = /(?<=[.!?]\s+|^)([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)|(?<=[a-z]\s)([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)/g;
|
|
337
|
-
|
|
338
|
-
let match;
|
|
339
|
-
while ((match = pattern.exec(text)) !== null) {
|
|
340
|
-
const name = (match[1] || match[2]).trim();
|
|
341
|
-
const words = name.split(/\s+/);
|
|
342
|
-
|
|
343
|
-
// Filter out stopwords, nationality/religion words, places, and single common words
|
|
344
|
-
const cleanWords = words.filter(
|
|
345
|
-
(w) => !STOPWORDS.has(w.toLowerCase()) &&
|
|
346
|
-
!NOT_PERSON_NAMES.has(w.toLowerCase()) &&
|
|
347
|
-
!KNOWN_PLACES.has(w.toLowerCase()) &&
|
|
348
|
-
w.length > 1
|
|
349
|
-
);
|
|
350
|
-
|
|
351
|
-
if (cleanWords.length === 0) continue;
|
|
352
|
-
|
|
353
|
-
const cleanName = cleanWords.join(" ");
|
|
354
|
-
|
|
355
|
-
// Skip if it's just a common word
|
|
356
|
-
if (cleanWords.length === 1 && cleanWords[0].length < 4) continue;
|
|
357
|
-
|
|
358
|
-
// Higher confidence for multi-word names
|
|
359
|
-
const confidence = cleanWords.length >= 2 ? 0.8 : 0.5;
|
|
360
|
-
|
|
361
|
-
results.push({
|
|
362
|
-
name: cleanName,
|
|
363
|
-
type: "person",
|
|
364
|
-
confidence,
|
|
365
|
-
span: { start: match.index, end: match.index + match[0].length },
|
|
366
|
-
});
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
return results;
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
/**
|
|
373
|
-
* Extract names from possessive patterns like "Sarah's brother"
|
|
374
|
-
*/
|
|
375
|
-
private extractFromPossessives(text: string): ExtractedEntity[] {
|
|
376
|
-
const results: ExtractedEntity[] = [];
|
|
377
|
-
|
|
378
|
-
// Match "Name's something"
|
|
379
|
-
const pattern = /([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)'s\s+(\w+)/g;
|
|
380
|
-
|
|
381
|
-
let match;
|
|
382
|
-
while ((match = pattern.exec(text)) !== null) {
|
|
383
|
-
const name = match[1].trim();
|
|
384
|
-
const following = match[2].toLowerCase();
|
|
385
|
-
|
|
386
|
-
// Higher confidence if followed by a relationship word
|
|
387
|
-
const isRelation = RELATION_WORDS.includes(following);
|
|
388
|
-
const confidence = isRelation ? 0.95 : 0.7;
|
|
389
|
-
|
|
390
|
-
if (!STOPWORDS.has(name.toLowerCase())) {
|
|
391
|
-
results.push({
|
|
392
|
-
name,
|
|
393
|
-
type: "person",
|
|
394
|
-
confidence,
|
|
395
|
-
span: { start: match.index, end: match.index + name.length },
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
// If followed by relationship word, the whole thing might reference another person
|
|
400
|
-
// e.g., "Sarah's brother" - we create a derived entity
|
|
401
|
-
if (isRelation) {
|
|
402
|
-
results.push({
|
|
403
|
-
name: `${name}'s ${following}`,
|
|
404
|
-
type: "person",
|
|
405
|
-
confidence: 0.6,
|
|
406
|
-
span: { start: match.index, end: match.index + match[0].length },
|
|
407
|
-
});
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
return results;
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
/**
|
|
415
|
-
* Extract from relationship patterns like "her brother", "my friend John"
|
|
416
|
-
*/
|
|
417
|
-
private extractFromRelations(text: string): ExtractedEntity[] {
|
|
418
|
-
const results: ExtractedEntity[] = [];
|
|
419
|
-
|
|
420
|
-
// Pattern: possessive + relation word + optional name
|
|
421
|
-
const pronouns = ["my", "his", "her", "their", "our"];
|
|
422
|
-
const relationPattern = new RegExp(
|
|
423
|
-
`(${pronouns.join("|")})\\s+(${RELATION_WORDS.join("|")})(?:\\s+([A-Z][a-z]+))?`,
|
|
424
|
-
"gi"
|
|
425
|
-
);
|
|
426
|
-
|
|
427
|
-
let match;
|
|
428
|
-
while ((match = relationPattern.exec(text)) !== null) {
|
|
429
|
-
const pronoun = match[1];
|
|
430
|
-
const relation = match[2];
|
|
431
|
-
const name = match[3];
|
|
432
|
-
|
|
433
|
-
if (name && !STOPWORDS.has(name.toLowerCase())) {
|
|
434
|
-
// Explicit name mentioned
|
|
435
|
-
results.push({
|
|
436
|
-
name,
|
|
437
|
-
type: "person",
|
|
438
|
-
confidence: 0.9,
|
|
439
|
-
span: {
|
|
440
|
-
start: match.index + match[0].length - name.length,
|
|
441
|
-
end: match.index + match[0].length,
|
|
442
|
-
},
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
return results;
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/**
|
|
451
|
-
* Extract relationship mentions (not entities, but useful for graph)
|
|
452
|
-
*/
|
|
453
|
-
extractRelationships(text: string): Array<{
|
|
454
|
-
subject: string;
|
|
455
|
-
relation: string;
|
|
456
|
-
object: string;
|
|
457
|
-
confidence: number;
|
|
458
|
-
}> {
|
|
459
|
-
const relationships: Array<{
|
|
460
|
-
subject: string;
|
|
461
|
-
relation: string;
|
|
462
|
-
object: string;
|
|
463
|
-
confidence: number;
|
|
464
|
-
}> = [];
|
|
465
|
-
|
|
466
|
-
// Pattern: "X's [relation]" implies relationship
|
|
467
|
-
const possessivePattern = /([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)'s\s+(\w+)/g;
|
|
468
|
-
|
|
469
|
-
let match;
|
|
470
|
-
while ((match = possessivePattern.exec(text)) !== null) {
|
|
471
|
-
const subject = match[1].trim();
|
|
472
|
-
const relWord = match[2].toLowerCase();
|
|
473
|
-
|
|
474
|
-
if (RELATION_WORDS.includes(relWord)) {
|
|
475
|
-
relationships.push({
|
|
476
|
-
subject,
|
|
477
|
-
relation: relWord,
|
|
478
|
-
object: `${subject}'s ${relWord}`, // placeholder name
|
|
479
|
-
confidence: 0.7,
|
|
480
|
-
});
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
return relationships;
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// Singleton instance
|
|
489
|
-
export const entityExtractor = new EntityExtractor();
|