n8n-nodes-engram 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +334 -0
- package/dist/community/CommunityDetector.d.ts +11 -0
- package/dist/community/CommunityDetector.js +126 -0
- package/dist/community/CommunityDetector.js.map +1 -0
- package/dist/community/CommunitySummarizer.d.ts +8 -0
- package/dist/community/CommunitySummarizer.js +56 -0
- package/dist/community/CommunitySummarizer.js.map +1 -0
- package/dist/community/index.d.ts +2 -0
- package/dist/community/index.js +8 -0
- package/dist/community/index.js.map +1 -0
- package/dist/credentials/EngramExtractionApi.credentials.d.ts +8 -0
- package/dist/credentials/EngramExtractionApi.credentials.js +41 -0
- package/dist/credentials/EngramExtractionApi.credentials.js.map +1 -0
- package/dist/credentials/EngramNeo4jApi.credentials.d.ts +8 -0
- package/dist/credentials/EngramNeo4jApi.credentials.js +59 -0
- package/dist/credentials/EngramNeo4jApi.credentials.js.map +1 -0
- package/dist/descriptions.d.ts +4 -0
- package/dist/descriptions.js +41 -0
- package/dist/descriptions.js.map +1 -0
- package/dist/embeddings/EmbeddingService.d.ts +24 -0
- package/dist/embeddings/EmbeddingService.js +64 -0
- package/dist/embeddings/EmbeddingService.js.map +1 -0
- package/dist/embeddings/cosine.d.ts +1 -0
- package/dist/embeddings/cosine.js +21 -0
- package/dist/embeddings/cosine.js.map +1 -0
- package/dist/embeddings/index.d.ts +2 -0
- package/dist/embeddings/index.js +8 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/extraction/ContradictionDetector.d.ts +11 -0
- package/dist/extraction/ContradictionDetector.js +35 -0
- package/dist/extraction/ContradictionDetector.js.map +1 -0
- package/dist/extraction/EntityDeduplicator.d.ts +17 -0
- package/dist/extraction/EntityDeduplicator.js +39 -0
- package/dist/extraction/EntityDeduplicator.js.map +1 -0
- package/dist/extraction/EntityExtractor.d.ts +11 -0
- package/dist/extraction/EntityExtractor.js +33 -0
- package/dist/extraction/EntityExtractor.js.map +1 -0
- package/dist/extraction/ExtractionPipeline.d.ts +24 -0
- package/dist/extraction/ExtractionPipeline.js +126 -0
- package/dist/extraction/ExtractionPipeline.js.map +1 -0
- package/dist/extraction/LlmClient.d.ts +36 -0
- package/dist/extraction/LlmClient.js +73 -0
- package/dist/extraction/LlmClient.js.map +1 -0
- package/dist/extraction/RelationshipExtractor.d.ts +12 -0
- package/dist/extraction/RelationshipExtractor.js +38 -0
- package/dist/extraction/RelationshipExtractor.js.map +1 -0
- package/dist/extraction/index.d.ts +6 -0
- package/dist/extraction/index.js +16 -0
- package/dist/extraction/index.js.map +1 -0
- package/dist/extraction/prompts.d.ts +16 -0
- package/dist/extraction/prompts.js +101 -0
- package/dist/extraction/prompts.js.map +1 -0
- package/dist/memory/EngramChatMemory.d.ts +42 -0
- package/dist/memory/EngramChatMemory.js +162 -0
- package/dist/memory/EngramChatMemory.js.map +1 -0
- package/dist/memory/EngramChatMessageHistory.d.ts +22 -0
- package/dist/memory/EngramChatMessageHistory.js +72 -0
- package/dist/memory/EngramChatMessageHistory.js.map +1 -0
- package/dist/memory/index.d.ts +2 -0
- package/dist/memory/index.js +8 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/nodes/EngramAdmin/EngramAdmin.node.d.ts +5 -0
- package/dist/nodes/EngramAdmin/EngramAdmin.node.js +798 -0
- package/dist/nodes/EngramAdmin/EngramAdmin.node.js.map +1 -0
- package/dist/nodes/EngramAdmin/engram-admin.png +0 -0
- package/dist/nodes/EngramExplorer/EngramExplorer.node.d.ts +5 -0
- package/dist/nodes/EngramExplorer/EngramExplorer.node.js +932 -0
- package/dist/nodes/EngramExplorer/EngramExplorer.node.js.map +1 -0
- package/dist/nodes/EngramExplorer/engram-explorer.png +0 -0
- package/dist/nodes/EngramMemory/EngramMemory.node.d.ts +10 -0
- package/dist/nodes/EngramMemory/EngramMemory.node.js +462 -0
- package/dist/nodes/EngramMemory/EngramMemory.node.js.map +1 -0
- package/dist/nodes/EngramMemory/engram.png +0 -0
- package/dist/nodes/EngramTrigger/EngramTrigger.node.d.ts +5 -0
- package/dist/nodes/EngramTrigger/EngramTrigger.node.js +146 -0
- package/dist/nodes/EngramTrigger/EngramTrigger.node.js.map +1 -0
- package/dist/nodes/EngramTrigger/engram-trigger.png +0 -0
- package/dist/schemas/Community.schema.d.ts +656 -0
- package/dist/schemas/Community.schema.js +26 -0
- package/dist/schemas/Community.schema.js.map +1 -0
- package/dist/schemas/EntityEdge.schema.d.ts +86 -0
- package/dist/schemas/EntityEdge.schema.js +34 -0
- package/dist/schemas/EntityEdge.schema.js.map +1 -0
- package/dist/schemas/EntityNode.schema.d.ts +56 -0
- package/dist/schemas/EntityNode.schema.js +24 -0
- package/dist/schemas/EntityNode.schema.js.map +1 -0
- package/dist/schemas/EpisodicNode.schema.d.ts +53 -0
- package/dist/schemas/EpisodicNode.schema.js +23 -0
- package/dist/schemas/EpisodicNode.schema.js.map +1 -0
- package/dist/schemas/GraphData.schema.d.ts +220 -0
- package/dist/schemas/GraphData.schema.js +25 -0
- package/dist/schemas/GraphData.schema.js.map +1 -0
- package/dist/schemas/index.d.ts +5 -0
- package/dist/schemas/index.js +20 -0
- package/dist/schemas/index.js.map +1 -0
- package/dist/search/HybridSearchEngine.d.ts +31 -0
- package/dist/search/HybridSearchEngine.js +140 -0
- package/dist/search/HybridSearchEngine.js.map +1 -0
- package/dist/search/MinisearchProvider.d.ts +20 -0
- package/dist/search/MinisearchProvider.js +77 -0
- package/dist/search/MinisearchProvider.js.map +1 -0
- package/dist/search/TextSearchProvider.d.ts +20 -0
- package/dist/search/TextSearchProvider.js +3 -0
- package/dist/search/TextSearchProvider.js.map +1 -0
- package/dist/search/index.d.ts +3 -0
- package/dist/search/index.js +8 -0
- package/dist/search/index.js.map +1 -0
- package/dist/storage/GraphologyStorage.d.ts +42 -0
- package/dist/storage/GraphologyStorage.js +665 -0
- package/dist/storage/GraphologyStorage.js.map +1 -0
- package/dist/storage/IGraphStorage.d.ts +64 -0
- package/dist/storage/IGraphStorage.js +3 -0
- package/dist/storage/IGraphStorage.js.map +1 -0
- package/dist/storage/Neo4jStorage.d.ts +45 -0
- package/dist/storage/Neo4jStorage.js +949 -0
- package/dist/storage/Neo4jStorage.js.map +1 -0
- package/dist/storage/StorageFactory.d.ts +14 -0
- package/dist/storage/StorageFactory.js +26 -0
- package/dist/storage/StorageFactory.js.map +1 -0
- package/dist/storage/index.d.ts +3 -0
- package/dist/storage/index.js +8 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/traversal/EpisodeTraverser.d.ts +10 -0
- package/dist/traversal/EpisodeTraverser.js +46 -0
- package/dist/traversal/EpisodeTraverser.js.map +1 -0
- package/dist/traversal/GraphTraverser.d.ts +24 -0
- package/dist/traversal/GraphTraverser.js +89 -0
- package/dist/traversal/GraphTraverser.js.map +1 -0
- package/dist/traversal/index.d.ts +2 -0
- package/dist/traversal/index.js +8 -0
- package/dist/traversal/index.js.map +1 -0
- package/dist/utils/descriptions.d.ts +6 -0
- package/dist/utils/descriptions.js +95 -0
- package/dist/utils/descriptions.js.map +1 -0
- package/dist/utils/helpers.d.ts +23 -0
- package/dist/utils/helpers.js +146 -0
- package/dist/utils/helpers.js.map +1 -0
- package/dist/utils/logWrapper.d.ts +26 -0
- package/dist/utils/logWrapper.js +300 -0
- package/dist/utils/logWrapper.js.map +1 -0
- package/dist/utils/sharedFields.d.ts +6 -0
- package/dist/utils/sharedFields.js +121 -0
- package/dist/utils/sharedFields.js.map +1 -0
- package/dist/utils/temporal.d.ts +22 -0
- package/dist/utils/temporal.js +44 -0
- package/dist/utils/temporal.js.map +1 -0
- package/dist/utils/tracing.d.ts +7 -0
- package/dist/utils/tracing.js +20 -0
- package/dist/utils/tracing.js.map +1 -0
- package/dist/utils/uuid.d.ts +2 -0
- package/dist/utils/uuid.js +13 -0
- package/dist/utils/uuid.js.map +1 -0
- package/package.json +108 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EngramNeo4jApi = void 0;
|
|
4
|
+
class EngramNeo4jApi {
|
|
5
|
+
constructor() {
|
|
6
|
+
this.name = 'engramNeo4jApi';
|
|
7
|
+
this.displayName = 'Engram Neo4j Connection';
|
|
8
|
+
this.documentationUrl = 'https://neo4j.com/docs/';
|
|
9
|
+
this.properties = [
|
|
10
|
+
{
|
|
11
|
+
displayName: 'URI',
|
|
12
|
+
name: 'uri',
|
|
13
|
+
type: 'string',
|
|
14
|
+
default: 'bolt://localhost:7687',
|
|
15
|
+
placeholder: 'bolt://localhost:7687',
|
|
16
|
+
description: 'The Neo4j connection URI (bolt:// or neo4j://)',
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
displayName: 'Username',
|
|
20
|
+
name: 'username',
|
|
21
|
+
type: 'string',
|
|
22
|
+
default: 'neo4j',
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
displayName: 'Password',
|
|
26
|
+
name: 'password',
|
|
27
|
+
type: 'string',
|
|
28
|
+
typeOptions: { password: true },
|
|
29
|
+
default: '',
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
displayName: 'Database',
|
|
33
|
+
name: 'database',
|
|
34
|
+
type: 'string',
|
|
35
|
+
default: 'neo4j',
|
|
36
|
+
description: 'The Neo4j database to use',
|
|
37
|
+
},
|
|
38
|
+
];
|
|
39
|
+
this.test = {
|
|
40
|
+
request: {
|
|
41
|
+
method: 'POST',
|
|
42
|
+
baseURL: '={{$credentials.uri.replace("bolt://", "http://").replace("neo4j://", "http://").replace(":7687", ":7474").replace("://localhost:", "://127.0.0.1:")}}',
|
|
43
|
+
url: '=/db/{{$credentials.database}}/tx/commit',
|
|
44
|
+
headers: {
|
|
45
|
+
'Content-Type': 'application/json',
|
|
46
|
+
},
|
|
47
|
+
auth: {
|
|
48
|
+
username: '={{$credentials.username}}',
|
|
49
|
+
password: '={{$credentials.password}}',
|
|
50
|
+
},
|
|
51
|
+
body: {
|
|
52
|
+
statements: [{ statement: 'RETURN 1' }],
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
exports.EngramNeo4jApi = EngramNeo4jApi;
|
|
59
|
+
//# sourceMappingURL=EngramNeo4jApi.credentials.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EngramNeo4jApi.credentials.js","sourceRoot":"","sources":["../../src/credentials/EngramNeo4jApi.credentials.ts"],"names":[],"mappings":";;;AAEA,MAAa,cAAc;IAA3B;QACE,SAAI,GAAG,gBAAgB,CAAC;QAExB,gBAAW,GAAG,yBAAyB,CAAC;QAExC,qBAAgB,GAAG,yBAAyB,CAAC;QAE7C,eAAU,GAAsB;YAC9B;gBACE,WAAW,EAAE,KAAK;gBAClB,IAAI,EAAE,KAAK;gBACX,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,uBAAuB;gBAChC,WAAW,EAAE,uBAAuB;gBACpC,WAAW,EAAE,gDAAgD;aAC9D;YACD;gBACE,WAAW,EAAE,UAAU;gBACvB,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,OAAO;aACjB;YACD;gBACE,WAAW,EAAE,UAAU;gBACvB,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC/B,OAAO,EAAE,EAAE;aACZ;YACD;gBACE,WAAW,EAAE,UAAU;gBACvB,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,OAAO;gBAChB,WAAW,EAAE,2BAA2B;aACzC;SACF,CAAC;QAGF,SAAI,GAA2B;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM;gBACd,OAAO,EACL,wJAAwJ;gBAC1J,GAAG,EAAE,0CAA0C;gBAC/C,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE;oBACJ,QAAQ,EAAE,4BAA4B;oBACtC,QAAQ,EAAE,4BAA4B;iBACvC;gBACD,IAAI,EAAE;oBACJ,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;iBACxC;aACF;SACF,CAAC;IACJ,CAAC;CAAA;AAzDD,wCAyDC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.contextWindowLengthProperty = exports.sessionKeyProperty = exports.sessionIdOption = void 0;
|
|
4
|
+
exports.sessionIdOption = {
|
|
5
|
+
displayName: 'Session ID',
|
|
6
|
+
name: 'sessionIdType',
|
|
7
|
+
type: 'options',
|
|
8
|
+
options: [
|
|
9
|
+
{
|
|
10
|
+
name: 'Take from previous node automatically',
|
|
11
|
+
value: 'fromInput',
|
|
12
|
+
description: 'Looks for an input field called sessionId',
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
name: 'Define below',
|
|
16
|
+
value: 'customKey',
|
|
17
|
+
description: 'Use an expression to reference data in previous nodes or enter static text',
|
|
18
|
+
},
|
|
19
|
+
],
|
|
20
|
+
default: 'fromInput',
|
|
21
|
+
};
|
|
22
|
+
exports.sessionKeyProperty = {
|
|
23
|
+
displayName: 'Key',
|
|
24
|
+
name: 'sessionKey',
|
|
25
|
+
type: 'string',
|
|
26
|
+
default: '',
|
|
27
|
+
description: 'The key to use to store session ID in the memory',
|
|
28
|
+
displayOptions: {
|
|
29
|
+
show: {
|
|
30
|
+
sessionIdType: ['customKey'],
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
exports.contextWindowLengthProperty = {
|
|
35
|
+
displayName: 'Context Window Length',
|
|
36
|
+
name: 'contextWindowLength',
|
|
37
|
+
type: 'number',
|
|
38
|
+
default: 5,
|
|
39
|
+
hint: 'How many past interactions the model receives as context',
|
|
40
|
+
};
|
|
41
|
+
//# sourceMappingURL=descriptions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"descriptions.js","sourceRoot":"","sources":["../src/descriptions.ts"],"names":[],"mappings":";;;AAEa,QAAA,eAAe,GAAoB;IAC9C,WAAW,EAAE,YAAY;IACzB,IAAI,EAAE,eAAe;IACrB,IAAI,EAAE,SAAS;IACf,OAAO,EAAE;QACP;YAEE,IAAI,EAAE,uCAAuC;YAC7C,KAAK,EAAE,WAAW;YAClB,WAAW,EAAE,2CAA2C;SACzD;QACD;YAEE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,WAAW;YAClB,WAAW,EAAE,4EAA4E;SAC1F;KACF;IACD,OAAO,EAAE,WAAW;CACrB,CAAC;AAEW,QAAA,kBAAkB,GAAoB;IACjD,WAAW,EAAE,KAAK;IAClB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,kDAAkD;IAC/D,cAAc,EAAE;QACd,IAAI,EAAE;YACJ,aAAa,EAAE,CAAC,WAAW,CAAC;SAC7B;KACF;CACF,CAAC;AAEW,QAAA,2BAA2B,GAAoB;IAC1D,WAAW,EAAE,uBAAuB;IACpC,IAAI,EAAE,qBAAqB;IAC3B,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,0DAA0D;CACjE,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface EmbeddingConfig {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
baseUrl: string;
|
|
4
|
+
model: string;
|
|
5
|
+
dimensions?: number;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface EmbeddingResult {
|
|
9
|
+
embedding: number[];
|
|
10
|
+
usage?: {
|
|
11
|
+
prompt_tokens: number;
|
|
12
|
+
total_tokens: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export declare class EmbeddingService {
|
|
16
|
+
private apiKey;
|
|
17
|
+
private baseUrl;
|
|
18
|
+
private model;
|
|
19
|
+
private dimensions;
|
|
20
|
+
private timeoutMs;
|
|
21
|
+
constructor(config: EmbeddingConfig);
|
|
22
|
+
embed(text: string): Promise<EmbeddingResult>;
|
|
23
|
+
embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EmbeddingService = void 0;
|
|
4
|
+
class EmbeddingService {
|
|
5
|
+
constructor(config) {
|
|
6
|
+
var _a;
|
|
7
|
+
this.apiKey = config.apiKey;
|
|
8
|
+
this.baseUrl = config.baseUrl.replace(/\/$/, '');
|
|
9
|
+
this.model = config.model;
|
|
10
|
+
this.dimensions = config.dimensions;
|
|
11
|
+
this.timeoutMs = (_a = config.timeoutMs) !== null && _a !== void 0 ? _a : 30000;
|
|
12
|
+
}
|
|
13
|
+
async embed(text) {
|
|
14
|
+
const results = await this.embedBatch([text]);
|
|
15
|
+
return results[0];
|
|
16
|
+
}
|
|
17
|
+
async embedBatch(texts) {
|
|
18
|
+
if (texts.length === 0)
|
|
19
|
+
return [];
|
|
20
|
+
const url = `${this.baseUrl}/embeddings`;
|
|
21
|
+
const body = {
|
|
22
|
+
model: this.model,
|
|
23
|
+
input: texts,
|
|
24
|
+
};
|
|
25
|
+
if (this.dimensions) {
|
|
26
|
+
body.dimensions = this.dimensions;
|
|
27
|
+
}
|
|
28
|
+
const controller = new AbortController();
|
|
29
|
+
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
30
|
+
let response;
|
|
31
|
+
try {
|
|
32
|
+
response = await fetch(url, {
|
|
33
|
+
method: 'POST',
|
|
34
|
+
headers: {
|
|
35
|
+
'Content-Type': 'application/json',
|
|
36
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
37
|
+
},
|
|
38
|
+
body: JSON.stringify(body),
|
|
39
|
+
signal: controller.signal,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
44
|
+
throw new Error(`Embedding API request timed out after ${this.timeoutMs}ms`);
|
|
45
|
+
}
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
clearTimeout(timeout);
|
|
50
|
+
}
|
|
51
|
+
if (!response.ok) {
|
|
52
|
+
const errorText = await response.text();
|
|
53
|
+
throw new Error(`Embedding API error (${response.status}): ${errorText}`);
|
|
54
|
+
}
|
|
55
|
+
const data = (await response.json());
|
|
56
|
+
const sorted = [...data.data].sort((a, b) => a.index - b.index);
|
|
57
|
+
return sorted.map((item) => ({
|
|
58
|
+
embedding: item.embedding,
|
|
59
|
+
usage: data.usage,
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
exports.EmbeddingService = EmbeddingService;
|
|
64
|
+
//# sourceMappingURL=EmbeddingService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EmbeddingService.js","sourceRoot":"","sources":["../../src/embeddings/EmbeddingService.ts"],"names":[],"mappings":";;;AAoBA,MAAa,gBAAgB;IAO3B,YAAY,MAAuB;;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,CAAC,SAAS,mCAAI,KAAM,CAAC;IAC9C,CAAC;IAGD,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAGD,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,aAAa,CAAC;QAEzC,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK;SACb,CAAC;QAEF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAErE,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAGlC,CAAC;QAGF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEhE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AA7ED,4CA6EC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.cosineSimilarity = cosineSimilarity;
|
|
4
|
+
function cosineSimilarity(a, b) {
|
|
5
|
+
if (a.length !== b.length) {
|
|
6
|
+
throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
|
|
7
|
+
}
|
|
8
|
+
let dotProduct = 0;
|
|
9
|
+
let normA = 0;
|
|
10
|
+
let normB = 0;
|
|
11
|
+
for (let i = 0; i < a.length; i++) {
|
|
12
|
+
dotProduct += a[i] * b[i];
|
|
13
|
+
normA += a[i] * a[i];
|
|
14
|
+
normB += b[i] * b[i];
|
|
15
|
+
}
|
|
16
|
+
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
17
|
+
if (denominator === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
return dotProduct / denominator;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=cosine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cosine.js","sourceRoot":"","sources":["../../src/embeddings/cosine.ts"],"names":[],"mappings":";;AAIA,4CAmBC;AAnBD,SAAgB,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,OAAO,UAAU,GAAG,WAAW,CAAC;AAClC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.cosineSimilarity = exports.EmbeddingService = void 0;
|
|
4
|
+
var EmbeddingService_1 = require("./EmbeddingService");
|
|
5
|
+
Object.defineProperty(exports, "EmbeddingService", { enumerable: true, get: function () { return EmbeddingService_1.EmbeddingService; } });
|
|
6
|
+
var cosine_1 = require("./cosine");
|
|
7
|
+
Object.defineProperty(exports, "cosineSimilarity", { enumerable: true, get: function () { return cosine_1.cosineSimilarity; } });
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":";;;AAAA,uDAAkG;AAAzF,oHAAA,gBAAgB,OAAA;AACzB,mCAA4C;AAAnC,0GAAA,gBAAgB,OAAA"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { LlmClient } from './LlmClient';
|
|
2
|
+
export interface ContradictionResolution {
|
|
3
|
+
isContradiction: boolean;
|
|
4
|
+
explanation?: string;
|
|
5
|
+
expiredAt?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class ContradictionDetector {
|
|
8
|
+
private llm;
|
|
9
|
+
constructor(llm: LlmClient);
|
|
10
|
+
detect(existingFact: string, newFact: string, sourceEntity: string, targetEntity: string): Promise<ContradictionResolution>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ContradictionDetector = void 0;
|
|
4
|
+
const prompts_1 = require("./prompts");
|
|
5
|
+
const temporal_1 = require("../utils/temporal");
|
|
6
|
+
class ContradictionDetector {
|
|
7
|
+
constructor(llm) {
|
|
8
|
+
this.llm = llm;
|
|
9
|
+
}
|
|
10
|
+
async detect(existingFact, newFact, sourceEntity, targetEntity) {
|
|
11
|
+
try {
|
|
12
|
+
const result = await this.llm.chatJson([
|
|
13
|
+
{ role: 'system', content: prompts_1.CONTRADICTION_SYSTEM },
|
|
14
|
+
{
|
|
15
|
+
role: 'user',
|
|
16
|
+
content: (0, prompts_1.contradictionUser)(existingFact, newFact, sourceEntity, targetEntity),
|
|
17
|
+
},
|
|
18
|
+
]);
|
|
19
|
+
if (result.is_contradiction) {
|
|
20
|
+
return {
|
|
21
|
+
isContradiction: true,
|
|
22
|
+
explanation: result.explanation,
|
|
23
|
+
expiredAt: (0, temporal_1.nowIso)(),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
return { isContradiction: false };
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
console.warn('Engram: Contradiction detection failed:', error.message);
|
|
30
|
+
return { isContradiction: false };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
exports.ContradictionDetector = ContradictionDetector;
|
|
35
|
+
//# sourceMappingURL=ContradictionDetector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ContradictionDetector.js","sourceRoot":"","sources":["../../src/extraction/ContradictionDetector.ts"],"names":[],"mappings":";;;AACA,uCAAoE;AACpE,gDAA2C;AAc3C,MAAa,qBAAqB;IAGhC,YAAY,GAAc;QACxB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAMD,KAAK,CAAC,MAAM,CACV,YAAoB,EACpB,OAAe,EACf,YAAoB,EACpB,YAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAsB;gBAC1D,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,8BAAoB,EAAE;gBACjD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,IAAA,2BAAiB,EAAC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,CAAC;iBAC9E;aACF,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO;oBACL,eAAe,EAAE,IAAI;oBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,SAAS,EAAE,IAAA,iBAAM,GAAE;iBACpB,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;YAClF,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;CACF;AAzCD,sDAyCC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { LlmClient } from './LlmClient';
|
|
2
|
+
export declare class EntityDeduplicator {
|
|
3
|
+
private llm;
|
|
4
|
+
constructor(llm: LlmClient);
|
|
5
|
+
isDuplicate(newEntity: {
|
|
6
|
+
name: string;
|
|
7
|
+
summary: string;
|
|
8
|
+
entity_type: string;
|
|
9
|
+
}, existingEntity: {
|
|
10
|
+
name: string;
|
|
11
|
+
summary: string;
|
|
12
|
+
entity_type: string;
|
|
13
|
+
}): Promise<{
|
|
14
|
+
isDuplicate: boolean;
|
|
15
|
+
mergedSummary?: string;
|
|
16
|
+
}>;
|
|
17
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EntityDeduplicator = void 0;
|
|
4
|
+
const prompts_1 = require("./prompts");
|
|
5
|
+
class EntityDeduplicator {
|
|
6
|
+
constructor(llm) {
|
|
7
|
+
this.llm = llm;
|
|
8
|
+
}
|
|
9
|
+
async isDuplicate(newEntity, existingEntity) {
|
|
10
|
+
if (newEntity.name.toLowerCase().trim() === existingEntity.name.toLowerCase().trim()) {
|
|
11
|
+
const mergedSummary = newEntity.summary.length > existingEntity.summary.length
|
|
12
|
+
? newEntity.summary
|
|
13
|
+
: existingEntity.summary;
|
|
14
|
+
return { isDuplicate: true, mergedSummary };
|
|
15
|
+
}
|
|
16
|
+
if (newEntity.entity_type !== existingEntity.entity_type) {
|
|
17
|
+
return { isDuplicate: false };
|
|
18
|
+
}
|
|
19
|
+
try {
|
|
20
|
+
const result = await this.llm.chatJson([
|
|
21
|
+
{ role: 'system', content: prompts_1.DEDUPLICATION_SYSTEM },
|
|
22
|
+
{
|
|
23
|
+
role: 'user',
|
|
24
|
+
content: (0, prompts_1.deduplicationUser)(newEntity, existingEntity),
|
|
25
|
+
},
|
|
26
|
+
]);
|
|
27
|
+
return {
|
|
28
|
+
isDuplicate: result.is_duplicate === true,
|
|
29
|
+
mergedSummary: result.is_duplicate ? result.merged_summary : undefined,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
console.warn('Engram: Entity deduplication failed:', error.message);
|
|
34
|
+
return { isDuplicate: false };
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
exports.EntityDeduplicator = EntityDeduplicator;
|
|
39
|
+
//# sourceMappingURL=EntityDeduplicator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EntityDeduplicator.js","sourceRoot":"","sources":["../../src/extraction/EntityDeduplicator.ts"],"names":[],"mappings":";;;AACA,uCAAoE;AAOpE,MAAa,kBAAkB;IAG7B,YAAY,GAAc;QACxB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAMD,KAAK,CAAC,WAAW,CACf,SAAiE,EACjE,cAAsE;QAGtE,IAAI,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,KAAK,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAErF,MAAM,aAAa,GACjB,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM;gBACtD,CAAC,CAAC,SAAS,CAAC,OAAO;gBACnB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC;YAC7B,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC9C,CAAC;QAGD,IAAI,SAAS,CAAC,WAAW,KAAK,cAAc,CAAC,WAAW,EAAE,CAAC;YACzD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;QAGD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAsB;gBAC1D,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,8BAAoB,EAAE;gBACjD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,IAAA,2BAAiB,EAAC,SAAS,EAAE,cAAc,CAAC;iBACtD;aACF,CAAC,CAAC;YAEH,OAAO;gBACL,WAAW,EAAE,MAAM,CAAC,YAAY,KAAK,IAAI;gBACzC,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;aACvE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;YAC/E,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAlDD,gDAkDC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { LlmClient } from './LlmClient';
|
|
2
|
+
export interface ExtractedEntity {
|
|
3
|
+
name: string;
|
|
4
|
+
entity_type: string;
|
|
5
|
+
summary: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class EntityExtractor {
|
|
8
|
+
private llm;
|
|
9
|
+
constructor(llm: LlmClient);
|
|
10
|
+
extract(humanMessage: string, aiMessage: string, entityTypes: string[], existingEntities: string[]): Promise<ExtractedEntity[]>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EntityExtractor = void 0;
|
|
4
|
+
const prompts_1 = require("./prompts");
|
|
5
|
+
class EntityExtractor {
|
|
6
|
+
constructor(llm) {
|
|
7
|
+
this.llm = llm;
|
|
8
|
+
}
|
|
9
|
+
async extract(humanMessage, aiMessage, entityTypes, existingEntities) {
|
|
10
|
+
try {
|
|
11
|
+
const result = await this.llm.chatJson([
|
|
12
|
+
{ role: 'system', content: prompts_1.ENTITY_EXTRACTION_SYSTEM },
|
|
13
|
+
{
|
|
14
|
+
role: 'user',
|
|
15
|
+
content: (0, prompts_1.entityExtractionUser)(humanMessage, aiMessage, entityTypes, existingEntities),
|
|
16
|
+
},
|
|
17
|
+
]);
|
|
18
|
+
if (!result.entities || !Array.isArray(result.entities)) {
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
return result.entities.filter((e) => typeof e.name === 'string' &&
|
|
22
|
+
e.name.trim() !== '' &&
|
|
23
|
+
typeof e.entity_type === 'string' &&
|
|
24
|
+
typeof e.summary === 'string');
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
console.warn('Engram: Entity extraction failed:', error.message);
|
|
28
|
+
return [];
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
exports.EntityExtractor = EntityExtractor;
|
|
33
|
+
//# sourceMappingURL=EntityExtractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EntityExtractor.js","sourceRoot":"","sources":["../../src/extraction/EntityExtractor.ts"],"names":[],"mappings":";;;AACA,uCAA2E;AAY3E,MAAa,eAAe;IAG1B,YAAY,GAAc;QACxB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,YAAoB,EACpB,SAAiB,EACjB,WAAqB,EACrB,gBAA0B;QAE1B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAyB;gBAC7D,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,kCAAwB,EAAE;gBACrD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,IAAA,8BAAoB,EAAC,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,gBAAgB,CAAC;iBACtF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxD,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC3B,CAAC,CAAC,EAAE,EAAE,CACJ,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;gBAC1B,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBACpB,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;gBACjC,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAChC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;YAC5E,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAvCD,0CAuCC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { IGraphStorage } from '../storage/IGraphStorage';
|
|
2
|
+
import { type LlmClientConfig } from './LlmClient';
|
|
3
|
+
import { type EmbeddingConfig } from '../embeddings';
|
|
4
|
+
export interface ExtractionPipelineConfig {
|
|
5
|
+
llmConfig: LlmClientConfig;
|
|
6
|
+
entityTypes: string[];
|
|
7
|
+
groupId: string;
|
|
8
|
+
embeddingConfig?: EmbeddingConfig;
|
|
9
|
+
}
|
|
10
|
+
export declare class ExtractionPipeline {
|
|
11
|
+
private storage;
|
|
12
|
+
private entityExtractor;
|
|
13
|
+
private relationshipExtractor;
|
|
14
|
+
private deduplicator;
|
|
15
|
+
private contradictionDetector;
|
|
16
|
+
private embeddingService;
|
|
17
|
+
private entityTypes;
|
|
18
|
+
private groupId;
|
|
19
|
+
constructor(storage: IGraphStorage, config: ExtractionPipelineConfig);
|
|
20
|
+
process(humanMessage: string, aiMessage: string, episodeUuid?: string): Promise<void>;
|
|
21
|
+
private resolveEntity;
|
|
22
|
+
private persistRelationship;
|
|
23
|
+
private findEntity;
|
|
24
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExtractionPipeline = void 0;
|
|
4
|
+
const LlmClient_1 = require("./LlmClient");
|
|
5
|
+
const EntityExtractor_1 = require("./EntityExtractor");
|
|
6
|
+
const RelationshipExtractor_1 = require("./RelationshipExtractor");
|
|
7
|
+
const EntityDeduplicator_1 = require("./EntityDeduplicator");
|
|
8
|
+
const ContradictionDetector_1 = require("./ContradictionDetector");
|
|
9
|
+
const embeddings_1 = require("../embeddings");
|
|
10
|
+
const temporal_1 = require("../utils/temporal");
|
|
11
|
+
class ExtractionPipeline {
|
|
12
|
+
constructor(storage, config) {
|
|
13
|
+
this.embeddingService = null;
|
|
14
|
+
const llm = new LlmClient_1.LlmClient(config.llmConfig);
|
|
15
|
+
this.storage = storage;
|
|
16
|
+
this.entityExtractor = new EntityExtractor_1.EntityExtractor(llm);
|
|
17
|
+
this.relationshipExtractor = new RelationshipExtractor_1.RelationshipExtractor(llm);
|
|
18
|
+
this.deduplicator = new EntityDeduplicator_1.EntityDeduplicator(llm);
|
|
19
|
+
this.contradictionDetector = new ContradictionDetector_1.ContradictionDetector(llm);
|
|
20
|
+
this.entityTypes = config.entityTypes;
|
|
21
|
+
this.groupId = config.groupId;
|
|
22
|
+
if (config.embeddingConfig) {
|
|
23
|
+
this.embeddingService = new embeddings_1.EmbeddingService(config.embeddingConfig);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
async process(humanMessage, aiMessage, episodeUuid) {
|
|
27
|
+
const existingEntities = await this.storage.listEntities(this.groupId);
|
|
28
|
+
const existingNames = existingEntities.map((e) => e.name);
|
|
29
|
+
const extractedEntities = await this.entityExtractor.extract(humanMessage, aiMessage, this.entityTypes, existingNames);
|
|
30
|
+
if (extractedEntities.length === 0)
|
|
31
|
+
return;
|
|
32
|
+
const resolvedEntities = new Map();
|
|
33
|
+
for (const extracted of extractedEntities) {
|
|
34
|
+
const resolved = await this.resolveEntity(extracted, existingEntities);
|
|
35
|
+
resolvedEntities.set(extracted.name.toLowerCase(), resolved);
|
|
36
|
+
}
|
|
37
|
+
const allEntityNames = [
|
|
38
|
+
...new Set([...existingNames, ...[...resolvedEntities.values()].map((e) => e.name)]),
|
|
39
|
+
];
|
|
40
|
+
const extractedRelationships = await this.relationshipExtractor.extract(humanMessage, aiMessage, allEntityNames);
|
|
41
|
+
for (const rel of extractedRelationships) {
|
|
42
|
+
await this.persistRelationship(rel, resolvedEntities, existingEntities, episodeUuid);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
async resolveEntity(extracted, existingEntities) {
|
|
46
|
+
for (const existing of existingEntities) {
|
|
47
|
+
const { isDuplicate, mergedSummary } = await this.deduplicator.isDuplicate(extracted, {
|
|
48
|
+
name: existing.name,
|
|
49
|
+
summary: existing.summary,
|
|
50
|
+
entity_type: existing.entity_type,
|
|
51
|
+
});
|
|
52
|
+
if (isDuplicate) {
|
|
53
|
+
if (mergedSummary && mergedSummary !== existing.summary) {
|
|
54
|
+
return await this.storage.updateEntity(existing.uuid, {
|
|
55
|
+
summary: mergedSummary,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return existing;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
let nameEmbedding;
|
|
62
|
+
if (this.embeddingService) {
|
|
63
|
+
try {
|
|
64
|
+
const result = await this.embeddingService.embed(extracted.name);
|
|
65
|
+
nameEmbedding = result.embedding;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
console.warn('Engram: Failed to generate entity name embedding:', error.message);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return await this.storage.addEntity({
|
|
72
|
+
name: extracted.name,
|
|
73
|
+
group_id: this.groupId,
|
|
74
|
+
summary: extracted.summary,
|
|
75
|
+
entity_type: extracted.entity_type,
|
|
76
|
+
name_embedding: nameEmbedding !== null && nameEmbedding !== void 0 ? nameEmbedding : null,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
async persistRelationship(rel, resolvedEntities, existingEntities, episodeUuid) {
|
|
80
|
+
const sourceEntity = this.findEntity(rel.source_entity, resolvedEntities, existingEntities);
|
|
81
|
+
const targetEntity = this.findEntity(rel.target_entity, resolvedEntities, existingEntities);
|
|
82
|
+
if (!sourceEntity || !targetEntity)
|
|
83
|
+
return;
|
|
84
|
+
const existingEdges = await this.storage.getEdgesBetween(sourceEntity.uuid, targetEntity.uuid);
|
|
85
|
+
for (const existingEdge of existingEdges) {
|
|
86
|
+
if (existingEdge.expired_at)
|
|
87
|
+
continue;
|
|
88
|
+
const resolution = await this.contradictionDetector.detect(existingEdge.fact, rel.fact, sourceEntity.name, targetEntity.name);
|
|
89
|
+
if (resolution.isContradiction && resolution.expiredAt) {
|
|
90
|
+
await this.storage.updateEdge(existingEdge.uuid, {
|
|
91
|
+
expired_at: resolution.expiredAt,
|
|
92
|
+
invalid_at: resolution.expiredAt,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
let factEmbedding;
|
|
97
|
+
if (this.embeddingService) {
|
|
98
|
+
try {
|
|
99
|
+
const result = await this.embeddingService.embed(rel.fact);
|
|
100
|
+
factEmbedding = result.embedding;
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
console.warn('Engram: Failed to generate edge fact embedding:', error.message);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
await this.storage.addEdge({
|
|
107
|
+
group_id: this.groupId,
|
|
108
|
+
source_node_uuid: sourceEntity.uuid,
|
|
109
|
+
target_node_uuid: targetEntity.uuid,
|
|
110
|
+
name: rel.name.toUpperCase().replace(/\s+/g, '_'),
|
|
111
|
+
fact: rel.fact,
|
|
112
|
+
fact_embedding: factEmbedding !== null && factEmbedding !== void 0 ? factEmbedding : null,
|
|
113
|
+
valid_at: (0, temporal_1.nowIso)(),
|
|
114
|
+
episodes: episodeUuid ? [episodeUuid] : [],
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
findEntity(name, resolvedEntities, existingEntities) {
|
|
118
|
+
const normalized = name.toLowerCase().trim();
|
|
119
|
+
const resolved = resolvedEntities.get(normalized);
|
|
120
|
+
if (resolved)
|
|
121
|
+
return resolved;
|
|
122
|
+
return existingEntities.find((e) => e.name.toLowerCase().trim() === normalized);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
exports.ExtractionPipeline = ExtractionPipeline;
|
|
126
|
+
//# sourceMappingURL=ExtractionPipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ExtractionPipeline.js","sourceRoot":"","sources":["../../src/extraction/ExtractionPipeline.ts"],"names":[],"mappings":";;;AAEA,2CAA8D;AAC9D,uDAAoD;AACpD,mEAAgE;AAChE,6DAA0D;AAC1D,mEAAgE;AAChE,8CAAuE;AACvE,gDAA2C;AAmB3C,MAAa,kBAAkB;IAU7B,YAAY,OAAsB,EAAE,MAAgC;QAJ5D,qBAAgB,GAA4B,IAAI,CAAC;QAKvD,MAAM,GAAG,GAAG,IAAI,qBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,IAAI,iCAAe,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,qBAAqB,GAAG,IAAI,6CAAqB,CAAC,GAAG,CAAC,CAAC;QAC5D,IAAI,CAAC,YAAY,GAAG,IAAI,uCAAkB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,qBAAqB,GAAG,IAAI,6CAAqB,CAAC,GAAG,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAE9B,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,6BAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,YAAoB,EAAE,SAAiB,EAAE,WAAoB;QAEzE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAG1D,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAC1D,YAAY,EACZ,SAAS,EACT,IAAI,CAAC,WAAW,EAChB,aAAa,CACd,CAAC;QAEF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAG3C,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAsB,CAAC;QAEvD,KAAK,MAAM,SAAS,IAAI,iBAAiB,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YACvE,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC/D,CAAC;QAGD,MAAM,cAAc,GAAG;YACrB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,GAAG,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;SACrF,CAAC;QAEF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CACrE,YAAY,EACZ,SAAS,EACT,cAAc,CACf,CAAC;QAGF,KAAK,MAAM,GAAG,IAAI,sBAAsB,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,SAAiE,EACjE,gBAA8B;QAG9B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,SAAS,EAAE;gBACpF,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBAEhB,IAAI,aAAa,IAAI,aAAa,KAAK,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACxD,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE;wBACpD,OAAO,EAAE,aAAa;qBACvB,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;QACH,CAAC;QAGD,IAAI,aAAmC,CAAC;QACxC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACjE,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,mDAAmD,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YAClC,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,QAAQ,EAAE,IAAI,CAAC,OAAO;YACtB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,cAAc,EAAE,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,IAAI;SACtC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,GAKC,EACD,gBAAyC,EACzC,gBAA8B,EAC9B,WAAoB;QAGpB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QAC5F,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QAE5F,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY;YAAE,OAAO;QAG3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;QAE/F,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,IAAI,YAAY,CAAC,UAAU;gBAAE,SAAS;YAEtC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CACxD,YAAY,CAAC,IAAI,EACjB,GAAG,CAAC,IAAI,EACR,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,CAClB,CAAC;YAEF,IAAI,UAAU,CAAC,eAAe,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;gBAEvD,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE;oBAC/C,UAAU,EAAE,UAAU,CAAC,SAAS;oBAChC,UAAU,EAAE,UAAU,CAAC,SAAS;iBACjC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAGD,IAAI,aAAmC,CAAC;QACxC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3D,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,OAAO;YACtB,gBAAgB,EAAE,YAAY,CAAC,IAAI;YACnC,gBAAgB,EAAE,YAAY,CAAC,IAAI;YACnC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;YACjD,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,cAAc,EAAE,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,IAAI;YACrC,QAAQ,EAAE,IAAA,iBAAM,GAAE;YAClB,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE;SAC3C,CAAC,CAAC;IACL,CAAC;IAEO,UAAU,CAChB,IAAY,EACZ,gBAAyC,EACzC,gBAA8B;QAE9B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QAG7C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAG9B,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,KAAK,UAAU,CAAC,CAAC;IAClF,CAAC;CACF;AAxLD,gDAwLC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export interface LlmClientConfig {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
baseUrl: string;
|
|
4
|
+
model: string;
|
|
5
|
+
timeoutMs?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface LlmMessage {
|
|
8
|
+
role: 'system' | 'user' | 'assistant';
|
|
9
|
+
content: string;
|
|
10
|
+
}
|
|
11
|
+
export interface LlmResponse {
|
|
12
|
+
content: string;
|
|
13
|
+
usage?: {
|
|
14
|
+
prompt_tokens: number;
|
|
15
|
+
completion_tokens: number;
|
|
16
|
+
total_tokens: number;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export declare class LlmClient {
|
|
20
|
+
private apiKey;
|
|
21
|
+
private baseUrl;
|
|
22
|
+
private model;
|
|
23
|
+
private timeoutMs;
|
|
24
|
+
constructor(config: LlmClientConfig);
|
|
25
|
+
chat(messages: LlmMessage[], options?: {
|
|
26
|
+
temperature?: number;
|
|
27
|
+
maxTokens?: number;
|
|
28
|
+
responseFormat?: {
|
|
29
|
+
type: 'json_object' | 'text';
|
|
30
|
+
};
|
|
31
|
+
}): Promise<LlmResponse>;
|
|
32
|
+
chatJson<T>(messages: LlmMessage[], options?: {
|
|
33
|
+
temperature?: number;
|
|
34
|
+
maxTokens?: number;
|
|
35
|
+
}): Promise<T>;
|
|
36
|
+
}
|