mindgraph-core 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -5
- package/dist/extraction/basic-extractor.d.ts.map +1 -1
- package/dist/extraction/basic-extractor.js +9 -2
- package/dist/extraction/basic-extractor.js.map +1 -1
- package/dist/extraction/confidence-gate.d.ts.map +1 -1
- package/dist/extraction/confidence-gate.js +7 -0
- package/dist/extraction/confidence-gate.js.map +1 -1
- package/dist/extraction/entity-index.d.ts.map +1 -1
- package/dist/extraction/entity-index.js +12 -0
- package/dist/extraction/entity-index.js.map +1 -1
- package/dist/extraction/entity-resolver.d.ts +19 -1
- package/dist/extraction/entity-resolver.d.ts.map +1 -1
- package/dist/extraction/entity-resolver.js +95 -0
- package/dist/extraction/entity-resolver.js.map +1 -1
- package/dist/extraction/extractor-interface.d.ts +65 -0
- package/dist/extraction/extractor-interface.d.ts.map +1 -1
- package/dist/extraction/llm-extractor.d.ts +14 -3
- package/dist/extraction/llm-extractor.d.ts.map +1 -1
- package/dist/extraction/llm-extractor.js +146 -8
- package/dist/extraction/llm-extractor.js.map +1 -1
- package/dist/extraction/prompts/emotion-extraction.d.ts +2 -0
- package/dist/extraction/prompts/emotion-extraction.d.ts.map +1 -0
- package/dist/extraction/prompts/emotion-extraction.js +44 -0
- package/dist/extraction/prompts/emotion-extraction.js.map +1 -0
- package/dist/extraction/prompts/entity-extraction.d.ts +1 -1
- package/dist/extraction/prompts/entity-extraction.d.ts.map +1 -1
- package/dist/extraction/prompts/entity-extraction.js +40 -2
- package/dist/extraction/prompts/entity-extraction.js.map +1 -1
- package/dist/extraction/prompts/proposition-extraction.d.ts +1 -1
- package/dist/extraction/prompts/proposition-extraction.d.ts.map +1 -1
- package/dist/extraction/prompts/proposition-extraction.js +15 -2
- package/dist/extraction/prompts/proposition-extraction.js.map +1 -1
- package/dist/extraction/prompts/reasoning-extraction.d.ts +2 -0
- package/dist/extraction/prompts/reasoning-extraction.d.ts.map +1 -0
- package/dist/extraction/prompts/reasoning-extraction.js +38 -0
- package/dist/extraction/prompts/reasoning-extraction.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/ingestion/chunk-id.d.ts +24 -0
- package/dist/ingestion/chunk-id.d.ts.map +1 -1
- package/dist/ingestion/chunk-id.js +41 -0
- package/dist/ingestion/chunk-id.js.map +1 -1
- package/dist/ingestion/pipeline.d.ts +19 -0
- package/dist/ingestion/pipeline.d.ts.map +1 -1
- package/dist/ingestion/pipeline.js +481 -6
- package/dist/ingestion/pipeline.js.map +1 -1
- package/dist/schema/validation.d.ts.map +1 -1
- package/dist/schema/validation.js +57 -2
- package/dist/schema/validation.js.map +1 -1
- package/package.json +1 -1
|
@@ -42,4 +42,45 @@ export function generateThoughtId(statementHash) {
|
|
|
42
42
|
export function generateNoteId(notePath) {
|
|
43
43
|
return `note:${hashContent(notePath)}`;
|
|
44
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Generate a stable node ID for a location.
|
|
47
|
+
*/
|
|
48
|
+
export function generateLocationId(name) {
|
|
49
|
+
const normalized = name.toLowerCase().replace(/\s+/g, '_');
|
|
50
|
+
return `location:${normalized}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Generate a stable node ID for an event.
|
|
54
|
+
*/
|
|
55
|
+
export function generateEventId(name) {
|
|
56
|
+
const normalized = name.toLowerCase().replace(/\s+/g, '_');
|
|
57
|
+
return `event:${normalized}`;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Generate a stable node ID for an emotion.
|
|
61
|
+
*/
|
|
62
|
+
export function generateEmotionId(name) {
|
|
63
|
+
const normalized = name.toLowerCase().replace(/\s+/g, '_');
|
|
64
|
+
return `emotion:${normalized}`;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Generate a stable node ID for an emotional event.
|
|
68
|
+
*/
|
|
69
|
+
export function generateEmotionalEventId(emotion, trigger) {
|
|
70
|
+
const hash = hashContent(`${emotion}::${trigger}`);
|
|
71
|
+
return `emoevt:${hash}`;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Generate a stable node ID for a reasoning chain.
|
|
75
|
+
*/
|
|
76
|
+
export function generateReasoningChainId(topic, conclusion) {
|
|
77
|
+
const hash = hashContent(`${topic}::${conclusion}`);
|
|
78
|
+
return `rchain:${hash}`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Generate a stable node ID for a reasoning step.
|
|
82
|
+
*/
|
|
83
|
+
export function generateReasoningStepId(chainId, stepIndex) {
|
|
84
|
+
return `rstep:${chainId}:${stepIndex}`;
|
|
85
|
+
}
|
|
45
86
|
//# sourceMappingURL=chunk-id.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-id.js","sourceRoot":"","sources":["../../src/ingestion/chunk-id.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,KAAa,EAAE,IAAY;IAC3E,qEAAqE;IACrE,MAAM,IAAI,GAAG,GAAG,QAAQ,KAAK,KAAK,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,WAAW,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,UAAU,UAAU,EAAE,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,WAAW,UAAU,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAqB;IACzD,OAAO,QAAQ,aAAa,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,aAAqB;IACrD,OAAO,WAAW,aAAa,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,QAAQ,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"chunk-id.js","sourceRoot":"","sources":["../../src/ingestion/chunk-id.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,KAAa,EAAE,IAAY;IAC3E,qEAAqE;IACrE,MAAM,IAAI,GAAG,GAAG,QAAQ,KAAK,KAAK,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,WAAW,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,UAAU,UAAU,EAAE,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,WAAW,UAAU,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAqB;IACzD,OAAO,QAAQ,aAAa,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,aAAqB;IACrD,OAAO,WAAW,aAAa,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,QAAQ,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,YAAY,UAAU,EAAE,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,SAAS,UAAU,EAAE,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3D,OAAO,WAAW,UAAU,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAe,EAAE,OAAe;IACvE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,OAAO,KAAK,OAAO,EAAE,CAAC,CAAC;IACnD,OAAO,UAAU,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAa,EAAE,UAAkB;IACxE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,KAAK,KAAK,UAAU,EAAE,CAAC,CAAC;IACpD,OAAO,UAAU,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAe,EAAE,SAAiB;IACxE,OAAO,SAAS,OAAO,IAAI,SAAS,EAAE,CAAC;AACzC,CAAC"}
|
|
@@ -14,6 +14,12 @@ export interface PipelineStats {
|
|
|
14
14
|
conceptsExtracted: number;
|
|
15
15
|
propositionsExtracted: number;
|
|
16
16
|
thoughtsExtracted: number;
|
|
17
|
+
locationsExtracted: number;
|
|
18
|
+
eventsExtracted: number;
|
|
19
|
+
emotionsExtracted: number;
|
|
20
|
+
emotionalEventsExtracted: number;
|
|
21
|
+
reasoningChainsExtracted: number;
|
|
22
|
+
relationshipsCreated: number;
|
|
17
23
|
embeddingsCreated: number;
|
|
18
24
|
contradictionsDetected: number;
|
|
19
25
|
skipped: boolean;
|
|
@@ -30,6 +36,11 @@ export declare class IngestionPipeline {
|
|
|
30
36
|
private entityResolver;
|
|
31
37
|
private contradictionDetector;
|
|
32
38
|
private entityIndex;
|
|
39
|
+
private pendingRelationships;
|
|
40
|
+
private pendingPropRelationships;
|
|
41
|
+
private pendingEmotions;
|
|
42
|
+
private pendingEmotionalEvents;
|
|
43
|
+
private pendingReasoningChains;
|
|
33
44
|
constructor(storage: StorageAdapter, embedder: EmbeddingProvider | null, extractor: ExtractionProvider, options?: PipelineOptions);
|
|
34
45
|
private getEntityIndex;
|
|
35
46
|
/**
|
|
@@ -47,6 +58,14 @@ export declare class IngestionPipeline {
|
|
|
47
58
|
private processChunk;
|
|
48
59
|
private storeProposition;
|
|
49
60
|
private storeThought;
|
|
61
|
+
private storeLocation;
|
|
62
|
+
private storeEvent;
|
|
63
|
+
private storeEmotion;
|
|
64
|
+
private storeEmotionalEvent;
|
|
65
|
+
private storeReasoningChain;
|
|
66
|
+
private storeAuthoredBy;
|
|
67
|
+
private storeExtractedRelationships;
|
|
68
|
+
private storePropositionRelationships;
|
|
50
69
|
private createAboutEdgesFromChunkMentions;
|
|
51
70
|
private createAboutEdgesForThought;
|
|
52
71
|
private storeResolvedEntities;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,KAAK,EACV,kBAAkB,EAOnB,MAAM,sCAAsC,CAAC;AAK9C,OAAO,EAA+C,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AAiBhG,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,4BAA4B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAID;;;GAGG;AACH,qBAAa,iBAAiB;IAa1B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,OAAO;IAfjB,OAAO,CAAC,cAAc,CAAwB;IAC9C,OAAO,CAAC,qBAAqB,CAAwB;IACrD,OAAO,CAAC,WAAW,CAA4B;IAG/C,OAAO,CAAC,oBAAoB,CAA+B;IAC3D,OAAO,CAAC,wBAAwB,CAA0C;IAC1E,OAAO,CAAC,eAAe,CAA0B;IACjD,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,sBAAsB,CAA4D;gBAGhF,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,iBAAiB,GAAG,IAAI,EAClC,SAAS,EAAE,kBAAkB,EAC7B,OAAO,GAAE,eAAoB;YAKzB,cAAc;IAQ5B;;OAEG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAqI1E;;OAEG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQjD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;YAShB,YAAY;YAsFZ,gBAAgB;YAgFhB,YAAY;YAmEZ,aAAa;YAqFb,UAAU;YA2GV,YAAY;YAuBZ,mBAAmB;YAmEnB,mBAAmB;YA+EnB,eAAe;YA8Bf,2BAA2B;YAoC3B,6BAA6B;YAmC7B,iCAAiC;YAgCjC,0BAA0B;YAyC1B,qBAAqB;IA4N7B,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;YAqBzB,iBAAiB;YAuBjB,oBAAoB;YAuBpB,kBAAkB;CAgBjC"}
|
|
@@ -3,7 +3,7 @@ import { EntityIndex } from '../extraction/entity-index.js';
|
|
|
3
3
|
import { applyConfidenceGate } from '../extraction/confidence-gate.js';
|
|
4
4
|
import { ContradictionDetector } from '../extraction/contradiction-detector.js';
|
|
5
5
|
import { chunkMarkdown, stripFrontmatter } from './chunker.js';
|
|
6
|
-
import { generateNoteId, generateEntityId, generateConceptId, generatePropositionId, generateThoughtId } from './chunk-id.js';
|
|
6
|
+
import { generateNoteId, generateEntityId, generateConceptId, generatePropositionId, generateThoughtId, generateLocationId, generateEventId, generateEmotionId, generateEmotionalEventId, generateReasoningChainId, generateReasoningStepId, } from './chunk-id.js';
|
|
7
7
|
import { hashContent, hashStatement } from './hasher.js';
|
|
8
8
|
import { NodeType, RelType } from '../schema/types.js';
|
|
9
9
|
const DEFAULT_AGENT_ID = 'agent:self';
|
|
@@ -19,6 +19,12 @@ export class IngestionPipeline {
|
|
|
19
19
|
this.options = options;
|
|
20
20
|
this.entityResolver = new EntityResolver();
|
|
21
21
|
this.entityIndex = null;
|
|
22
|
+
// Accumulated per-note extraction results for post-chunk processing
|
|
23
|
+
this.pendingRelationships = [];
|
|
24
|
+
this.pendingPropRelationships = [];
|
|
25
|
+
this.pendingEmotions = [];
|
|
26
|
+
this.pendingEmotionalEvents = [];
|
|
27
|
+
this.pendingReasoningChains = [];
|
|
22
28
|
this.contradictionDetector = new ContradictionDetector(storage, embedder);
|
|
23
29
|
}
|
|
24
30
|
async getEntityIndex() {
|
|
@@ -39,6 +45,12 @@ export class IngestionPipeline {
|
|
|
39
45
|
conceptsExtracted: 0,
|
|
40
46
|
propositionsExtracted: 0,
|
|
41
47
|
thoughtsExtracted: 0,
|
|
48
|
+
locationsExtracted: 0,
|
|
49
|
+
eventsExtracted: 0,
|
|
50
|
+
emotionsExtracted: 0,
|
|
51
|
+
emotionalEventsExtracted: 0,
|
|
52
|
+
reasoningChainsExtracted: 0,
|
|
53
|
+
relationshipsCreated: 0,
|
|
42
54
|
embeddingsCreated: 0,
|
|
43
55
|
contradictionsDetected: 0,
|
|
44
56
|
skipped: false,
|
|
@@ -76,6 +88,10 @@ export class IngestionPipeline {
|
|
|
76
88
|
updatedAt: now,
|
|
77
89
|
};
|
|
78
90
|
await this.storage.upsertNode(noteNode);
|
|
91
|
+
// AUTHORED_BY from frontmatter
|
|
92
|
+
if (frontmatter.author) {
|
|
93
|
+
await this.storeAuthoredBy(noteId, String(frontmatter.author), now);
|
|
94
|
+
}
|
|
79
95
|
// Clean up old chunks for this note, then remove orphaned entities
|
|
80
96
|
const affectedIds = await this.cleanupNoteChunks(noteId, notePath);
|
|
81
97
|
await this.cleanupOrphanedNodes(affectedIds);
|
|
@@ -86,6 +102,11 @@ export class IngestionPipeline {
|
|
|
86
102
|
await this.ensureDefaultAgent();
|
|
87
103
|
// Process each chunk
|
|
88
104
|
this.entityResolver.clear();
|
|
105
|
+
this.pendingRelationships = [];
|
|
106
|
+
this.pendingPropRelationships = [];
|
|
107
|
+
this.pendingEmotions = [];
|
|
108
|
+
this.pendingEmotionalEvents = [];
|
|
109
|
+
this.pendingReasoningChains = [];
|
|
89
110
|
for (const chunk of chunks) {
|
|
90
111
|
await this.processChunk(chunk, noteId, notePath, body, stats);
|
|
91
112
|
}
|
|
@@ -109,8 +130,24 @@ export class IngestionPipeline {
|
|
|
109
130
|
// Embedding failure is non-fatal
|
|
110
131
|
}
|
|
111
132
|
}
|
|
112
|
-
// Store resolved entities
|
|
133
|
+
// Store resolved entities, concepts, locations, events (merged across all chunks)
|
|
113
134
|
await this.storeResolvedEntities(noteId, notePath, stats);
|
|
135
|
+
// Store emotions (global singletons)
|
|
136
|
+
for (const emotion of this.pendingEmotions) {
|
|
137
|
+
await this.storeEmotion(emotion, stats);
|
|
138
|
+
}
|
|
139
|
+
// Store emotional events
|
|
140
|
+
for (const emoEvent of this.pendingEmotionalEvents) {
|
|
141
|
+
await this.storeEmotionalEvent(emoEvent, emoEvent.chunkId, stats);
|
|
142
|
+
}
|
|
143
|
+
// Store reasoning chains
|
|
144
|
+
for (const chain of this.pendingReasoningChains) {
|
|
145
|
+
await this.storeReasoningChain(chain, chain.chunkId, stats);
|
|
146
|
+
}
|
|
147
|
+
// Store extracted relationships (after entities/locations/events are resolved)
|
|
148
|
+
await this.storeExtractedRelationships(stats);
|
|
149
|
+
// Store proposition relationships
|
|
150
|
+
await this.storePropositionRelationships(stats);
|
|
114
151
|
return stats;
|
|
115
152
|
}
|
|
116
153
|
/**
|
|
@@ -165,13 +202,30 @@ export class IngestionPipeline {
|
|
|
165
202
|
});
|
|
166
203
|
// FTS entry
|
|
167
204
|
await this.storage.upsertFTSEntry(chunk.id, chunk.text);
|
|
168
|
-
// Extract entities, concepts, propositions, thoughts
|
|
205
|
+
// Extract entities, concepts, propositions, thoughts, locations, events, emotions, reasoning
|
|
169
206
|
try {
|
|
170
207
|
let extraction = await this.extractor.extract(chunk, noteContent);
|
|
171
208
|
extraction = applyConfidenceGate(extraction, this.options.confidenceThreshold ?? 0.5);
|
|
172
209
|
// Collect for resolution
|
|
173
210
|
this.entityResolver.addEntities(extraction.entities);
|
|
174
211
|
this.entityResolver.addConcepts(extraction.concepts);
|
|
212
|
+
this.entityResolver.addLocations(extraction.locations);
|
|
213
|
+
this.entityResolver.addEvents(extraction.events);
|
|
214
|
+
// Accumulate relationships for post-chunk processing
|
|
215
|
+
this.pendingRelationships.push(...extraction.relationships);
|
|
216
|
+
this.pendingPropRelationships.push(...extraction.propositionRelationships);
|
|
217
|
+
// Deduplicate emotions across chunks, accumulate emotional events and reasoning chains
|
|
218
|
+
for (const emotion of extraction.emotions) {
|
|
219
|
+
if (!this.pendingEmotions.find((e) => e.name.toLowerCase() === emotion.name.toLowerCase())) {
|
|
220
|
+
this.pendingEmotions.push(emotion);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
for (const emoEvent of extraction.emotionalEvents) {
|
|
224
|
+
this.pendingEmotionalEvents.push({ ...emoEvent, chunkId: chunk.id });
|
|
225
|
+
}
|
|
226
|
+
for (const chain of extraction.reasoningChains) {
|
|
227
|
+
this.pendingReasoningChains.push({ ...chain, chunkId: chunk.id });
|
|
228
|
+
}
|
|
175
229
|
// Store propositions immediately (they're per-chunk)
|
|
176
230
|
for (const prop of extraction.propositions) {
|
|
177
231
|
await this.storeProposition(prop, chunk, notePath, stats);
|
|
@@ -308,6 +362,413 @@ export class IngestionPipeline {
|
|
|
308
362
|
await this.createAboutEdgesForThought(thoughtId, thought.subject, now);
|
|
309
363
|
stats.thoughtsExtracted++;
|
|
310
364
|
}
|
|
365
|
+
async storeLocation(location, noteId, entityIndex, stats) {
|
|
366
|
+
const now = new Date().toISOString();
|
|
367
|
+
const existingId = entityIndex.findMatch(location.name, []);
|
|
368
|
+
let locationId;
|
|
369
|
+
if (existingId) {
|
|
370
|
+
locationId = existingId;
|
|
371
|
+
const existingNode = await this.storage.getNode(existingId);
|
|
372
|
+
if (existingNode) {
|
|
373
|
+
const existingData = existingNode.data;
|
|
374
|
+
await this.storage.upsertNode({
|
|
375
|
+
id: existingId,
|
|
376
|
+
nodeType: NodeType.Location,
|
|
377
|
+
data: {
|
|
378
|
+
name: existingData.name,
|
|
379
|
+
locationType: existingData.locationType || location.locationType,
|
|
380
|
+
...(location.coordinates ? { coordinates: location.coordinates } : {}),
|
|
381
|
+
...(existingData.coordinates ? { coordinates: existingData.coordinates } : {}),
|
|
382
|
+
confidence: Math.max(existingData.confidence, location.confidence),
|
|
383
|
+
},
|
|
384
|
+
createdAt: existingNode.createdAt,
|
|
385
|
+
updatedAt: now,
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
else {
|
|
389
|
+
locationId = generateLocationId(location.name);
|
|
390
|
+
await this.storage.upsertNode({
|
|
391
|
+
id: locationId,
|
|
392
|
+
nodeType: NodeType.Location,
|
|
393
|
+
data: {
|
|
394
|
+
name: location.name,
|
|
395
|
+
locationType: location.locationType,
|
|
396
|
+
...(location.coordinates ? { coordinates: location.coordinates } : {}),
|
|
397
|
+
confidence: location.confidence,
|
|
398
|
+
},
|
|
399
|
+
createdAt: now,
|
|
400
|
+
updatedAt: now,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
else {
|
|
405
|
+
locationId = generateLocationId(location.name);
|
|
406
|
+
await this.storage.upsertNode({
|
|
407
|
+
id: locationId,
|
|
408
|
+
nodeType: NodeType.Location,
|
|
409
|
+
data: {
|
|
410
|
+
name: location.name,
|
|
411
|
+
locationType: location.locationType,
|
|
412
|
+
...(location.coordinates ? { coordinates: location.coordinates } : {}),
|
|
413
|
+
confidence: location.confidence,
|
|
414
|
+
},
|
|
415
|
+
createdAt: now,
|
|
416
|
+
updatedAt: now,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
entityIndex.register(locationId, location.name, []);
|
|
420
|
+
// MENTIONS edges from chunks that mention this location
|
|
421
|
+
const noteChunks = await this.storage.queryEdges({
|
|
422
|
+
sourceId: noteId,
|
|
423
|
+
relType: RelType.HAS_CHUNK,
|
|
424
|
+
});
|
|
425
|
+
for (const chunkEdge of noteChunks) {
|
|
426
|
+
const chunkNode = await this.storage.getNode(chunkEdge.targetId);
|
|
427
|
+
if (!chunkNode)
|
|
428
|
+
continue;
|
|
429
|
+
const chunkText = chunkNode.data.text ?? '';
|
|
430
|
+
if (chunkText.toLowerCase().includes(location.name.toLowerCase())) {
|
|
431
|
+
await this.storage.upsertEdge({
|
|
432
|
+
id: `edge:mentions:${chunkEdge.targetId}:${locationId}`,
|
|
433
|
+
sourceId: chunkEdge.targetId,
|
|
434
|
+
targetId: locationId,
|
|
435
|
+
relType: RelType.MENTIONS,
|
|
436
|
+
data: { count: 1, spans: [] },
|
|
437
|
+
createdAt: now,
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
stats.locationsExtracted++;
|
|
442
|
+
}
|
|
443
|
+
async storeEvent(event, noteId, entityIndex, stats) {
|
|
444
|
+
const now = new Date().toISOString();
|
|
445
|
+
const existingId = entityIndex.findMatch(event.name, []);
|
|
446
|
+
let eventId;
|
|
447
|
+
if (existingId) {
|
|
448
|
+
eventId = existingId;
|
|
449
|
+
const existingNode = await this.storage.getNode(existingId);
|
|
450
|
+
if (existingNode) {
|
|
451
|
+
const existingData = existingNode.data;
|
|
452
|
+
const existingParticipants = existingData.participants ?? [];
|
|
453
|
+
const allParticipants = new Set([...existingParticipants, ...event.participants]);
|
|
454
|
+
await this.storage.upsertNode({
|
|
455
|
+
id: existingId,
|
|
456
|
+
nodeType: NodeType.Event,
|
|
457
|
+
data: {
|
|
458
|
+
name: existingData.name,
|
|
459
|
+
description: existingData.description || event.description,
|
|
460
|
+
...(event.date ? { date: event.date } : existingData.date ? { date: existingData.date } : {}),
|
|
461
|
+
...(event.dateGranularity ? { dateGranularity: event.dateGranularity } : existingData.dateGranularity ? { dateGranularity: existingData.dateGranularity } : {}),
|
|
462
|
+
participants: Array.from(allParticipants),
|
|
463
|
+
confidence: Math.max(existingData.confidence, event.confidence),
|
|
464
|
+
},
|
|
465
|
+
createdAt: existingNode.createdAt,
|
|
466
|
+
updatedAt: now,
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
else {
|
|
470
|
+
eventId = generateEventId(event.name);
|
|
471
|
+
await this.storage.upsertNode({
|
|
472
|
+
id: eventId,
|
|
473
|
+
nodeType: NodeType.Event,
|
|
474
|
+
data: {
|
|
475
|
+
name: event.name,
|
|
476
|
+
description: event.description,
|
|
477
|
+
...(event.date ? { date: event.date } : {}),
|
|
478
|
+
...(event.dateGranularity ? { dateGranularity: event.dateGranularity } : {}),
|
|
479
|
+
participants: event.participants,
|
|
480
|
+
confidence: event.confidence,
|
|
481
|
+
},
|
|
482
|
+
createdAt: now,
|
|
483
|
+
updatedAt: now,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
eventId = generateEventId(event.name);
|
|
489
|
+
await this.storage.upsertNode({
|
|
490
|
+
id: eventId,
|
|
491
|
+
nodeType: NodeType.Event,
|
|
492
|
+
data: {
|
|
493
|
+
name: event.name,
|
|
494
|
+
description: event.description,
|
|
495
|
+
...(event.date ? { date: event.date } : {}),
|
|
496
|
+
...(event.dateGranularity ? { dateGranularity: event.dateGranularity } : {}),
|
|
497
|
+
participants: event.participants,
|
|
498
|
+
confidence: event.confidence,
|
|
499
|
+
},
|
|
500
|
+
createdAt: now,
|
|
501
|
+
updatedAt: now,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
entityIndex.register(eventId, event.name, []);
|
|
505
|
+
// MENTIONS edges from chunks
|
|
506
|
+
const noteChunks = await this.storage.queryEdges({
|
|
507
|
+
sourceId: noteId,
|
|
508
|
+
relType: RelType.HAS_CHUNK,
|
|
509
|
+
});
|
|
510
|
+
for (const chunkEdge of noteChunks) {
|
|
511
|
+
const chunkNode = await this.storage.getNode(chunkEdge.targetId);
|
|
512
|
+
if (!chunkNode)
|
|
513
|
+
continue;
|
|
514
|
+
const chunkText = chunkNode.data.text ?? '';
|
|
515
|
+
if (chunkText.toLowerCase().includes(event.name.toLowerCase())) {
|
|
516
|
+
await this.storage.upsertEdge({
|
|
517
|
+
id: `edge:mentions:${chunkEdge.targetId}:${eventId}`,
|
|
518
|
+
sourceId: chunkEdge.targetId,
|
|
519
|
+
targetId: eventId,
|
|
520
|
+
relType: RelType.MENTIONS,
|
|
521
|
+
data: { count: 1, spans: [] },
|
|
522
|
+
createdAt: now,
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
// PARTICIPATED_IN edges for participants
|
|
527
|
+
for (const participantName of event.participants) {
|
|
528
|
+
const participantId = entityIndex.findMatch(participantName, []);
|
|
529
|
+
if (participantId) {
|
|
530
|
+
await this.storage.upsertEdge({
|
|
531
|
+
id: `edge:participated:${participantId}:${eventId}`,
|
|
532
|
+
sourceId: participantId,
|
|
533
|
+
targetId: eventId,
|
|
534
|
+
relType: RelType.PARTICIPATED_IN,
|
|
535
|
+
data: { role: 'participant' },
|
|
536
|
+
createdAt: now,
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
stats.eventsExtracted++;
|
|
541
|
+
}
|
|
542
|
+
async storeEmotion(emotion, stats) {
|
|
543
|
+
const now = new Date().toISOString();
|
|
544
|
+
const emotionId = generateEmotionId(emotion.name);
|
|
545
|
+
const existingNode = await this.storage.getNode(emotionId);
|
|
546
|
+
await this.storage.upsertNode({
|
|
547
|
+
id: emotionId,
|
|
548
|
+
nodeType: NodeType.Emotion,
|
|
549
|
+
data: {
|
|
550
|
+
name: emotion.name,
|
|
551
|
+
valence: emotion.valence,
|
|
552
|
+
arousal: emotion.arousal,
|
|
553
|
+
},
|
|
554
|
+
createdAt: existingNode?.createdAt ?? now,
|
|
555
|
+
updatedAt: now,
|
|
556
|
+
});
|
|
557
|
+
stats.emotionsExtracted++;
|
|
558
|
+
}
|
|
559
|
+
async storeEmotionalEvent(emoEvent, chunkId, stats) {
|
|
560
|
+
const now = new Date().toISOString();
|
|
561
|
+
const emoEventId = generateEmotionalEventId(emoEvent.emotion, emoEvent.trigger);
|
|
562
|
+
await this.storage.upsertNode({
|
|
563
|
+
id: emoEventId,
|
|
564
|
+
nodeType: NodeType.EmotionalEvent,
|
|
565
|
+
data: {
|
|
566
|
+
emotion: emoEvent.emotion,
|
|
567
|
+
intensity: emoEvent.intensity,
|
|
568
|
+
trigger: emoEvent.trigger,
|
|
569
|
+
context: emoEvent.context,
|
|
570
|
+
...(emoEvent.date ? { date: emoEvent.date } : {}),
|
|
571
|
+
},
|
|
572
|
+
createdAt: now,
|
|
573
|
+
updatedAt: now,
|
|
574
|
+
});
|
|
575
|
+
// DERIVED_FROM edge to chunk
|
|
576
|
+
await this.storage.upsertEdge({
|
|
577
|
+
id: `edge:derived:${emoEventId}:${chunkId}`,
|
|
578
|
+
sourceId: emoEventId,
|
|
579
|
+
targetId: chunkId,
|
|
580
|
+
relType: RelType.DERIVED_FROM,
|
|
581
|
+
data: {
|
|
582
|
+
provenance: { notePath: '', chunkId, startOffset: 0, endOffset: 0, quoteText: '', quoteHash: '' },
|
|
583
|
+
extractionMethod: this.extractor.name,
|
|
584
|
+
extractedAt: now,
|
|
585
|
+
},
|
|
586
|
+
createdAt: now,
|
|
587
|
+
});
|
|
588
|
+
// FEELS edge from default agent to the emotion
|
|
589
|
+
const emotionId = generateEmotionId(emoEvent.emotion);
|
|
590
|
+
const emotionNode = await this.storage.getNode(emotionId);
|
|
591
|
+
if (emotionNode) {
|
|
592
|
+
await this.storage.upsertEdge({
|
|
593
|
+
id: `edge:feels:${DEFAULT_AGENT_ID}:${emotionId}:${emoEventId}`,
|
|
594
|
+
sourceId: DEFAULT_AGENT_ID,
|
|
595
|
+
targetId: emotionId,
|
|
596
|
+
relType: RelType.FEELS,
|
|
597
|
+
data: { intensity: emoEvent.intensity, date: emoEvent.date ?? now },
|
|
598
|
+
createdAt: now,
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
// TRIGGERED_BY edge to matching entity/event
|
|
602
|
+
const entityIndex = await this.getEntityIndex();
|
|
603
|
+
const triggerId = entityIndex.findMatch(emoEvent.trigger, []);
|
|
604
|
+
if (triggerId) {
|
|
605
|
+
await this.storage.upsertEdge({
|
|
606
|
+
id: `edge:triggered:${emoEventId}:${triggerId}`,
|
|
607
|
+
sourceId: emoEventId,
|
|
608
|
+
targetId: triggerId,
|
|
609
|
+
relType: RelType.TRIGGERED_BY,
|
|
610
|
+
data: { mechanism: 'extracted' },
|
|
611
|
+
createdAt: now,
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
stats.emotionalEventsExtracted++;
|
|
615
|
+
}
|
|
616
|
+
async storeReasoningChain(chain, chunkId, stats) {
|
|
617
|
+
const now = new Date().toISOString();
|
|
618
|
+
const chainId = generateReasoningChainId(chain.topic, chain.conclusion);
|
|
619
|
+
await this.storage.upsertNode({
|
|
620
|
+
id: chainId,
|
|
621
|
+
nodeType: NodeType.ReasoningChain,
|
|
622
|
+
data: {
|
|
623
|
+
topic: chain.topic,
|
|
624
|
+
conclusion: chain.conclusion,
|
|
625
|
+
strength: chain.strength,
|
|
626
|
+
},
|
|
627
|
+
createdAt: now,
|
|
628
|
+
updatedAt: now,
|
|
629
|
+
});
|
|
630
|
+
// DERIVED_FROM edge to chunk
|
|
631
|
+
await this.storage.upsertEdge({
|
|
632
|
+
id: `edge:derived:${chainId}:${chunkId}`,
|
|
633
|
+
sourceId: chainId,
|
|
634
|
+
targetId: chunkId,
|
|
635
|
+
relType: RelType.DERIVED_FROM,
|
|
636
|
+
data: {
|
|
637
|
+
provenance: { notePath: '', chunkId, startOffset: 0, endOffset: 0, quoteText: '', quoteHash: '' },
|
|
638
|
+
extractionMethod: this.extractor.name,
|
|
639
|
+
extractedAt: now,
|
|
640
|
+
},
|
|
641
|
+
createdAt: now,
|
|
642
|
+
});
|
|
643
|
+
// Store each step
|
|
644
|
+
let prevStepId = null;
|
|
645
|
+
for (const step of chain.steps) {
|
|
646
|
+
const stepId = generateReasoningStepId(chainId, step.stepIndex);
|
|
647
|
+
await this.storage.upsertNode({
|
|
648
|
+
id: stepId,
|
|
649
|
+
nodeType: NodeType.ReasoningStep,
|
|
650
|
+
data: {
|
|
651
|
+
stepIndex: step.stepIndex,
|
|
652
|
+
claim: step.claim,
|
|
653
|
+
evidence: step.evidence,
|
|
654
|
+
stepType: step.stepType,
|
|
655
|
+
},
|
|
656
|
+
createdAt: now,
|
|
657
|
+
updatedAt: now,
|
|
658
|
+
});
|
|
659
|
+
// HAS_STEP edge from chain to step
|
|
660
|
+
await this.storage.upsertEdge({
|
|
661
|
+
id: `edge:has_step:${chainId}:${stepId}`,
|
|
662
|
+
sourceId: chainId,
|
|
663
|
+
targetId: stepId,
|
|
664
|
+
relType: RelType.HAS_STEP,
|
|
665
|
+
data: { stepIndex: step.stepIndex },
|
|
666
|
+
createdAt: now,
|
|
667
|
+
});
|
|
668
|
+
// NEXT_STEP edge from previous step
|
|
669
|
+
if (prevStepId) {
|
|
670
|
+
await this.storage.upsertEdge({
|
|
671
|
+
id: `edge:next_step:${prevStepId}:${stepId}`,
|
|
672
|
+
sourceId: prevStepId,
|
|
673
|
+
targetId: stepId,
|
|
674
|
+
relType: RelType.NEXT_STEP,
|
|
675
|
+
data: { transition: '' },
|
|
676
|
+
createdAt: now,
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
prevStepId = stepId;
|
|
680
|
+
}
|
|
681
|
+
stats.reasoningChainsExtracted++;
|
|
682
|
+
}
|
|
683
|
+
async storeAuthoredBy(noteId, authorName, now) {
|
|
684
|
+
const authorId = `agent:${authorName.toLowerCase().replace(/\s+/g, '_')}`;
|
|
685
|
+
// Upsert Agent node for author
|
|
686
|
+
const existingAgent = await this.storage.getNode(authorId);
|
|
687
|
+
if (!existingAgent) {
|
|
688
|
+
await this.storage.upsertNode({
|
|
689
|
+
id: authorId,
|
|
690
|
+
nodeType: NodeType.Agent,
|
|
691
|
+
data: {
|
|
692
|
+
name: authorName,
|
|
693
|
+
agentType: 'person',
|
|
694
|
+
description: `Author: ${authorName}`,
|
|
695
|
+
},
|
|
696
|
+
createdAt: now,
|
|
697
|
+
updatedAt: now,
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
// AUTHORED_BY edge from Note to Agent
|
|
701
|
+
await this.storage.upsertEdge({
|
|
702
|
+
id: `edge:authored:${noteId}:${authorId}`,
|
|
703
|
+
sourceId: noteId,
|
|
704
|
+
targetId: authorId,
|
|
705
|
+
relType: RelType.AUTHORED_BY,
|
|
706
|
+
data: { role: 'author' },
|
|
707
|
+
createdAt: now,
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
async storeExtractedRelationships(stats) {
|
|
711
|
+
const now = new Date().toISOString();
|
|
712
|
+
const entityIndex = await this.getEntityIndex();
|
|
713
|
+
const relTypeMap = {
|
|
714
|
+
IS_A: RelType.IS_A,
|
|
715
|
+
PART_OF: RelType.PART_OF,
|
|
716
|
+
RELATED_TO: RelType.RELATED_TO,
|
|
717
|
+
LOCATED_IN: RelType.LOCATED_IN,
|
|
718
|
+
OCCURRED_AT: RelType.OCCURRED_AT,
|
|
719
|
+
PRECEDES: RelType.PRECEDES,
|
|
720
|
+
CAUSES: RelType.CAUSES,
|
|
721
|
+
PARTICIPATED_IN: RelType.PARTICIPATED_IN,
|
|
722
|
+
};
|
|
723
|
+
for (const rel of this.pendingRelationships) {
|
|
724
|
+
const sourceId = entityIndex.findMatch(rel.sourceRef, []);
|
|
725
|
+
const targetId = entityIndex.findMatch(rel.targetRef, []);
|
|
726
|
+
if (!sourceId || !targetId)
|
|
727
|
+
continue;
|
|
728
|
+
const edgeRelType = relTypeMap[rel.relType];
|
|
729
|
+
if (!edgeRelType)
|
|
730
|
+
continue;
|
|
731
|
+
await this.storage.upsertEdge({
|
|
732
|
+
id: `edge:rel:${edgeRelType}:${sourceId}:${targetId}`,
|
|
733
|
+
sourceId,
|
|
734
|
+
targetId,
|
|
735
|
+
relType: edgeRelType,
|
|
736
|
+
data: { ...rel.data, confidence: rel.confidence },
|
|
737
|
+
createdAt: now,
|
|
738
|
+
});
|
|
739
|
+
stats.relationshipsCreated++;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
async storePropositionRelationships(stats) {
|
|
743
|
+
const now = new Date().toISOString();
|
|
744
|
+
const relTypeMap = {
|
|
745
|
+
SUPPORTS: RelType.SUPPORTS,
|
|
746
|
+
QUALIFIES: RelType.QUALIFIES,
|
|
747
|
+
};
|
|
748
|
+
for (const rel of this.pendingPropRelationships) {
|
|
749
|
+
const sourceHash = hashStatement(rel.sourceStatement);
|
|
750
|
+
const targetHash = hashStatement(rel.targetStatement);
|
|
751
|
+
const sourceId = generatePropositionId(sourceHash);
|
|
752
|
+
const targetId = generatePropositionId(targetHash);
|
|
753
|
+
// Verify both propositions exist
|
|
754
|
+
const sourceNode = await this.storage.getNode(sourceId);
|
|
755
|
+
const targetNode = await this.storage.getNode(targetId);
|
|
756
|
+
if (!sourceNode || !targetNode)
|
|
757
|
+
continue;
|
|
758
|
+
const edgeRelType = relTypeMap[rel.relType];
|
|
759
|
+
if (!edgeRelType)
|
|
760
|
+
continue;
|
|
761
|
+
await this.storage.upsertEdge({
|
|
762
|
+
id: `edge:proprel:${edgeRelType}:${sourceId}:${targetId}`,
|
|
763
|
+
sourceId,
|
|
764
|
+
targetId,
|
|
765
|
+
relType: edgeRelType,
|
|
766
|
+
data: { ...rel.data, strength: rel.confidence, explanation: '' },
|
|
767
|
+
createdAt: now,
|
|
768
|
+
});
|
|
769
|
+
stats.relationshipsCreated++;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
311
772
|
async createAboutEdgesFromChunkMentions(propId, chunkId, now) {
|
|
312
773
|
// Find entities/concepts mentioned in the source chunk
|
|
313
774
|
const mentionEdges = await this.storage.queryEdges({
|
|
@@ -319,7 +780,8 @@ export class IngestionPipeline {
|
|
|
319
780
|
if (targetNode &&
|
|
320
781
|
(targetNode.nodeType === NodeType.Entity ||
|
|
321
782
|
targetNode.nodeType === NodeType.Concept ||
|
|
322
|
-
targetNode.nodeType === NodeType.Event
|
|
783
|
+
targetNode.nodeType === NodeType.Event ||
|
|
784
|
+
targetNode.nodeType === NodeType.Location)) {
|
|
323
785
|
await this.storage.upsertEdge({
|
|
324
786
|
id: `edge:about:${propId}:${targetNode.id}`,
|
|
325
787
|
sourceId: propId,
|
|
@@ -370,6 +832,8 @@ export class IngestionPipeline {
|
|
|
370
832
|
const now = new Date().toISOString();
|
|
371
833
|
const entities = this.entityResolver.getResolvedEntities();
|
|
372
834
|
const concepts = this.entityResolver.getResolvedConcepts();
|
|
835
|
+
const locations = this.entityResolver.getResolvedLocations();
|
|
836
|
+
const events = this.entityResolver.getResolvedEvents();
|
|
373
837
|
const entityIndex = await this.getEntityIndex();
|
|
374
838
|
for (const entity of entities) {
|
|
375
839
|
const existingId = entityIndex.findMatch(entity.name, entity.aliases);
|
|
@@ -556,11 +1020,19 @@ export class IngestionPipeline {
|
|
|
556
1020
|
entityIndex.register(conceptId, cData.name, cData.aliases ?? []);
|
|
557
1021
|
stats.conceptsExtracted++;
|
|
558
1022
|
}
|
|
1023
|
+
// Store locations
|
|
1024
|
+
for (const location of locations) {
|
|
1025
|
+
await this.storeLocation(location, noteId, entityIndex, stats);
|
|
1026
|
+
}
|
|
1027
|
+
// Store events
|
|
1028
|
+
for (const event of events) {
|
|
1029
|
+
await this.storeEvent(event, noteId, entityIndex, stats);
|
|
1030
|
+
}
|
|
559
1031
|
}
|
|
560
1032
|
async cleanupOrphans() {
|
|
561
1033
|
const entityIndex = await this.getEntityIndex();
|
|
562
1034
|
let cleaned = 0;
|
|
563
|
-
for (const nodeType of [NodeType.Entity, NodeType.Concept]) {
|
|
1035
|
+
for (const nodeType of [NodeType.Entity, NodeType.Concept, NodeType.Location, NodeType.Event]) {
|
|
564
1036
|
const nodes = await this.storage.queryNodes({ nodeType });
|
|
565
1037
|
for (const node of nodes) {
|
|
566
1038
|
const mentionEdges = await this.storage.queryEdges({
|
|
@@ -605,7 +1077,10 @@ export class IngestionPipeline {
|
|
|
605
1077
|
const node = await this.storage.getNode(nodeId);
|
|
606
1078
|
if (!node)
|
|
607
1079
|
continue;
|
|
608
|
-
if (node.nodeType !== NodeType.Entity &&
|
|
1080
|
+
if (node.nodeType !== NodeType.Entity &&
|
|
1081
|
+
node.nodeType !== NodeType.Concept &&
|
|
1082
|
+
node.nodeType !== NodeType.Location &&
|
|
1083
|
+
node.nodeType !== NodeType.Event)
|
|
609
1084
|
continue;
|
|
610
1085
|
const mentions = await this.storage.queryEdges({ targetId: nodeId, relType: RelType.MENTIONS });
|
|
611
1086
|
if (mentions.length === 0) {
|