mindgraph-core 0.1.0

package/dist/ingestion/pipeline.js

@@ -0,0 +1,653 @@
+import { EntityResolver } from '../extraction/entity-resolver.js';
+import { EntityIndex } from '../extraction/entity-index.js';
+import { applyConfidenceGate } from '../extraction/confidence-gate.js';
+import { ContradictionDetector } from '../extraction/contradiction-detector.js';
+import { chunkMarkdown, stripFrontmatter } from './chunker.js';
+import { generateNoteId, generateEntityId, generateConceptId, generatePropositionId, generateThoughtId } from './chunk-id.js';
+import { hashContent, hashStatement } from './hasher.js';
+import { NodeType, RelType } from '../schema/types.js';
+const DEFAULT_AGENT_ID = 'agent:self';
+/**
+ * The ingestion pipeline orchestrates:
+ * hash check → chunk → extract → embed → store → detect contradictions
+ */
+export class IngestionPipeline {
+    constructor(storage, embedder, extractor, options = {}) {
+        this.storage = storage;
+        this.embedder = embedder;
+        this.extractor = extractor;
+        this.options = options;
+        this.entityResolver = new EntityResolver();
+        this.entityIndex = null;
+        this.contradictionDetector = new ContradictionDetector(storage, embedder);
+    }
+    async getEntityIndex() {
+        if (!this.entityIndex) {
+            this.entityIndex = new EntityIndex();
+            await this.entityIndex.loadFromStorage(this.storage);
+        }
+        return this.entityIndex;
+    }
+    /**
+     * Index a single note. Idempotent — skips if content hash unchanged.
+     */
+    async indexNote(notePath, content) {
+        const stats = {
+            notePath,
+            chunksCreated: 0,
+            entitiesExtracted: 0,
+            conceptsExtracted: 0,
+            propositionsExtracted: 0,
+            thoughtsExtracted: 0,
+            embeddingsCreated: 0,
+            contradictionsDetected: 0,
+            skipped: false,
+        };
+        const contentHash = hashContent(content);
+        const noteId = generateNoteId(notePath);
+        // Check if note already indexed with same content
+        const existingNote = await this.storage.getNode(noteId);
+        if (existingNote && existingNote.data.contentHash === contentHash) {
+            stats.skipped = true;
+            return stats;
+        }
+        // Strip frontmatter
+        const { frontmatter, body } = stripFrontmatter(content);
+        // Extract metadata
+        const title = frontmatter.title ??
+            notePath.split('/').pop()?.replace(/\.md$/, '') ?? notePath;
+        const tags = extractTags(body, frontmatter);
+        const aliases = frontmatter.aliases ?? [];
+        const now = new Date().toISOString();
+        // Upsert Note node
+        const noteNode = {
+            id: noteId,
+            nodeType: NodeType.Note,
+            data: {
+                title,
+                path: notePath,
+                contentHash,
+                wordCount: body.split(/\s+/).length,
+                tags,
+                aliases,
+                frontmatter,
+            },
+            createdAt: existingNote?.createdAt ?? now,
+            updatedAt: now,
+        };
+        await this.storage.upsertNode(noteNode);
+        // Clean up old chunks for this note, then remove orphaned entities
+        const affectedIds = await this.cleanupNoteChunks(noteId, notePath);
+        await this.cleanupOrphanedNodes(affectedIds);
+        // Chunk the note
+        const chunks = chunkMarkdown(notePath, body, this.options.chunkerOptions);
+        stats.chunksCreated = chunks.length;
+        // Ensure default agent exists
+        await this.ensureDefaultAgent();
+        // Process each chunk
+        this.entityResolver.clear();
+        for (const chunk of chunks) {
+            await this.processChunk(chunk, noteId, notePath, body, stats);
+        }
+        // Batch embed all chunks
+        if (this.embedder && chunks.length > 0) {
+            try {
+                const vectors = await this.embedder.embedBatch(chunks.map((c) => c.text));
+                const embNow = new Date().toISOString();
+                for (let i = 0; i < vectors.length; i++) {
+                    await this.storage.upsertEmbedding({
+                        chunkId: chunks[i].id,
+                        vector: vectors[i],
+                        modelId: this.embedder.modelId,
+                        dimensions: this.embedder.dimensions,
+                        createdAt: embNow,
+                    });
+                    stats.embeddingsCreated++;
+                }
+            }
+            catch {
+                // Embedding failure is non-fatal
+            }
+        }
+        // Store resolved entities and concepts (merged across all chunks)
+        await this.storeResolvedEntities(noteId, notePath, stats);
+        return stats;
+    }
+    /**
+     * Remove a note and all its derived data.
+     */
+    async forgetNote(notePath) {
+        const noteId = generateNoteId(notePath);
+        const affectedIds = await this.cleanupNoteChunks(noteId, notePath);
+        await this.storage.deleteEdgesByNode(noteId);
+        await this.storage.deleteNode(noteId);
+        await this.cleanupOrphanedNodes(affectedIds);
+    }
+    /**
+     * Wipe all data.
+     */
+    async wipeAll() {
+        // This is a destructive operation — must be called explicitly
+        const allNodes = await this.storage.queryNodes({});
+        for (const node of allNodes) {
+            await this.storage.deleteEdgesByNode(node.id);
+            await this.storage.deleteNode(node.id);
+        }
+    }
+    async processChunk(chunk, noteId, notePath, noteContent, stats) {
+        const now = new Date().toISOString();
+        // Store chunk node
+        const chunkNode = {
+            id: chunk.id,
+            nodeType: NodeType.Chunk,
+            data: {
+                notePath: chunk.notePath,
+                text: chunk.text,
+                heading: chunk.heading,
+                headingLevel: chunk.headingLevel,
+                startOffset: chunk.startOffset,
+                endOffset: chunk.endOffset,
+                index: chunk.index,
+                contentHash: chunk.contentHash,
+            },
+            createdAt: now,
+            updatedAt: now,
+        };
+        await this.storage.upsertNode(chunkNode);
+        // HAS_CHUNK edge
+        await this.storage.upsertEdge({
+            id: `edge:has_chunk:${noteId}:${chunk.id}`,
+            sourceId: noteId,
+            targetId: chunk.id,
+            relType: RelType.HAS_CHUNK,
+            data: { index: chunk.index },
+            createdAt: now,
+        });
+        // FTS entry
+        await this.storage.upsertFTSEntry(chunk.id, chunk.text);
+        // Extract entities, concepts, propositions, thoughts
+        try {
+            let extraction = await this.extractor.extract(chunk, noteContent);
+            extraction = applyConfidenceGate(extraction, this.options.confidenceThreshold ?? 0.5);
+            // Collect for resolution
+            this.entityResolver.addEntities(extraction.entities);
+            this.entityResolver.addConcepts(extraction.concepts);
+            // Store propositions immediately (they're per-chunk)
+            for (const prop of extraction.propositions) {
+                await this.storeProposition(prop, chunk, notePath, stats);
+            }
+            // Store thoughts immediately
+            for (const thought of extraction.thoughts) {
+                await this.storeThought(thought, chunk, notePath, stats);
+            }
+        }
+        catch {
+            // Extraction failure is non-fatal
+        }
+    }
+    async storeProposition(prop, chunk, notePath, stats) {
+        const now = new Date().toISOString();
+        const stmtHash = hashStatement(prop.statement);
+        const propId = generatePropositionId(stmtHash);
+        const propNode = {
+            id: propId,
+            nodeType: NodeType.Proposition,
+            data: {
+                statement: prop.statement,
+                domain: prop.domain,
+                confidence: prop.confidence,
+                isNegated: prop.isNegated,
+                statementHash: stmtHash,
+            },
+            createdAt: now,
+            updatedAt: now,
+        };
+        await this.storage.upsertNode(propNode);
+        // DERIVED_FROM edge with provenance
+        const provenance = {
+            notePath,
+            chunkId: chunk.id,
+            startOffset: prop.quoteStart + chunk.startOffset,
+            endOffset: prop.quoteEnd + chunk.startOffset,
+            quoteText: prop.quoteText,
+            quoteHash: hashContent(prop.quoteText),
+        };
+        await this.storage.upsertEdge({
+            id: `edge:derived:${propId}:${chunk.id}`,
+            sourceId: propId,
+            targetId: chunk.id,
+            relType: RelType.DERIVED_FROM,
+            data: {
+                provenance,
+                extractionMethod: this.extractor.name,
+                extractedAt: now,
+            },
+            createdAt: now,
+        });
+        // BELIEVES edge from default agent
+        await this.storage.upsertEdge({
+            id: `edge:believes:${DEFAULT_AGENT_ID}:${propId}`,
+            sourceId: DEFAULT_AGENT_ID,
+            targetId: propId,
+            relType: RelType.BELIEVES,
+            data: {
+                confidence: prop.confidence,
+                asOf: now,
+            },
+            createdAt: now,
+        });
+        // ABOUT edges — link proposition to entities/concepts mentioned in same chunk
+        await this.createAboutEdgesFromChunkMentions(propId, chunk.id, now);
+        stats.propositionsExtracted++;
+        // Detect contradictions
+        if (this.options.enableContradictionDetection !== false) {
+            try {
+                const candidates = await this.contradictionDetector.findCandidates(propId, 5, 0.6);
+                if (candidates.length > 0) {
+                    const edgeIds = await this.contradictionDetector.createContradictionEdges(candidates);
+                    stats.contradictionsDetected += edgeIds.length;
+                }
+            }
+            catch {
+                // Non-fatal
+            }
+        }
+    }
+    async storeThought(thought, chunk, notePath, stats) {
+        const now = new Date().toISOString();
+        const stmtHash = hashStatement(thought.statement);
+        const thoughtId = generateThoughtId(stmtHash);
+        const thoughtNode = {
+            id: thoughtId,
+            nodeType: NodeType.Thought,
+            data: {
+                statement: thought.statement,
+                stance: thought.stance,
+                subject: thought.subject,
+                confidence: thought.confidence,
+                implications: thought.implications,
+            },
+            createdAt: now,
+            updatedAt: now,
+        };
+        await this.storage.upsertNode(thoughtNode);
+        // DERIVED_FROM edge
+        const provenance = {
+            notePath,
+            chunkId: chunk.id,
+            startOffset: thought.quoteStart + chunk.startOffset,
+            endOffset: thought.quoteEnd + chunk.startOffset,
+            quoteText: thought.quoteText,
+            quoteHash: hashContent(thought.quoteText),
+        };
+        await this.storage.upsertEdge({
+            id: `edge:derived:${thoughtId}:${chunk.id}`,
+            sourceId: thoughtId,
+            targetId: chunk.id,
+            relType: RelType.DERIVED_FROM,
+            data: {
+                provenance,
+                extractionMethod: this.extractor.name,
+                extractedAt: now,
+            },
+            createdAt: now,
+        });
+        // ENDORSES edge from default agent (thoughts use ENDORSES, not BELIEVES)
+        await this.storage.upsertEdge({
+            id: `edge:endorses:${DEFAULT_AGENT_ID}:${thoughtId}`,
+            sourceId: DEFAULT_AGENT_ID,
+            targetId: thoughtId,
+            relType: RelType.ENDORSES,
+            data: {
+                confidence: thought.confidence,
+                asOf: now,
+            },
+            createdAt: now,
+        });
+        // ABOUT edges — link thought to matching entity/concept by subject
+        await this.createAboutEdgesForThought(thoughtId, thought.subject, now);
+        stats.thoughtsExtracted++;
+    }
+    async createAboutEdgesFromChunkMentions(propId, chunkId, now) {
+        // Find entities/concepts mentioned in the source chunk
+        const mentionEdges = await this.storage.queryEdges({
+            sourceId: chunkId,
+            relType: RelType.MENTIONS,
+        });
+        for (const mentionEdge of mentionEdges) {
+            const targetNode = await this.storage.getNode(mentionEdge.targetId);
+            if (targetNode &&
+                (targetNode.nodeType === NodeType.Entity ||
+                    targetNode.nodeType === NodeType.Concept ||
+                    targetNode.nodeType === NodeType.Event)) {
+                await this.storage.upsertEdge({
+                    id: `edge:about:${propId}:${targetNode.id}`,
+                    sourceId: propId,
+                    targetId: targetNode.id,
+                    relType: RelType.ABOUT,
+                    data: { relevance: 0.7 },
+                    createdAt: now,
+                });
+            }
+        }
+    }
+    async createAboutEdgesForThought(thoughtId, subject, now) {
+        if (!subject)
+            return;
+        const subjectLower = subject.toLowerCase();
+        // Search entities matching subject
+        const entities = await this.storage.queryNodes({ nodeType: NodeType.Entity });
+        for (const entity of entities) {
+            const name = entity.data.name;
+            if (name && name.toLowerCase().includes(subjectLower)) {
+                await this.storage.upsertEdge({
+                    id: `edge:about:${thoughtId}:${entity.id}`,
+                    sourceId: thoughtId,
+                    targetId: entity.id,
+                    relType: RelType.ABOUT,
+                    data: { relevance: 0.8 },
+                    createdAt: now,
+                });
+            }
+        }
+        // Search concepts matching subject
+        const concepts = await this.storage.queryNodes({ nodeType: NodeType.Concept });
+        for (const concept of concepts) {
+            const name = concept.data.name;
+            if (name && name.toLowerCase().includes(subjectLower)) {
+                await this.storage.upsertEdge({
+                    id: `edge:about:${thoughtId}:${concept.id}`,
+                    sourceId: thoughtId,
+                    targetId: concept.id,
+                    relType: RelType.ABOUT,
+                    data: { relevance: 0.8 },
+                    createdAt: now,
+                });
+            }
+        }
+    }
+    async storeResolvedEntities(noteId, notePath, stats) {
+        const now = new Date().toISOString();
+        const entities = this.entityResolver.getResolvedEntities();
+        const concepts = this.entityResolver.getResolvedConcepts();
+        const entityIndex = await this.getEntityIndex();
+        for (const entity of entities) {
+            const existingId = entityIndex.findMatch(entity.name, entity.aliases);
+            let entityId;
+            let finalNode;
+            if (existingId) {
+                // Merge with existing entity
+                entityId = existingId;
+                const existingNode = await this.storage.getNode(existingId);
+                if (existingNode) {
+                    const existingData = existingNode.data;
+                    const existingAliases = existingData.aliases ?? [];
+                    const existingTypeCounts = existingData.typeCounts ?? {};
+                    // If the existing name was classified under a single type, seed typeCounts
+                    if (Object.keys(existingTypeCounts).length === 0 && existingData.entityType) {
+                        existingTypeCounts[existingData.entityType] = 1;
+                    }
+                    // Increment count for the new entity's type
+                    const newTypeCounts = { ...existingTypeCounts };
+                    newTypeCounts[entity.entityType] = (newTypeCounts[entity.entityType] ?? 0) + 1;
+                    // Determine winning type (highest count)
+                    let winningType = entity.entityType;
+                    let maxCount = 0;
+                    for (const [type, count] of Object.entries(newTypeCounts)) {
+                        if (count > maxCount) {
+                            maxCount = count;
+                            winningType = type;
+                        }
+                    }
+                    // Union aliases
+                    const allAliases = new Set(existingAliases);
+                    for (const alias of entity.aliases)
+                        allAliases.add(alias);
+                    // Add new name as alias if different from existing name
+                    const existingName = existingData.name;
+                    if (entity.name.toLowerCase() !== existingName.toLowerCase()) {
+                        allAliases.add(entity.name);
+                    }
+                    // Remove canonical name from aliases
+                    allAliases.delete(existingName);
+                    finalNode = {
+                        id: existingId,
+                        nodeType: NodeType.Entity,
+                        data: {
+                            name: existingName,
+                            entityType: winningType,
+                            aliases: Array.from(allAliases),
+                            description: existingData.description || entity.description,
+                            confidence: Math.max(existingData.confidence, entity.confidence),
+                            typeCounts: newTypeCounts,
+                        },
+                        createdAt: existingNode.createdAt,
+                        updatedAt: now,
+                    };
+                }
+                else {
+                    // Existing ID found in index but node missing from storage — create fresh
+                    entityId = generateEntityId(entity.name);
+                    finalNode = {
+                        id: entityId,
+                        nodeType: NodeType.Entity,
+                        data: {
+                            name: entity.name,
+                            entityType: entity.entityType,
+                            aliases: entity.aliases,
+                            description: entity.description,
+                            confidence: entity.confidence,
+                            typeCounts: { [entity.entityType]: 1 },
+                        },
+                        createdAt: now,
+                        updatedAt: now,
+                    };
+                }
+            }
+            else {
+                // New entity
+                entityId = generateEntityId(entity.name);
+                finalNode = {
+                    id: entityId,
+                    nodeType: NodeType.Entity,
+                    data: {
+                        name: entity.name,
+                        entityType: entity.entityType,
+                        aliases: entity.aliases,
+                        description: entity.description,
+                        confidence: entity.confidence,
+                        typeCounts: { [entity.entityType]: 1 },
+                    },
+                    createdAt: now,
+                    updatedAt: now,
+                };
+            }
+            await this.storage.upsertNode(finalNode);
+            // Keep index current
+            const finalData = finalNode.data;
+            entityIndex.register(entityId, finalData.name, finalData.aliases ?? []);
+            // MENTIONS edges from chunks that mention this entity
+            if (entity.spans.length > 0) {
+                const noteChunks = await this.storage.queryEdges({
+                    sourceId: noteId,
+                    relType: RelType.HAS_CHUNK,
+                });
+                for (const chunkEdge of noteChunks) {
+                    const chunkNode = await this.storage.getNode(chunkEdge.targetId);
+                    if (!chunkNode)
+                        continue;
+                    const chunkText = chunkNode.data.text ?? '';
+                    if (chunkText.toLowerCase().includes(entity.name.toLowerCase())) {
+                        await this.storage.upsertEdge({
+                            id: `edge:mentions:${chunkEdge.targetId}:${entityId}`,
+                            sourceId: chunkEdge.targetId,
+                            targetId: entityId,
+                            relType: RelType.MENTIONS,
+                            data: { count: 1, spans: [] },
+                            createdAt: now,
+                        });
+                    }
+                }
+            }
+            stats.entitiesExtracted++;
+        }
+        for (const concept of concepts) {
+            const existingConceptId = entityIndex.findMatch(concept.name, []);
+            let conceptId;
+            let finalConceptNode;
+            if (existingConceptId) {
+                // Merge with existing concept
+                conceptId = existingConceptId;
+                const existingNode = await this.storage.getNode(existingConceptId);
+                if (existingNode) {
+                    const existingData = existingNode.data;
+                    const existingAliases = existingData.aliases ?? [];
+                    const allAliases = new Set(existingAliases);
+                    finalConceptNode = {
+                        id: existingConceptId,
+                        nodeType: NodeType.Concept,
+                        data: {
+                            name: existingData.name,
+                            domain: existingData.domain || concept.domain,
+                            description: existingData.description || concept.description,
+                            aliases: Array.from(allAliases),
+                            confidence: Math.max(existingData.confidence, concept.confidence),
+                        },
+                        createdAt: existingNode.createdAt,
+                        updatedAt: now,
+                    };
+                }
+                else {
+                    conceptId = generateConceptId(concept.name);
+                    finalConceptNode = {
+                        id: conceptId,
+                        nodeType: NodeType.Concept,
+                        data: {
+                            name: concept.name,
+                            domain: concept.domain,
+                            description: concept.description,
+                            aliases: [],
+                            confidence: concept.confidence,
+                        },
+                        createdAt: now,
+                        updatedAt: now,
+                    };
+                }
+            }
+            else {
+                conceptId = generateConceptId(concept.name);
+                finalConceptNode = {
+                    id: conceptId,
+                    nodeType: NodeType.Concept,
+                    data: {
+                        name: concept.name,
+                        domain: concept.domain,
+                        description: concept.description,
+                        aliases: [],
+                        confidence: concept.confidence,
+                    },
+                    createdAt: now,
+                    updatedAt: now,
+                };
+            }
+            await this.storage.upsertNode(finalConceptNode);
+            // Keep index current
+            const cData = finalConceptNode.data;
+            entityIndex.register(conceptId, cData.name, cData.aliases ?? []);
+            stats.conceptsExtracted++;
+        }
+    }
+    async cleanupOrphans() {
+        const entityIndex = await this.getEntityIndex();
+        let cleaned = 0;
+        for (const nodeType of [NodeType.Entity, NodeType.Concept]) {
+            const nodes = await this.storage.queryNodes({ nodeType });
+            for (const node of nodes) {
+                const mentionEdges = await this.storage.queryEdges({
+                    targetId: node.id,
+                    relType: RelType.MENTIONS,
+                });
+                if (mentionEdges.length === 0) {
+                    await this.storage.deleteEdgesByNode(node.id);
+                    await this.storage.deleteNode(node.id);
+                    entityIndex.remove(node.id);
+                    cleaned++;
+                }
+            }
+        }
+        return cleaned;
+    }
+    async cleanupNoteChunks(noteId, _notePath) {
+        const affectedEntityIds = new Set();
+        const existingEdges = await this.storage.queryEdges({
+            sourceId: noteId,
+            relType: RelType.HAS_CHUNK,
+        });
+        for (const edge of existingEdges) {
+            // Collect entities mentioned by this chunk before deleting
+            const mentions = await this.storage.queryEdges({
+                sourceId: edge.targetId,
+                relType: RelType.MENTIONS,
+            });
+            for (const m of mentions)
+                affectedEntityIds.add(m.targetId);
+            await this.storage.deleteFTSEntry(edge.targetId);
+            await this.storage.deleteEmbedding(edge.targetId);
+            await this.storage.deleteEdgesByNode(edge.targetId);
+            await this.storage.deleteNode(edge.targetId);
+        }
+        return affectedEntityIds;
+    }
+    async cleanupOrphanedNodes(candidateIds) {
+        const entityIndex = await this.getEntityIndex();
+        let cleaned = 0;
+        for (const nodeId of candidateIds) {
+            const node = await this.storage.getNode(nodeId);
+            if (!node)
+                continue;
+            if (node.nodeType !== NodeType.Entity && node.nodeType !== NodeType.Concept)
+                continue;
+            const mentions = await this.storage.queryEdges({ targetId: nodeId, relType: RelType.MENTIONS });
+            if (mentions.length === 0) {
+                await this.storage.deleteEdgesByNode(nodeId);
+                await this.storage.deleteNode(nodeId);
+                entityIndex.remove(nodeId);
+                cleaned++;
+            }
+        }
+        return cleaned;
+    }
+    async ensureDefaultAgent() {
+        const existing = await this.storage.getNode(DEFAULT_AGENT_ID);
+        if (!existing) {
+            await this.storage.upsertNode({
+                id: DEFAULT_AGENT_ID,
+                nodeType: NodeType.Agent,
+                data: {
+                    name: 'self',
+                    agentType: 'self',
+                    description: 'The vault owner / note author',
+                },
+                createdAt: new Date().toISOString(),
+                updatedAt: new Date().toISOString(),
+            });
+        }
+    }
+}
+function extractTags(content, frontmatter) {
+    const tags = new Set();
+    // Frontmatter tags
+    if (Array.isArray(frontmatter.tags)) {
+        for (const tag of frontmatter.tags) {
+            tags.add(String(tag));
+        }
+    }
+    // Inline #tags
+    const tagRegex = /#([a-zA-Z][\w/-]*)/g;
+    let match;
+    while ((match = tagRegex.exec(content)) !== null) {
+        tags.add(match[1]);
+    }
+    return Array.from(tags);
+}
+//# sourceMappingURL=pipeline.js.map