@echoes-io/mcp-server 4.1.0 → 4.1.1

package/src/rag/graph-rag.js

@@ -0,0 +1,311 @@
+/**
+ * GraphRAG implementation for Echoes storytelling platform
+ * Based on Mastra.ai GraphRAG with adaptations for chapter relationships
+ */
+export class GraphRAG {
+    nodes;
+    edges;
+    dimension;
+    threshold;
+    constructor(dimension = 384, threshold = 0.7) {
+        this.nodes = new Map();
+        this.edges = [];
+        this.dimension = dimension;
+        this.threshold = threshold;
+    }
+    // Add a node to the graph
+    addNode(node) {
+        if (!node.embedding) {
+            throw new Error('Node must have an embedding');
+        }
+        if (node.embedding.length !== this.dimension) {
+            throw new Error(`Embedding dimension must be ${this.dimension}`);
+        }
+        this.nodes.set(node.id, node);
+    }
+    // Add an edge between two nodes
+    addEdge(edge) {
+        if (!this.nodes.has(edge.source) || !this.nodes.has(edge.target)) {
+            throw new Error('Both source and target nodes must exist');
+        }
+        this.edges.push(edge);
+        // Add reverse edge for undirected relationships
+        if (edge.type === 'semantic' || edge.type === 'character') {
+            this.edges.push({
+                source: edge.target,
+                target: edge.source,
+                weight: edge.weight,
+                type: edge.type,
+            });
+        }
+    }
+    // Helper methods
+    getNodes() {
+        return Array.from(this.nodes.values());
+    }
+    getEdges() {
+        return this.edges;
+    }
+    getEdgesByType(type) {
+        return this.edges.filter((edge) => edge.type === type);
+    }
+    clear() {
+        this.nodes.clear();
+        this.edges = [];
+    }
+    updateNodeContent(id, newContent) {
+        const node = this.nodes.get(id);
+        if (!node) {
+            throw new Error(`Node ${id} not found`);
+        }
+        node.content = newContent;
+    }
+    // Get neighbors of a node
+    getNeighbors(nodeId, edgeType) {
+        return this.edges
+            .filter((edge) => edge.source === nodeId && (!edgeType || edge.type === edgeType))
+            .map((edge) => ({
+            id: edge.target,
+            weight: edge.weight,
+        }));
+    }
+    // Calculate cosine similarity between two vectors
+    cosineSimilarity(vec1, vec2) {
+        if (!vec1 || !vec2) {
+            throw new Error('Vectors must not be null or undefined');
+        }
+        if (vec1.length !== vec2.length) {
+            throw new Error(`Vector dimensions must match: vec1(${vec1.length}) !== vec2(${vec2.length})`);
+        }
+        let dotProduct = 0;
+        let normVec1 = 0;
+        let normVec2 = 0;
+        for (let i = 0; i < vec1.length; i++) {
+            const a = vec1[i];
+            const b = vec2[i];
+            dotProduct += a * b;
+            normVec1 += a * a;
+            normVec2 += b * b;
+        }
+        const magnitudeProduct = Math.sqrt(normVec1 * normVec2);
+        if (magnitudeProduct === 0)
+            return 0;
+        const similarity = dotProduct / magnitudeProduct;
+        return Math.max(-1, Math.min(1, similarity));
+    }
+    // Create graph from chapters with multiple edge types
+    createGraph(chunks, embeddings) {
+        if (!chunks?.length || !embeddings?.length) {
+            throw new Error('Chunks and embeddings arrays must not be empty');
+        }
+        if (chunks.length !== embeddings.length) {
+            throw new Error('Chunks and embeddings must have the same length');
+        }
+        // Create nodes from chunks
+        chunks.forEach((chunk, index) => {
+            const node = {
+                id: index.toString(),
+                content: chunk.text,
+                embedding: embeddings[index]?.vector,
+                metadata: chunk.metadata,
+            };
+            this.addNode(node);
+        });
+        // Create semantic edges based on cosine similarity (high threshold)
+        this.createSemanticEdges(embeddings);
+        // Create character relationship edges
+        this.createCharacterEdges();
+        // Create temporal edges (sequential chapters)
+        this.createTemporalEdges();
+        // Create location edges
+        this.createLocationEdges();
+    }
+    createSemanticEdges(embeddings) {
+        for (let i = 0; i < embeddings.length; i++) {
+            const firstEmbedding = embeddings[i]?.vector;
+            for (let j = i + 1; j < embeddings.length; j++) {
+                const secondEmbedding = embeddings[j]?.vector;
+                const similarity = this.cosineSimilarity(firstEmbedding, secondEmbedding);
+                if (similarity > this.threshold) {
+                    this.addEdge({
+                        source: i.toString(),
+                        target: j.toString(),
+                        weight: similarity,
+                        type: 'semantic',
+                    });
+                }
+            }
+        }
+    }
+    createCharacterEdges() {
+        const nodes = Array.from(this.nodes.values());
+        for (let i = 0; i < nodes.length; i++) {
+            for (let j = i + 1; j < nodes.length; j++) {
+                const node1 = nodes[i];
+                const node2 = nodes[j];
+                // Find common characters (only real character names, not noise)
+                const chars1 = new Set(node1.metadata.characters.filter((char) => char.length >= 3 && char[0] === char[0].toUpperCase()));
+                const chars2 = new Set(node2.metadata.characters.filter((char) => char.length >= 3 && char[0] === char[0].toUpperCase()));
+                const commonChars = [...chars1].filter((char) => chars2.has(char));
+                // Only create edge if there are 2+ common characters (meaningful relationship)
+                if (commonChars.length >= 2) {
+                    // Weight based on number of common characters, but capped
+                    const weight = Math.min(commonChars.length / Math.max(chars1.size, chars2.size), 0.8);
+                    this.addEdge({
+                        source: node1.id,
+                        target: node2.id,
+                        weight: weight,
+                        type: 'character',
+                    });
+                }
+            }
+        }
+    }
+    createTemporalEdges() {
+        const nodes = Array.from(this.nodes.values());
+        // Group by arc and episode
+        const episodeGroups = new Map();
+        nodes.forEach((node) => {
+            const key = `${node.metadata.arc}-${node.metadata.episode}`;
+            if (!episodeGroups.has(key)) {
+                episodeGroups.set(key, []);
+            }
+            episodeGroups.get(key)?.push(node);
+        });
+        // Create temporal edges within episodes
+        episodeGroups.forEach((episodeNodes) => {
+            episodeNodes.sort((a, b) => a.metadata.chapter - b.metadata.chapter);
+            for (let i = 0; i < episodeNodes.length - 1; i++) {
+                this.addEdge({
+                    source: episodeNodes[i].id,
+                    target: episodeNodes[i + 1].id,
+                    weight: 0.8, // High weight for sequential chapters
+                    type: 'temporal',
+                });
+            }
+        });
+    }
+    createLocationEdges() {
+        const nodes = Array.from(this.nodes.values());
+        for (let i = 0; i < nodes.length; i++) {
+            for (let j = i + 1; j < nodes.length; j++) {
+                const node1 = nodes[i];
+                const node2 = nodes[j];
+                if (node1.metadata.location &&
+                    node2.metadata.location &&
+                    node1.metadata.location === node2.metadata.location) {
+                    this.addEdge({
+                        source: node1.id,
+                        target: node2.id,
+                        weight: 0.6, // Moderate weight for same location
+                        type: 'location',
+                    });
+                }
+            }
+        }
+    }
+    // Weighted random neighbor selection for random walk
+    selectWeightedNeighbor(neighbors) {
+        const totalWeight = neighbors.reduce((sum, n) => sum + n.weight, 0);
+        let remainingWeight = Math.random() * totalWeight;
+        for (const neighbor of neighbors) {
+            remainingWeight -= neighbor.weight;
+            if (remainingWeight <= 0) {
+                return neighbor.id;
+            }
+        }
+        return neighbors[neighbors.length - 1]?.id || '';
+    }
+    // Random walk with restart for node ranking
+    randomWalkWithRestart(startNodeId, steps, restartProb, allowedNodeIds) {
+        const visits = new Map();
+        let currentNodeId = startNodeId;
+        for (let step = 0; step < steps; step++) {
+            visits.set(currentNodeId, (visits.get(currentNodeId) || 0) + 1);
+            if (Math.random() < restartProb) {
+                currentNodeId = startNodeId;
+                continue;
+            }
+            let neighbors = this.getNeighbors(currentNodeId);
+            if (allowedNodeIds) {
+                neighbors = neighbors.filter((n) => allowedNodeIds.has(n.id));
+            }
+            if (neighbors.length === 0) {
+                currentNodeId = startNodeId;
+                continue;
+            }
+            currentNodeId = this.selectWeightedNeighbor(neighbors);
+        }
+        // Normalize visits
+        const totalVisits = Array.from(visits.values()).reduce((a, b) => a + b, 0);
+        const normalizedVisits = new Map();
+        for (const [nodeId, count] of visits) {
+            normalizedVisits.set(nodeId, count / totalVisits);
+        }
+        return normalizedVisits;
+    }
+    // Main query method with character and metadata filtering
+    query({ query, topK = 10, randomWalkSteps = 100, restartProb = 0.15, characters, allCharacters = false, arc, pov, }) {
+        if (!query || query.length !== this.dimension) {
+            throw new Error(`Query embedding must have dimension ${this.dimension}`);
+        }
+        // Filter nodes based on metadata
+        const nodesToSearch = Array.from(this.nodes.values()).filter((node) => {
+            // Arc filter
+            if (arc && node.metadata.arc !== arc)
+                return false;
+            // POV filter
+            if (pov && node.metadata.pov !== pov)
+                return false;
+            // Character filter
+            if (characters?.length) {
+                const nodeChars = new Set(node.metadata.characters);
+                if (allCharacters) {
+                    // All characters must be present
+                    return characters.every((char) => nodeChars.has(char));
+                }
+                else {
+                    // At least one character must be present
+                    return characters.some((char) => nodeChars.has(char));
+                }
+            }
+            return true;
+        });
+        if (nodesToSearch.length === 0) {
+            return [];
+        }
+        // Calculate initial similarities
+        const similarities = nodesToSearch
+            .filter((node) => !!node.embedding)
+            .map((node) => ({
+            node,
+            similarity: this.cosineSimilarity(query, node.embedding),
+        }));
+        similarities.sort((a, b) => b.similarity - a.similarity);
+        const topNodes = similarities.slice(0, topK);
+        // Re-rank using random walk
+        const allowedNodeIds = new Set(nodesToSearch.map((n) => n.id));
+        const rerankedNodes = new Map();
+        for (const { node, similarity } of topNodes) {
+            const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb, allowedNodeIds);
+            for (const [nodeId, walkScore] of walkScores) {
+                const graphNode = this.nodes.get(nodeId);
+                if (!graphNode)
+                    continue;
+                const existingScore = rerankedNodes.get(nodeId)?.score || 0;
+                rerankedNodes.set(nodeId, {
+                    node: graphNode,
+                    score: existingScore + similarity * walkScore,
+                });
+            }
+        }
+        // Return final ranked results
+        return Array.from(rerankedNodes.values())
+            .sort((a, b) => b.score - a.score)
+            .slice(0, topK)
+            .map((item) => ({
+            ...item.node,
+            score: item.score,
+        }));
+    }
+}

package/src/rag/hybrid-rag.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Hybrid RAG System
+ * Combines GraphRAG (primary) with sqlite-vec (fallback) for maximum reliability
+ */
+import type { DatabaseType } from '../database/index.js';
+import { type EmbeddingConfig } from './embeddings.js';
+export interface HybridRAGConfig {
+    embedding: EmbeddingConfig;
+    graphRAG: {
+        threshold: number;
+        randomWalkSteps: number;
+        restartProb: number;
+    };
+    fallback: {
+        enabled: boolean;
+        timeout: number;
+    };
+}
+export interface SearchOptions {
+    topK?: number;
+    characters?: string[];
+    allCharacters?: boolean;
+    arc?: string;
+    pov?: string;
+    useGraphRAG?: boolean;
+}
+export interface SearchResult {
+    id: string;
+    chapterId: string;
+    content: string;
+    characters: string[];
+    metadata: Record<string, unknown>;
+    score: number;
+    source: 'graphrag' | 'vector';
+}
+export declare class HybridRAG {
+    private graphRAG;
+    private vectorStore;
+    private embedder;
+    private dbSync;
+    private config;
+    private isGraphReady;
+    constructor(db: DatabaseType, config: HybridRAGConfig);
+    /**
+     * Index chapters into both GraphRAG and vector store
+     */
+    indexChapters(chapters: Array<{
+        id: string;
+        content: string;
+        characters: string[];
+        metadata: {
+            chapterId: string;
+            arc: string;
+            episode: number;
+            chapter: number;
+            pov: string;
+            location?: string;
+            timeline?: string;
+            title?: string;
+            summary?: string;
+            filePath?: string;
+            [key: string]: unknown;
+        };
+    }>): Promise<{
+        graphNodes: number;
+        vectorEmbeddings: number;
+        dbSync: {
+            timelines: number;
+            arcs: number;
+            episodes: number;
+            chapters: number;
+        };
+    }>;
+    /**
+     * Search using hybrid approach: GraphRAG first, vector fallback
+     */
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    private searchWithGraphRAG;
+    private searchWithVector;
+    /**
+     * Get characters that co-occur with a specific character
+     */
+    getCoOccurringCharacters(character: string): Promise<string[]>;
+    /**
+     * Get system status and statistics
+     */
+    getStatus(): {
+        graphRAG: {
+            ready: boolean;
+            nodes: number;
+            edges: number;
+        };
+        vectorStore: {
+            ready: boolean;
+        };
+        embedder: {
+            name: string;
+            dimension: number;
+        };
+    };
+    /**
+     * Clear all indexes
+     */
+    clear(): Promise<void>;
+}
+/**
+ * Default configuration for hybrid RAG
+ */
+export declare const DEFAULT_HYBRID_CONFIG: HybridRAGConfig;

package/src/rag/hybrid-rag.js

@@ -0,0 +1,255 @@
+/**
+ * Hybrid RAG System
+ * Combines GraphRAG (primary) with sqlite-vec (fallback) for maximum reliability
+ */
+import { VectorStore } from '../database/vector.js';
+import { DatabaseSync } from './database-sync.js';
+import { createEmbeddingProvider, } from './embeddings.js';
+import { GraphRAG } from './graph-rag.js';
+export class HybridRAG {
+    graphRAG;
+    vectorStore;
+    embedder;
+    dbSync;
+    config;
+    isGraphReady = false;
+    constructor(db, config) {
+        this.config = config;
+        this.embedder = createEmbeddingProvider(config.embedding);
+        this.graphRAG = new GraphRAG(this.embedder.dimension, config.graphRAG.threshold);
+        this.vectorStore = new VectorStore(db);
+        this.dbSync = new DatabaseSync(db);
+    }
+    /**
+     * Index chapters into both GraphRAG and vector store
+     */
+    async indexChapters(chapters) {
+        if (chapters.length === 0) {
+            return {
+                graphNodes: 0,
+                vectorEmbeddings: 0,
+                dbSync: { timelines: 0, arcs: 0, episodes: 0, chapters: 0 },
+            };
+        }
+        try {
+            // 1. Sync database first - ensure all timeline/arc/episode/chapter records exist
+            const chapterRecords = chapters.map((ch) => ({
+                chapterId: ch.metadata.chapterId,
+                timeline: ch.metadata.timeline || 'default',
+                arc: ch.metadata.arc,
+                episode: ch.metadata.episode,
+                chapter: ch.metadata.chapter,
+                pov: ch.metadata.pov,
+                title: ch.metadata.title,
+                summary: ch.metadata.summary,
+                location: ch.metadata.location,
+                filePath: ch.metadata.filePath,
+            }));
+            const syncStats = await this.dbSync.syncChapters(chapterRecords);
+            // 2. Generate embeddings for all chapters
+            const texts = chapters.map((ch) => ch.content);
+            const embeddings = await this.embedder.embed(texts);
+            // 3. Index into GraphRAG
+            const chunks = chapters.map((ch, _i) => ({
+                text: ch.content,
+                metadata: {
+                    ...ch.metadata,
+                    characters: ch.characters,
+                    timeline: ch.metadata.timeline || 'default',
+                },
+            }));
+            const graphEmbeddings = embeddings.map((emb) => ({ vector: emb }));
+            this.graphRAG.clear();
+            this.graphRAG.createGraph(chunks, graphEmbeddings);
+            this.isGraphReady = true;
+            // 4. Index into vector store (fallback) - get real chapter IDs from database
+            let vectorCount = 0;
+            for (let i = 0; i < chapters.length; i++) {
+                const ch = chapters[i];
+                const realChapterId = await this.dbSync.getChapterId(ch.metadata.timeline || 'default', ch.metadata.arc, ch.metadata.episode, ch.metadata.chapter);
+                if (realChapterId) {
+                    await this.vectorStore.insert(realChapterId, ch.content, new Float32Array(embeddings[i]), ch.characters, ch.metadata);
+                    vectorCount++;
+                }
+                else {
+                    console.warn(`Could not find database ID for chapter ${ch.metadata.chapterId}`);
+                }
+            }
+            return {
+                graphNodes: chunks.length,
+                vectorEmbeddings: vectorCount,
+                dbSync: syncStats,
+            };
+        }
+        catch (error) {
+            console.error('Failed to index chapters:', error);
+            throw error;
+        }
+    }
+    /**
+     * Search using hybrid approach: GraphRAG first, vector fallback
+     */
+    async search(query, options = {}) {
+        const { topK = 10, characters, allCharacters = false, arc, pov, useGraphRAG = true } = options;
+        try {
+            // Generate query embedding
+            const queryEmbedding = await this.embedder.embedSingle(query);
+            // Try GraphRAG first (if enabled and ready)
+            if (useGraphRAG && this.isGraphReady && this.config.fallback.enabled) {
+                try {
+                    const graphResults = await Promise.race([
+                        this.searchWithGraphRAG(queryEmbedding, {
+                            topK,
+                            characters,
+                            allCharacters,
+                            arc,
+                            pov,
+                        }),
+                        new Promise((_, reject) => setTimeout(() => reject(new Error('GraphRAG timeout')), this.config.fallback.timeout)),
+                    ]);
+                    if (graphResults.length > 0) {
+                        return graphResults.map((result) => ({
+                            ...result,
+                            source: 'graphrag',
+                        }));
+                    }
+                }
+                catch (error) {
+                    console.warn('GraphRAG failed, falling back to vector search:', error);
+                }
+            }
+            // Fallback to vector search
+            return await this.searchWithVector(queryEmbedding, {
+                topK,
+                characters,
+                allCharacters,
+                arc,
+                pov,
+            });
+        }
+        catch (error) {
+            console.error('Hybrid search failed:', error);
+            throw error;
+        }
+    }
+    async searchWithGraphRAG(queryEmbedding, options) {
+        const results = this.graphRAG.query({
+            query: queryEmbedding,
+            topK: options.topK,
+            randomWalkSteps: this.config.graphRAG.randomWalkSteps,
+            restartProb: this.config.graphRAG.restartProb,
+            characters: options.characters,
+            allCharacters: options.allCharacters,
+            arc: options.arc,
+            pov: options.pov,
+        });
+        return results.map((result) => ({
+            id: result.id,
+            chapterId: result.metadata.chapterId,
+            content: result.content,
+            characters: result.metadata.characters,
+            metadata: result.metadata,
+            score: result.score,
+            source: 'graphrag',
+        }));
+    }
+    async searchWithVector(queryEmbedding, options) {
+        try {
+            const results = await this.vectorStore.search(new Float32Array(queryEmbedding), {
+                characters: options.characters,
+                allCharacters: options.allCharacters,
+                limit: options.topK || 10,
+            });
+            return results.map((result) => ({
+                id: result.id,
+                chapterId: result.chapterId,
+                content: result.content,
+                characters: result.characters,
+                metadata: result.metadata,
+                score: result.similarity,
+                source: 'vector',
+            }));
+        }
+        catch (error) {
+            console.warn('Vector search failed:', error);
+            return [];
+        }
+    }
+    /**
+     * Get characters that co-occur with a specific character
+     */
+    async getCoOccurringCharacters(character) {
+        try {
+            // Try GraphRAG approach first
+            if (this.isGraphReady) {
+                const nodes = this.graphRAG.getNodes();
+                const coOccurring = new Set();
+                nodes.forEach((node) => {
+                    if (node.metadata.characters.includes(character)) {
+                        node.metadata.characters.forEach((char) => {
+                            if (char !== character) {
+                                coOccurring.add(char);
+                            }
+                        });
+                    }
+                });
+                if (coOccurring.size > 0) {
+                    return Array.from(coOccurring).sort();
+                }
+            }
+            // Fallback to vector store
+            return await this.vectorStore.getCharacters(character);
+        }
+        catch (error) {
+            console.error('Failed to get co-occurring characters:', error);
+            return [];
+        }
+    }
+    /**
+     * Get system status and statistics
+     */
+    getStatus() {
+        return {
+            graphRAG: {
+                ready: this.isGraphReady,
+                nodes: this.graphRAG.getNodes().length,
+                edges: this.graphRAG.getEdges().length,
+            },
+            vectorStore: {
+                ready: true, // Vector store is always ready if DB is available
+            },
+            embedder: {
+                name: this.embedder.name,
+                dimension: this.embedder.dimension,
+            },
+        };
+    }
+    /**
+     * Clear all indexes
+     */
+    async clear() {
+        this.graphRAG.clear();
+        this.isGraphReady = false;
+        // Note: Vector store clearing would require database operations
+        // This is handled at the database level
+    }
+}
+/**
+ * Default configuration for hybrid RAG
+ */
+export const DEFAULT_HYBRID_CONFIG = {
+    embedding: {
+        provider: 'bge-base',
+        batchSize: 32,
+        maxTokens: 512,
+    },
+    graphRAG: {
+        threshold: 0.8, // Higher threshold for more selective connections
+        randomWalkSteps: 100,
+        restartProb: 0.15,
+    },
+    fallback: {
+        enabled: true,
+        timeout: 5000, // 5 seconds
+    },
+};

package/src/rag/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * RAG Module - GraphRAG + Vector Search for Echoes
+ *
+ * Provides hybrid search capabilities combining:
+ * - GraphRAG: Semantic relationships, character connections, temporal sequences
+ * - Vector Search: Fast similarity search with sqlite-vec fallback
+ */
+export { BGEBaseEmbedding, batchArray, cosineSimilarity, createEmbeddingProvider, E5SmallEmbedding, type EmbeddingConfig, type EmbeddingProvider, GeminiEmbedding, normalizeEmbedding, } from './embeddings.js';
+export { type GraphChunk, type GraphEdge, type GraphEmbedding, type GraphNode, GraphRAG, type RankedNode, type SupportedEdgeType, } from './graph-rag.js';
+export { DEFAULT_HYBRID_CONFIG, HybridRAG, type HybridRAGConfig, type SearchOptions, type SearchResult, } from './hybrid-rag.js';
+/**
+ * Quick setup function for common use cases
+ */
+import type { DatabaseType } from '../database/index.js';
+import { HybridRAG, type HybridRAGConfig } from './hybrid-rag.js';
+export declare function createHybridRAG(db: DatabaseType, config?: Partial<HybridRAGConfig>): HybridRAG;