@echoes-io/mcp-server 4.1.1 → 6.0.0

package/src/rag/graph-rag.js

@@ -1,311 +0,0 @@
-/**
- * GraphRAG implementation for Echoes storytelling platform
- * Based on Mastra.ai GraphRAG with adaptations for chapter relationships
- */
-export class GraphRAG {
-    nodes;
-    edges;
-    dimension;
-    threshold;
-    constructor(dimension = 384, threshold = 0.7) {
-        this.nodes = new Map();
-        this.edges = [];
-        this.dimension = dimension;
-        this.threshold = threshold;
-    }
-    // Add a node to the graph
-    addNode(node) {
-        if (!node.embedding) {
-            throw new Error('Node must have an embedding');
-        }
-        if (node.embedding.length !== this.dimension) {
-            throw new Error(`Embedding dimension must be ${this.dimension}`);
-        }
-        this.nodes.set(node.id, node);
-    }
-    // Add an edge between two nodes
-    addEdge(edge) {
-        if (!this.nodes.has(edge.source) || !this.nodes.has(edge.target)) {
-            throw new Error('Both source and target nodes must exist');
-        }
-        this.edges.push(edge);
-        // Add reverse edge for undirected relationships
-        if (edge.type === 'semantic' || edge.type === 'character') {
-            this.edges.push({
-                source: edge.target,
-                target: edge.source,
-                weight: edge.weight,
-                type: edge.type,
-            });
-        }
-    }
-    // Helper methods
-    getNodes() {
-        return Array.from(this.nodes.values());
-    }
-    getEdges() {
-        return this.edges;
-    }
-    getEdgesByType(type) {
-        return this.edges.filter((edge) => edge.type === type);
-    }
-    clear() {
-        this.nodes.clear();
-        this.edges = [];
-    }
-    updateNodeContent(id, newContent) {
-        const node = this.nodes.get(id);
-        if (!node) {
-            throw new Error(`Node ${id} not found`);
-        }
-        node.content = newContent;
-    }
-    // Get neighbors of a node
-    getNeighbors(nodeId, edgeType) {
-        return this.edges
-            .filter((edge) => edge.source === nodeId && (!edgeType || edge.type === edgeType))
-            .map((edge) => ({
-            id: edge.target,
-            weight: edge.weight,
-        }));
-    }
-    // Calculate cosine similarity between two vectors
-    cosineSimilarity(vec1, vec2) {
-        if (!vec1 || !vec2) {
-            throw new Error('Vectors must not be null or undefined');
-        }
-        if (vec1.length !== vec2.length) {
-            throw new Error(`Vector dimensions must match: vec1(${vec1.length}) !== vec2(${vec2.length})`);
-        }
-        let dotProduct = 0;
-        let normVec1 = 0;
-        let normVec2 = 0;
-        for (let i = 0; i < vec1.length; i++) {
-            const a = vec1[i];
-            const b = vec2[i];
-            dotProduct += a * b;
-            normVec1 += a * a;
-            normVec2 += b * b;
-        }
-        const magnitudeProduct = Math.sqrt(normVec1 * normVec2);
-        if (magnitudeProduct === 0)
-            return 0;
-        const similarity = dotProduct / magnitudeProduct;
-        return Math.max(-1, Math.min(1, similarity));
-    }
-    // Create graph from chapters with multiple edge types
-    createGraph(chunks, embeddings) {
-        if (!chunks?.length || !embeddings?.length) {
-            throw new Error('Chunks and embeddings arrays must not be empty');
-        }
-        if (chunks.length !== embeddings.length) {
-            throw new Error('Chunks and embeddings must have the same length');
-        }
-        // Create nodes from chunks
-        chunks.forEach((chunk, index) => {
-            const node = {
-                id: index.toString(),
-                content: chunk.text,
-                embedding: embeddings[index]?.vector,
-                metadata: chunk.metadata,
-            };
-            this.addNode(node);
-        });
-        // Create semantic edges based on cosine similarity (high threshold)
-        this.createSemanticEdges(embeddings);
-        // Create character relationship edges
-        this.createCharacterEdges();
-        // Create temporal edges (sequential chapters)
-        this.createTemporalEdges();
-        // Create location edges
-        this.createLocationEdges();
-    }
-    createSemanticEdges(embeddings) {
-        for (let i = 0; i < embeddings.length; i++) {
-            const firstEmbedding = embeddings[i]?.vector;
-            for (let j = i + 1; j < embeddings.length; j++) {
-                const secondEmbedding = embeddings[j]?.vector;
-                const similarity = this.cosineSimilarity(firstEmbedding, secondEmbedding);
-                if (similarity > this.threshold) {
-                    this.addEdge({
-                        source: i.toString(),
-                        target: j.toString(),
-                        weight: similarity,
-                        type: 'semantic',
-                    });
-                }
-            }
-        }
-    }
-    createCharacterEdges() {
-        const nodes = Array.from(this.nodes.values());
-        for (let i = 0; i < nodes.length; i++) {
-            for (let j = i + 1; j < nodes.length; j++) {
-                const node1 = nodes[i];
-                const node2 = nodes[j];
-                // Find common characters (only real character names, not noise)
-                const chars1 = new Set(node1.metadata.characters.filter((char) => char.length >= 3 && char[0] === char[0].toUpperCase()));
-                const chars2 = new Set(node2.metadata.characters.filter((char) => char.length >= 3 && char[0] === char[0].toUpperCase()));
-                const commonChars = [...chars1].filter((char) => chars2.has(char));
-                // Only create edge if there are 2+ common characters (meaningful relationship)
-                if (commonChars.length >= 2) {
-                    // Weight based on number of common characters, but capped
-                    const weight = Math.min(commonChars.length / Math.max(chars1.size, chars2.size), 0.8);
-                    this.addEdge({
-                        source: node1.id,
-                        target: node2.id,
-                        weight: weight,
-                        type: 'character',
-                    });
-                }
-            }
-        }
-    }
-    createTemporalEdges() {
-        const nodes = Array.from(this.nodes.values());
-        // Group by arc and episode
-        const episodeGroups = new Map();
-        nodes.forEach((node) => {
-            const key = `${node.metadata.arc}-${node.metadata.episode}`;
-            if (!episodeGroups.has(key)) {
-                episodeGroups.set(key, []);
-            }
-            episodeGroups.get(key)?.push(node);
-        });
-        // Create temporal edges within episodes
-        episodeGroups.forEach((episodeNodes) => {
-            episodeNodes.sort((a, b) => a.metadata.chapter - b.metadata.chapter);
-            for (let i = 0; i < episodeNodes.length - 1; i++) {
-                this.addEdge({
-                    source: episodeNodes[i].id,
-                    target: episodeNodes[i + 1].id,
-                    weight: 0.8, // High weight for sequential chapters
-                    type: 'temporal',
-                });
-            }
-        });
-    }
-    createLocationEdges() {
-        const nodes = Array.from(this.nodes.values());
-        for (let i = 0; i < nodes.length; i++) {
-            for (let j = i + 1; j < nodes.length; j++) {
-                const node1 = nodes[i];
-                const node2 = nodes[j];
-                if (node1.metadata.location &&
-                    node2.metadata.location &&
-                    node1.metadata.location === node2.metadata.location) {
-                    this.addEdge({
-                        source: node1.id,
-                        target: node2.id,
-                        weight: 0.6, // Moderate weight for same location
-                        type: 'location',
-                    });
-                }
-            }
-        }
-    }
-    // Weighted random neighbor selection for random walk
-    selectWeightedNeighbor(neighbors) {
-        const totalWeight = neighbors.reduce((sum, n) => sum + n.weight, 0);
-        let remainingWeight = Math.random() * totalWeight;
-        for (const neighbor of neighbors) {
-            remainingWeight -= neighbor.weight;
-            if (remainingWeight <= 0) {
-                return neighbor.id;
-            }
-        }
-        return neighbors[neighbors.length - 1]?.id || '';
-    }
-    // Random walk with restart for node ranking
-    randomWalkWithRestart(startNodeId, steps, restartProb, allowedNodeIds) {
-        const visits = new Map();
-        let currentNodeId = startNodeId;
-        for (let step = 0; step < steps; step++) {
-            visits.set(currentNodeId, (visits.get(currentNodeId) || 0) + 1);
-            if (Math.random() < restartProb) {
-                currentNodeId = startNodeId;
-                continue;
-            }
-            let neighbors = this.getNeighbors(currentNodeId);
-            if (allowedNodeIds) {
-                neighbors = neighbors.filter((n) => allowedNodeIds.has(n.id));
-            }
-            if (neighbors.length === 0) {
-                currentNodeId = startNodeId;
-                continue;
-            }
-            currentNodeId = this.selectWeightedNeighbor(neighbors);
-        }
-        // Normalize visits
-        const totalVisits = Array.from(visits.values()).reduce((a, b) => a + b, 0);
-        const normalizedVisits = new Map();
-        for (const [nodeId, count] of visits) {
-            normalizedVisits.set(nodeId, count / totalVisits);
-        }
-        return normalizedVisits;
-    }
-    // Main query method with character and metadata filtering
-    query({ query, topK = 10, randomWalkSteps = 100, restartProb = 0.15, characters, allCharacters = false, arc, pov, }) {
-        if (!query || query.length !== this.dimension) {
-            throw new Error(`Query embedding must have dimension ${this.dimension}`);
-        }
-        // Filter nodes based on metadata
-        const nodesToSearch = Array.from(this.nodes.values()).filter((node) => {
-            // Arc filter
-            if (arc && node.metadata.arc !== arc)
-                return false;
-            // POV filter
-            if (pov && node.metadata.pov !== pov)
-                return false;
-            // Character filter
-            if (characters?.length) {
-                const nodeChars = new Set(node.metadata.characters);
-                if (allCharacters) {
-                    // All characters must be present
-                    return characters.every((char) => nodeChars.has(char));
-                }
-                else {
-                    // At least one character must be present
-                    return characters.some((char) => nodeChars.has(char));
-                }
-            }
-            return true;
-        });
-        if (nodesToSearch.length === 0) {
-            return [];
-        }
-        // Calculate initial similarities
-        const similarities = nodesToSearch
-            .filter((node) => !!node.embedding)
-            .map((node) => ({
-            node,
-            similarity: this.cosineSimilarity(query, node.embedding),
-        }));
-        similarities.sort((a, b) => b.similarity - a.similarity);
-        const topNodes = similarities.slice(0, topK);
-        // Re-rank using random walk
-        const allowedNodeIds = new Set(nodesToSearch.map((n) => n.id));
-        const rerankedNodes = new Map();
-        for (const { node, similarity } of topNodes) {
-            const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb, allowedNodeIds);
-            for (const [nodeId, walkScore] of walkScores) {
-                const graphNode = this.nodes.get(nodeId);
-                if (!graphNode)
-                    continue;
-                const existingScore = rerankedNodes.get(nodeId)?.score || 0;
-                rerankedNodes.set(nodeId, {
-                    node: graphNode,
-                    score: existingScore + similarity * walkScore,
-                });
-            }
-        }
-        // Return final ranked results
-        return Array.from(rerankedNodes.values())
-            .sort((a, b) => b.score - a.score)
-            .slice(0, topK)
-            .map((item) => ({
-            ...item.node,
-            score: item.score,
-        }));
-    }
-}

package/src/rag/hybrid-rag.d.ts

@@ -1,109 +0,0 @@
-/**
- * Hybrid RAG System
- * Combines GraphRAG (primary) with sqlite-vec (fallback) for maximum reliability
- */
-import type { DatabaseType } from '../database/index.js';
-import { type EmbeddingConfig } from './embeddings.js';
-export interface HybridRAGConfig {
-    embedding: EmbeddingConfig;
-    graphRAG: {
-        threshold: number;
-        randomWalkSteps: number;
-        restartProb: number;
-    };
-    fallback: {
-        enabled: boolean;
-        timeout: number;
-    };
-}
-export interface SearchOptions {
-    topK?: number;
-    characters?: string[];
-    allCharacters?: boolean;
-    arc?: string;
-    pov?: string;
-    useGraphRAG?: boolean;
-}
-export interface SearchResult {
-    id: string;
-    chapterId: string;
-    content: string;
-    characters: string[];
-    metadata: Record<string, unknown>;
-    score: number;
-    source: 'graphrag' | 'vector';
-}
-export declare class HybridRAG {
-    private graphRAG;
-    private vectorStore;
-    private embedder;
-    private dbSync;
-    private config;
-    private isGraphReady;
-    constructor(db: DatabaseType, config: HybridRAGConfig);
-    /**
-     * Index chapters into both GraphRAG and vector store
-     */
-    indexChapters(chapters: Array<{
-        id: string;
-        content: string;
-        characters: string[];
-        metadata: {
-            chapterId: string;
-            arc: string;
-            episode: number;
-            chapter: number;
-            pov: string;
-            location?: string;
-            timeline?: string;
-            title?: string;
-            summary?: string;
-            filePath?: string;
-            [key: string]: unknown;
-        };
-    }>): Promise<{
-        graphNodes: number;
-        vectorEmbeddings: number;
-        dbSync: {
-            timelines: number;
-            arcs: number;
-            episodes: number;
-            chapters: number;
-        };
-    }>;
-    /**
-     * Search using hybrid approach: GraphRAG first, vector fallback
-     */
-    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
-    private searchWithGraphRAG;
-    private searchWithVector;
-    /**
-     * Get characters that co-occur with a specific character
-     */
-    getCoOccurringCharacters(character: string): Promise<string[]>;
-    /**
-     * Get system status and statistics
-     */
-    getStatus(): {
-        graphRAG: {
-            ready: boolean;
-            nodes: number;
-            edges: number;
-        };
-        vectorStore: {
-            ready: boolean;
-        };
-        embedder: {
-            name: string;
-            dimension: number;
-        };
-    };
-    /**
-     * Clear all indexes
-     */
-    clear(): Promise<void>;
-}
-/**
- * Default configuration for hybrid RAG
- */
-export declare const DEFAULT_HYBRID_CONFIG: HybridRAGConfig;

package/src/rag/hybrid-rag.js

@@ -1,255 +0,0 @@
-/**
- * Hybrid RAG System
- * Combines GraphRAG (primary) with sqlite-vec (fallback) for maximum reliability
- */
-import { VectorStore } from '../database/vector.js';
-import { DatabaseSync } from './database-sync.js';
-import { createEmbeddingProvider, } from './embeddings.js';
-import { GraphRAG } from './graph-rag.js';
-export class HybridRAG {
-    graphRAG;
-    vectorStore;
-    embedder;
-    dbSync;
-    config;
-    isGraphReady = false;
-    constructor(db, config) {
-        this.config = config;
-        this.embedder = createEmbeddingProvider(config.embedding);
-        this.graphRAG = new GraphRAG(this.embedder.dimension, config.graphRAG.threshold);
-        this.vectorStore = new VectorStore(db);
-        this.dbSync = new DatabaseSync(db);
-    }
-    /**
-     * Index chapters into both GraphRAG and vector store
-     */
-    async indexChapters(chapters) {
-        if (chapters.length === 0) {
-            return {
-                graphNodes: 0,
-                vectorEmbeddings: 0,
-                dbSync: { timelines: 0, arcs: 0, episodes: 0, chapters: 0 },
-            };
-        }
-        try {
-            // 1. Sync database first - ensure all timeline/arc/episode/chapter records exist
-            const chapterRecords = chapters.map((ch) => ({
-                chapterId: ch.metadata.chapterId,
-                timeline: ch.metadata.timeline || 'default',
-                arc: ch.metadata.arc,
-                episode: ch.metadata.episode,
-                chapter: ch.metadata.chapter,
-                pov: ch.metadata.pov,
-                title: ch.metadata.title,
-                summary: ch.metadata.summary,
-                location: ch.metadata.location,
-                filePath: ch.metadata.filePath,
-            }));
-            const syncStats = await this.dbSync.syncChapters(chapterRecords);
-            // 2. Generate embeddings for all chapters
-            const texts = chapters.map((ch) => ch.content);
-            const embeddings = await this.embedder.embed(texts);
-            // 3. Index into GraphRAG
-            const chunks = chapters.map((ch, _i) => ({
-                text: ch.content,
-                metadata: {
-                    ...ch.metadata,
-                    characters: ch.characters,
-                    timeline: ch.metadata.timeline || 'default',
-                },
-            }));
-            const graphEmbeddings = embeddings.map((emb) => ({ vector: emb }));
-            this.graphRAG.clear();
-            this.graphRAG.createGraph(chunks, graphEmbeddings);
-            this.isGraphReady = true;
-            // 4. Index into vector store (fallback) - get real chapter IDs from database
-            let vectorCount = 0;
-            for (let i = 0; i < chapters.length; i++) {
-                const ch = chapters[i];
-                const realChapterId = await this.dbSync.getChapterId(ch.metadata.timeline || 'default', ch.metadata.arc, ch.metadata.episode, ch.metadata.chapter);
-                if (realChapterId) {
-                    await this.vectorStore.insert(realChapterId, ch.content, new Float32Array(embeddings[i]), ch.characters, ch.metadata);
-                    vectorCount++;
-                }
-                else {
-                    console.warn(`Could not find database ID for chapter ${ch.metadata.chapterId}`);
-                }
-            }
-            return {
-                graphNodes: chunks.length,
-                vectorEmbeddings: vectorCount,
-                dbSync: syncStats,
-            };
-        }
-        catch (error) {
-            console.error('Failed to index chapters:', error);
-            throw error;
-        }
-    }
-    /**
-     * Search using hybrid approach: GraphRAG first, vector fallback
-     */
-    async search(query, options = {}) {
-        const { topK = 10, characters, allCharacters = false, arc, pov, useGraphRAG = true } = options;
-        try {
-            // Generate query embedding
-            const queryEmbedding = await this.embedder.embedSingle(query);
-            // Try GraphRAG first (if enabled and ready)
-            if (useGraphRAG && this.isGraphReady && this.config.fallback.enabled) {
-                try {
-                    const graphResults = await Promise.race([
-                        this.searchWithGraphRAG(queryEmbedding, {
-                            topK,
-                            characters,
-                            allCharacters,
-                            arc,
-                            pov,
-                        }),
-                        new Promise((_, reject) => setTimeout(() => reject(new Error('GraphRAG timeout')), this.config.fallback.timeout)),
-                    ]);
-                    if (graphResults.length > 0) {
-                        return graphResults.map((result) => ({
-                            ...result,
-                            source: 'graphrag',
-                        }));
-                    }
-                }
-                catch (error) {
-                    console.warn('GraphRAG failed, falling back to vector search:', error);
-                }
-            }
-            // Fallback to vector search
-            return await this.searchWithVector(queryEmbedding, {
-                topK,
-                characters,
-                allCharacters,
-                arc,
-                pov,
-            });
-        }
-        catch (error) {
-            console.error('Hybrid search failed:', error);
-            throw error;
-        }
-    }
-    async searchWithGraphRAG(queryEmbedding, options) {
-        const results = this.graphRAG.query({
-            query: queryEmbedding,
-            topK: options.topK,
-            randomWalkSteps: this.config.graphRAG.randomWalkSteps,
-            restartProb: this.config.graphRAG.restartProb,
-            characters: options.characters,
-            allCharacters: options.allCharacters,
-            arc: options.arc,
-            pov: options.pov,
-        });
-        return results.map((result) => ({
-            id: result.id,
-            chapterId: result.metadata.chapterId,
-            content: result.content,
-            characters: result.metadata.characters,
-            metadata: result.metadata,
-            score: result.score,
-            source: 'graphrag',
-        }));
-    }
-    async searchWithVector(queryEmbedding, options) {
-        try {
-            const results = await this.vectorStore.search(new Float32Array(queryEmbedding), {
-                characters: options.characters,
-                allCharacters: options.allCharacters,
-                limit: options.topK || 10,
-            });
-            return results.map((result) => ({
-                id: result.id,
-                chapterId: result.chapterId,
-                content: result.content,
-                characters: result.characters,
-                metadata: result.metadata,
-                score: result.similarity,
-                source: 'vector',
-            }));
-        }
-        catch (error) {
-            console.warn('Vector search failed:', error);
-            return [];
-        }
-    }
-    /**
-     * Get characters that co-occur with a specific character
-     */
-    async getCoOccurringCharacters(character) {
-        try {
-            // Try GraphRAG approach first
-            if (this.isGraphReady) {
-                const nodes = this.graphRAG.getNodes();
-                const coOccurring = new Set();
-                nodes.forEach((node) => {
-                    if (node.metadata.characters.includes(character)) {
-                        node.metadata.characters.forEach((char) => {
-                            if (char !== character) {
-                                coOccurring.add(char);
-                            }
-                        });
-                    }
-                });
-                if (coOccurring.size > 0) {
-                    return Array.from(coOccurring).sort();
-                }
-            }
-            // Fallback to vector store
-            return await this.vectorStore.getCharacters(character);
-        }
-        catch (error) {
-            console.error('Failed to get co-occurring characters:', error);
-            return [];
-        }
-    }
-    /**
-     * Get system status and statistics
-     */
-    getStatus() {
-        return {
-            graphRAG: {
-                ready: this.isGraphReady,
-                nodes: this.graphRAG.getNodes().length,
-                edges: this.graphRAG.getEdges().length,
-            },
-            vectorStore: {
-                ready: true, // Vector store is always ready if DB is available
-            },
-            embedder: {
-                name: this.embedder.name,
-                dimension: this.embedder.dimension,
-            },
-        };
-    }
-    /**
-     * Clear all indexes
-     */
-    async clear() {
-        this.graphRAG.clear();
-        this.isGraphReady = false;
-        // Note: Vector store clearing would require database operations
-        // This is handled at the database level
-    }
-}
-/**
- * Default configuration for hybrid RAG
- */
-export const DEFAULT_HYBRID_CONFIG = {
-    embedding: {
-        provider: 'bge-base',
-        batchSize: 32,
-        maxTokens: 512,
-    },
-    graphRAG: {
-        threshold: 0.8, // Higher threshold for more selective connections
-        randomWalkSteps: 100,
-        restartProb: 0.15,
-    },
-    fallback: {
-        enabled: true,
-        timeout: 5000, // 5 seconds
-    },
-};

package/src/rag/index.d.ts

@@ -1,16 +0,0 @@
-/**
- * RAG Module - GraphRAG + Vector Search for Echoes
- *
- * Provides hybrid search capabilities combining:
- * - GraphRAG: Semantic relationships, character connections, temporal sequences
- * - Vector Search: Fast similarity search with sqlite-vec fallback
- */
-export { BGEBaseEmbedding, batchArray, cosineSimilarity, createEmbeddingProvider, E5SmallEmbedding, type EmbeddingConfig, type EmbeddingProvider, GeminiEmbedding, normalizeEmbedding, } from './embeddings.js';
-export { type GraphChunk, type GraphEdge, type GraphEmbedding, type GraphNode, GraphRAG, type RankedNode, type SupportedEdgeType, } from './graph-rag.js';
-export { DEFAULT_HYBRID_CONFIG, HybridRAG, type HybridRAGConfig, type SearchOptions, type SearchResult, } from './hybrid-rag.js';
-/**
- * Quick setup function for common use cases
- */
-import type { DatabaseType } from '../database/index.js';
-import { HybridRAG, type HybridRAGConfig } from './hybrid-rag.js';
-export declare function createHybridRAG(db: DatabaseType, config?: Partial<HybridRAGConfig>): HybridRAG;