@echoes-io/mcp-server 4.0.0 → 4.1.1

package/src/database/schema.js

@@ -0,0 +1,69 @@
+import { sql } from 'drizzle-orm';
+import { blob, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core';
+// Helpers
+const uuidColumn = (name) => text(name)
+    .primaryKey()
+    .$default(() => crypto.randomUUID());
+const timestampColumn = (name) => text(name).default(sql `(datetime('now'))`).notNull();
+// Tables
+export const timelines = sqliteTable('timelines', {
+    id: uuidColumn('id'),
+    name: text('name').notNull(),
+    description: text('description'),
+    createdAt: timestampColumn('created_at'),
+    updatedAt: timestampColumn('updated_at'),
+});
+export const arcs = sqliteTable('arcs', {
+    id: uuidColumn('id'),
+    timelineId: text('timeline_id')
+        .notNull()
+        .references(() => timelines.id),
+    name: text('name').notNull(),
+    slug: text('slug').notNull(),
+    description: text('description'),
+    order: integer('order').notNull(),
+    createdAt: timestampColumn('created_at'),
+    updatedAt: timestampColumn('updated_at'),
+});
+export const episodes = sqliteTable('episodes', {
+    id: uuidColumn('id'),
+    arcId: text('arc_id')
+        .notNull()
+        .references(() => arcs.id),
+    number: integer('number').notNull(),
+    title: text('title').notNull(),
+    slug: text('slug').notNull(),
+    description: text('description'),
+    createdAt: timestampColumn('created_at'),
+    updatedAt: timestampColumn('updated_at'),
+});
+export const chapters = sqliteTable('chapters', {
+    id: uuidColumn('id'),
+    episodeId: text('episode_id')
+        .notNull()
+        .references(() => episodes.id),
+    number: integer('number').notNull(),
+    part: integer('part').notNull().default(1),
+    pov: text('pov').notNull(),
+    title: text('title').notNull(),
+    summary: text('summary').notNull(),
+    location: text('location').notNull(),
+    outfit: text('outfit'),
+    kink: text('kink'),
+    date: text('date'), // Simple text field
+    stats: text('stats'), // Simple text field for JSON
+    filePath: text('file_path'),
+    createdAt: timestampColumn('created_at'),
+    updatedAt: timestampColumn('updated_at'),
+});
+export const embeddings = sqliteTable('embeddings', {
+    id: uuidColumn('id'),
+    chapterId: text('chapter_id')
+        .notNull()
+        .references(() => chapters.id),
+    content: text('content').notNull(),
+    embedding: blob('embedding').notNull(),
+    characters: text('characters'), // Simple text field for JSON array
+    metadata: text('metadata'), // Simple text field for JSON
+    createdAt: timestampColumn('created_at'),
+});

package/src/database/vector.d.ts

@@ -0,0 +1,25 @@
+import type { DatabaseType } from './index.js';
+export interface VectorSearchOptions {
+    characters?: string[];
+    allCharacters?: boolean;
+    arc?: string;
+    pov?: string;
+    limit?: number;
+}
+export interface VectorSearchResult {
+    id: string;
+    chapterId: string;
+    content: string;
+    characters: string[];
+    metadata: Record<string, unknown>;
+    similarity: number;
+}
+export declare class VectorStore {
+    private db;
+    constructor(db: DatabaseType);
+    insert(chapterId: string, content: string, embedding: Float32Array, characters?: string[], metadata?: Record<string, unknown>): Promise<string>;
+    search(queryEmbedding: Float32Array, options?: VectorSearchOptions): Promise<VectorSearchResult[]>;
+    private cosineSimilarity;
+    deleteByChapter(chapterId: string): Promise<void>;
+    getCharacters(character: string): Promise<string[]>;
+}

package/src/database/vector.js

@@ -0,0 +1,98 @@
+import { eq, sql } from 'drizzle-orm';
+import { embeddings } from './schema.js';
+export class VectorStore {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    async insert(chapterId, content, embedding, characters = [], metadata = {}) {
+        const id = crypto.randomUUID();
+        await this.db.insert(embeddings).values({
+            id,
+            chapterId,
+            content,
+            embedding: Buffer.from(embedding.buffer),
+            characters: JSON.stringify(characters),
+            metadata: JSON.stringify(metadata),
+        });
+        return id;
+    }
+    async search(queryEmbedding, options = {}) {
+        const { characters, allCharacters = false, limit = 10 } = options;
+        // Start with base query
+        const baseQuery = this.db
+            .select({
+            id: embeddings.id,
+            chapterId: embeddings.chapterId,
+            content: embeddings.content,
+            characters: embeddings.characters,
+            metadata: embeddings.metadata,
+            embedding: embeddings.embedding,
+        })
+            .from(embeddings);
+        // Apply character filters if needed
+        let query = baseQuery;
+        if (characters?.length) {
+            if (allCharacters) {
+                // All characters must be present (AND)
+                for (const char of characters) {
+                    // @ts-expect-error - Drizzle beta type issues
+                    query = query.where(sql `json_extract(${embeddings.characters}, '$') LIKE ${`%${char}%`}`);
+                }
+            }
+            else {
+                // At least one character must be present (OR)
+                const charConditions = characters.map((char) => sql `json_extract(${embeddings.characters}, '$') LIKE ${`%${char}%`}`);
+                // @ts-expect-error - Drizzle beta type issues
+                query = query.where(sql `(${sql.join(charConditions, sql ` OR `)})`);
+            }
+        }
+        const results = await query;
+        // Calculate cosine similarity in JavaScript
+        const resultsWithSimilarity = results.map((row) => {
+            const rowEmbedding = new Float32Array(row.embedding.buffer);
+            const similarity = this.cosineSimilarity(queryEmbedding, rowEmbedding);
+            return {
+                id: row.id,
+                chapterId: row.chapterId,
+                content: row.content,
+                characters: JSON.parse(row.characters || '[]'),
+                metadata: JSON.parse(row.metadata || '{}'),
+                similarity,
+            };
+        });
+        // Sort by similarity and limit
+        return resultsWithSimilarity.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length)
+            return 0;
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    async deleteByChapter(chapterId) {
+        await this.db.delete(embeddings).where(eq(embeddings.chapterId, chapterId));
+    }
+    async getCharacters(character) {
+        const results = await this.db
+            .select({ characters: embeddings.characters })
+            .from(embeddings)
+            .where(sql `json_extract(${embeddings.characters}, '$') LIKE ${`%${character}%`}`);
+        const allCharacters = new Set();
+        for (const row of results) {
+            const chars = JSON.parse(row.characters || '[]');
+            for (const char of chars) {
+                allCharacters.add(char);
+            }
+        }
+        allCharacters.delete(character); // Remove the query character
+        return Array.from(allCharacters).sort();
+    }
+}

package/src/index.d.ts

@@ -0,0 +1,5 @@
+export * from './database/index.js';
+export * from './server.js';
+export * from './tools/index.js';
+export * from './types/frontmatter.js';
+export * from './utils/index.js';

package/src/index.js

@@ -0,0 +1,5 @@
+export * from './database/index.js';
+export * from './server.js';
+export * from './tools/index.js';
+export * from './types/frontmatter.js';
+export * from './utils/index.js';

package/src/rag/character-ner.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Improved Named Entity Recognition for Character Detection
+ * Optimized for Italian storytelling content
+ */
+import type { ChapterMetadata } from '../types/frontmatter.js';
+export interface CharacterNER {
+    extractCharacters(text: string, metadata?: Partial<ChapterMetadata>): string[];
+    extractMainCharacters(chapters: Array<{
+        content: string;
+        metadata: Partial<ChapterMetadata>;
+    }>): string[];
+}
+/**
+ * Rule-based NER optimized for Italian names and storytelling
+ */
+export declare class ItalianCharacterNER implements CharacterNER {
+    private commonWords;
+    private namePatterns;
+    private dialoguePatterns;
+    constructor();
+    extractCharacters(text: string, metadata?: Partial<ChapterMetadata>): string[];
+    extractMainCharacters(chapters: Array<{
+        content: string;
+        metadata: Partial<ChapterMetadata>;
+    }>): string[];
+    private extractFromOutfit;
+    private extractFromDialogue;
+    private extractFromPatterns;
+    private isValidName;
+    private isValidCharacterName;
+    private normalizeName;
+}
+/**
+ * Factory function to create character NER
+ */
+export declare function createCharacterNER(): CharacterNER;

package/src/rag/character-ner.js

@@ -0,0 +1,416 @@
+/**
+ * Improved Named Entity Recognition for Character Detection
+ * Optimized for Italian storytelling content
+ */
+/**
+ * Rule-based NER optimized for Italian names and storytelling
+ */
+export class ItalianCharacterNER {
+    commonWords;
+    namePatterns;
+    dialoguePatterns;
+    constructor() {
+        // Common Italian words to exclude (expanded list)
+        this.commonWords = new Set([
+            // Pronouns and articles
+            'io',
+            'tu',
+            'lui',
+            'lei',
+            'noi',
+            'voi',
+            'loro',
+            'il',
+            'la',
+            'lo',
+            'gli',
+            'le',
+            'un',
+            'una',
+            'uno',
+            // Common verbs
+            'sono',
+            'sei',
+            'è',
+            'siamo',
+            'siete',
+            'hanno',
+            'ho',
+            'hai',
+            'ha',
+            'abbiamo',
+            'avete',
+            'fanno',
+            'faccio',
+            'fai',
+            'fa',
+            'facciamo',
+            'fate',
+            'vado',
+            'vai',
+            'va',
+            'andiamo',
+            'andate',
+            'vanno',
+            'dico',
+            'dici',
+            'dice',
+            'diciamo',
+            'dite',
+            'dicono',
+            'vedo',
+            'vedi',
+            'vede',
+            'vediamo',
+            'vedete',
+            'vedono',
+            // Common words and negations
+            'non',
+            'no',
+            'sì',
+            'si',
+            'ne',
+            'ci',
+            'vi',
+            'mi',
+            'ti',
+            'se',
+            'ma',
+            'o',
+            'e',
+            'che',
+            'di',
+            'da',
+            'in',
+            'con',
+            'su',
+            'per',
+            'tra',
+            'fra',
+            'tutto',
+            'tutti',
+            'tutte',
+            'niente',
+            'nulla',
+            'qualcosa',
+            'qualcuno',
+            'qualche',
+            'altro',
+            'altri',
+            'altre',
+            'voglio',
+            'vuoi',
+            'vuole',
+            'vogliamo',
+            'volete',
+            'vogliono',
+            'posso',
+            'puoi',
+            'può',
+            'possiamo',
+            'potete',
+            'possono',
+            'devo',
+            'devi',
+            'deve',
+            'dobbiamo',
+            'dovete',
+            'devono',
+            'perfetto',
+            'perfetta',
+            'perfetti',
+            'perfette',
+            'sento',
+            'senti',
+            'sente',
+            'sentiamo',
+            'sentite',
+            'sentono',
+            'sto',
+            'stai',
+            'sta',
+            'stiamo',
+            'state',
+            'stanno',
+            'anche',
+            'ancora',
+            'allora',
+            'quindi',
+            'però',
+            'infatti',
+            'comunque',
+            'davvero',
+            'veramente',
+            // Common adjectives/adverbs
+            'bene',
+            'male',
+            'molto',
+            'poco',
+            'tanto',
+            'più',
+            'meno',
+            'sempre',
+            'mai',
+            'già',
+            'ancora',
+            'oggi',
+            'ieri',
+            'domani',
+            'quando',
+            'dove',
+            'come',
+            'perché',
+            'perch',
+            'cosa',
+            'chi',
+            'quale',
+            'quanto',
+            'ogni',
+            'ogni',
+            'solo',
+            'prima',
+            'dopo',
+            'mentre',
+            'durante',
+            'contro',
+            'senza',
+            'dentro',
+            'fuori',
+            'sopra',
+            'sotto',
+            // Common nouns
+            'casa',
+            'tempo',
+            'giorno',
+            'notte',
+            'mattina',
+            'sera',
+            'anno',
+            'mese',
+            'settimana',
+            'ora',
+            'minuto',
+            'occhi',
+            'mano',
+            'mani',
+            'testa',
+            'cuore',
+            'mente',
+            'corpo',
+            'voce',
+            'parole',
+            'parola',
+            // Locations (generic)
+            'milano',
+            'roma',
+            'londra',
+            'italia',
+            'inghilterra',
+            'città',
+            'paese',
+            'posto',
+            'luogo',
+            // Common expressions
+            'cazzo',
+            'merda',
+            'cristo',
+            'dio',
+            'madonna',
+            'boh',
+            'ecco',
+            'allora',
+            'però',
+            'quindi',
+            'infatti',
+            'comunque',
+            'insomma',
+            'davvero',
+            'veramente',
+            'sicuramente',
+            'probabilmente',
+            // English common words (mixed content)
+            'the',
+            'and',
+            'you',
+            'that',
+            'was',
+            'for',
+            'are',
+            'with',
+            'his',
+            'they',
+            'this',
+            'have',
+            'from',
+            'not',
+            'but',
+            'what',
+            'can',
+            'out',
+            'other',
+            'were',
+            'all',
+            'your',
+            'when',
+            'said',
+            'there',
+            'each',
+            'which',
+            'she',
+            'how',
+            'will',
+            'about',
+            'get',
+            'made',
+            'may',
+        ]);
+        // Patterns for Italian names
+        this.namePatterns = [
+            /\b[A-Z][a-z]{2,}(?:\s+[A-Z][a-z]{2,})?\b/g, // Capitalized words (potential names)
+        ];
+        // Dialogue patterns to identify speakers
+        this.dialoguePatterns = [
+            /"([^"]*)",?\s*(?:dice|chiede|risponde|sussurra|grida)\s+([A-Z][a-z]+)/gi,
+            /([A-Z][a-z]+)\s*:\s*"([^"]*)"/gi,
+            /([A-Z][a-z]+)\s+(?:dice|chiede|risponde|sussurra|grida)\s*:/gi,
+        ];
+    }
+    extractCharacters(text, metadata) {
+        const characters = new Set();
+        // 1. Add POV character if available
+        if (metadata?.pov && this.isValidName(metadata.pov)) {
+            characters.add(this.normalizeName(metadata.pov));
+        }
+        // 2. Extract from outfit metadata (reliable source)
+        if (metadata?.outfit) {
+            const outfitNames = this.extractFromOutfit(metadata.outfit);
+            for (const name of outfitNames) {
+                characters.add(name);
+            }
+        }
+        // 3. Extract from dialogue patterns (high confidence)
+        const dialogueNames = this.extractFromDialogue(text);
+        for (const name of dialogueNames) {
+            characters.add(name);
+        }
+        // 4. Extract from name patterns with frequency filtering
+        const patternNames = this.extractFromPatterns(text);
+        for (const name of patternNames) {
+            characters.add(name);
+        }
+        // 5. Filter and validate results
+        const validCharacters = Array.from(characters)
+            .filter((name) => this.isValidCharacterName(name))
+            .slice(0, 8); // Limit to 8 most likely characters
+        return validCharacters;
+    }
+    extractMainCharacters(chapters) {
+        const characterFreq = new Map();
+        const characterContexts = new Map();
+        // Extract characters from all chapters
+        chapters.forEach((chapter) => {
+            const chars = this.extractCharacters(chapter.content, chapter.metadata);
+            chars.forEach((char) => {
+                characterFreq.set(char, (characterFreq.get(char) || 0) + 1);
+                if (!characterContexts.has(char)) {
+                    characterContexts.set(char, new Set());
+                }
+                characterContexts.get(char)?.add(chapter.metadata.arc || 'unknown');
+            });
+        });
+        // Score characters based on frequency and context diversity
+        const scoredCharacters = Array.from(characterFreq.entries())
+            .map(([name, freq]) => {
+            const contexts = characterContexts.get(name)?.size || 0;
+            const score = freq * (1 + contexts * 0.5); // Bonus for appearing in multiple arcs
+            return { name, freq, contexts, score };
+        })
+            .filter((char) => char.freq >= 3) // Must appear in at least 3 chapters
+            .sort((a, b) => b.score - a.score);
+        return scoredCharacters.slice(0, 20).map((char) => char.name);
+    }
+    extractFromOutfit(outfit) {
+        const names = [];
+        const matches = outfit.match(/([A-Z][a-z]+):/g);
+        if (matches) {
+            matches.forEach((match) => {
+                const name = match.replace(':', '').trim();
+                if (this.isValidName(name)) {
+                    names.push(this.normalizeName(name));
+                }
+            });
+        }
+        return names;
+    }
+    extractFromDialogue(text) {
+        const names = new Set();
+        this.dialoguePatterns.forEach((pattern) => {
+            let match = pattern.exec(text);
+            while (match !== null) {
+                // Extract potential names from different capture groups
+                for (let i = 1; i < match.length; i++) {
+                    const potential = match[i];
+                    if (potential && this.isValidName(potential)) {
+                        names.add(this.normalizeName(potential));
+                    }
+                }
+                match = pattern.exec(text);
+            }
+        });
+        return Array.from(names);
+    }
+    extractFromPatterns(text) {
+        const nameFreq = new Map();
+        this.namePatterns.forEach((pattern) => {
+            let match = pattern.exec(text);
+            while (match !== null) {
+                const name = match[0].trim();
+                if (this.isValidName(name)) {
+                    nameFreq.set(name, (nameFreq.get(name) || 0) + 1);
+                }
+                match = pattern.exec(text);
+            }
+        });
+        // Return names that appear multiple times
+        return Array.from(nameFreq.entries())
+            .filter(([_, freq]) => freq >= 2)
+            .map(([name, _]) => name);
+    }
+    isValidName(name) {
+        if (!name || name.length < 2 || name.length > 20)
+            return false;
+        // Must start with capital letter
+        if (name[0] !== name[0].toUpperCase())
+            return false;
+        // Check against common words
+        if (this.commonWords.has(name.toLowerCase()))
+            return false;
+        // Must contain only letters (and spaces for compound names)
+        if (!/^[A-Za-z\s]+$/.test(name))
+            return false;
+        return true;
+    }
+    isValidCharacterName(name) {
+        if (!this.isValidName(name))
+            return false;
+        // Additional checks for character names
+        // Exclude obvious non-names
+        const excludePatterns = [
+            /^(chapter|capitolo|episodio|parte|arc|timeline)$/i,
+            /^(monday|tuesday|wednesday|thursday|friday|saturday|sunday)$/i,
+            /^(gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|novembre|dicembre)$/i,
+            /^(january|february|march|april|may|june|july|august|september|october|november|december)$/i,
+        ];
+        return !excludePatterns.some((pattern) => pattern.test(name));
+    }
+    normalizeName(name) {
+        // Normalize case: first letter uppercase, rest lowercase
+        return name.charAt(0).toUpperCase() + name.slice(1).toLowerCase();
+    }
+}
+/**
+ * Factory function to create character NER
+ */
+export function createCharacterNER() {
+    return new ItalianCharacterNER();
+}

package/src/rag/database-sync.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Database synchronization utilities for HybridRAG
+ * Ensures timeline/arc/episode/chapter records exist before indexing
+ */
+import type { DatabaseType } from '../database/index.js';
+export interface ChapterRecord {
+    chapterId: string;
+    timeline: string;
+    arc: string;
+    episode: number;
+    chapter: number;
+    pov: string;
+    title?: string;
+    summary?: string;
+    location?: string;
+    filePath?: string;
+}
+export declare class DatabaseSync {
+    private db;
+    constructor(db: DatabaseType);
+    /**
+     * Ensure all required database records exist for chapters
+     */
+    syncChapters(chapterRecords: ChapterRecord[]): Promise<{
+        timelines: number;
+        arcs: number;
+        episodes: number;
+        chapters: number;
+    }>;
+    /**
+     * Get chapter ID from database for embedding insertion
+     */
+    getChapterId(timeline: string, arc: string, episode: number, chapter: number): Promise<string | null>;
+    /**
+     * Clean up orphaned embeddings
+     */
+    cleanupEmbeddings(): Promise<number>;
+}