memory-journal-mcp 3.0.0

package/src/types/sql.js.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Type declarations for sql.js (WASM-based SQLite)
+ */
+
+declare module 'sql.js' {
+    export type SqlValue = Record<string, unknown>;
+
+    export interface QueryExecResult {
+        columns: string[];
+        values: unknown[][];
+    }
+
+    export type ParamsObject = Record<string, unknown>;
+
+    export type BindParams = unknown[] | ParamsObject | null;
+
+    export interface Database {
+        run(sql: string, params?: BindParams): void;
+        exec(sql: string, params?: BindParams): QueryExecResult[];
+        each(sql: string, params: BindParams, callback: (row: Record<string, unknown>) => void): void;
+        export(): Uint8Array;
+        close(): void;
+    }
+
+    export interface SqlJsStatic {
+        Database: new (data?: ArrayLike<number>) => Database;
+    }
+
+    export interface InitSqlJsOptions {
+        locateFile?: (filename: string) => string;
+    }
+
+    export default function initSqlJs(options?: InitSqlJsOptions): Promise<SqlJsStatic>;
+}

package/src/utils/McpLogger.ts

@@ -0,0 +1,155 @@
+/**
+ * Memory Journal MCP Server - MCP Protocol Logger
+ * 
+ * Logging that sends structured messages via MCP notifications/message.
+ * Falls back to stderr when MCP server is not connected.
+ * Follows RFC 5424 severity levels as per MCP spec.
+ */
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+/**
+ * MCP log levels per RFC 5424
+ */
+export type McpLogLevel =
+    | 'debug'
+    | 'info'
+    | 'notice'
+    | 'warning'
+    | 'error'
+    | 'critical'
+    | 'alert'
+    | 'emergency';
+
+const LOG_LEVEL_PRIORITY: Record<McpLogLevel, number> = {
+    emergency: 0,
+    alert: 1,
+    critical: 2,
+    error: 3,
+    warning: 4,
+    notice: 5,
+    info: 6,
+    debug: 7,
+};
+
+interface LogData {
+    message: string;
+    module?: string;
+    operation?: string;
+    [key: string]: unknown;
+}
+
+/**
+ * MCP Protocol Logger
+ * 
+ * Sends structured log messages via MCP notifications/message protocol.
+ * Falls back to stderr when server not connected.
+ */
+export class McpLogger {
+    private server: McpServer | null = null;
+    private minLevel: McpLogLevel = 'info';
+
+    /**
+     * Connect to MCP server for protocol logging
+     */
+    setServer(server: McpServer): void {
+        this.server = server;
+    }
+
+    /**
+     * Set minimum log level (from logging/setLevel request)
+     */
+    setLevel(level: McpLogLevel): void {
+        if (level in LOG_LEVEL_PRIORITY) {
+            this.minLevel = level;
+        }
+    }
+
+    /**
+     * Get current minimum log level
+     */
+    getLevel(): McpLogLevel {
+        return this.minLevel;
+    }
+
+    /**
+     * Check if a level should be logged
+     */
+    private shouldLog(level: McpLogLevel): boolean {
+        return LOG_LEVEL_PRIORITY[level] <= LOG_LEVEL_PRIORITY[this.minLevel];
+    }
+
+    /**
+     * Format message for stderr fallback
+     */
+    private formatForStderr(level: McpLogLevel, loggerName: string, data: LogData): string {
+        const timestamp = new Date().toISOString();
+        const levelUpper = level.toUpperCase().padEnd(9);
+        const module = loggerName ? `[${loggerName}]` : '';
+
+        let formatted = `[${timestamp}] [${levelUpper}] ${module} ${data.message}`;
+
+        // Add extra context (filter out message and module)
+        const extras = Object.fromEntries(
+            Object.entries(data).filter(([key]) => key !== 'message' && key !== 'module')
+        );
+
+        if (Object.keys(extras).length > 0) {
+            formatted += ` ${JSON.stringify(extras)}`;
+        }
+
+        return formatted;
+    }
+
+    /**
+     * Send log message via MCP protocol or fallback to stderr
+     */
+    log(level: McpLogLevel, loggerName: string, data: LogData): void {
+        if (!this.shouldLog(level)) return;
+
+        // Send via MCP protocol if server connected
+        if (this.server) {
+            try {
+                void this.server.sendLoggingMessage({
+                    level,
+                    logger: loggerName,
+                    data,
+                });
+            } catch {
+                // Fallback to stderr if MCP send fails
+                console.error(this.formatForStderr(level, loggerName, data));
+            }
+        }
+
+        // Always also log to stderr for local debugging
+        console.error(this.formatForStderr(level, loggerName, data));
+    }
+
+    // Convenience methods
+    debug(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('debug', loggerName, { message, ...context });
+    }
+
+    info(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('info', loggerName, { message, ...context });
+    }
+
+    notice(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('notice', loggerName, { message, ...context });
+    }
+
+    warning(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('warning', loggerName, { message, ...context });
+    }
+
+    error(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('error', loggerName, { message, ...context });
+    }
+
+    critical(loggerName: string, message: string, context?: Record<string, unknown>): void {
+        this.log('critical', loggerName, { message, ...context });
+    }
+}
+
+// Singleton instance
+export const mcpLogger = new McpLogger();

package/src/utils/logger.ts

@@ -0,0 +1,98 @@
+/**
+ * Memory Journal MCP Server - Logger
+ * 
+ * Centralized logging to stderr only (stdout reserved for MCP protocol).
+ * Follows RFC 5424 severity levels.
+ */
+
+type LogLevel = 'debug' | 'info' | 'notice' | 'warning' | 'error' | 'critical';
+
+interface LogContext {
+    module?: string;
+    operation?: string;
+    entityId?: string | number;
+    [key: string]: unknown;
+}
+
+const LOG_LEVELS: Record<LogLevel, number> = {
+    debug: 7,
+    info: 6,
+    notice: 5,
+    warning: 4,
+    error: 3,
+    critical: 2,
+};
+
+class Logger {
+    private minLevel: number;
+
+    constructor(level: LogLevel = 'info') {
+        this.minLevel = LOG_LEVELS[level];
+    }
+
+    private shouldLog(level: LogLevel): boolean {
+        return LOG_LEVELS[level] <= this.minLevel;
+    }
+
+    private formatMessage(level: LogLevel, message: string, context?: LogContext): string {
+        const timestamp = new Date().toISOString();
+        const levelUpper = level.toUpperCase().padEnd(8);
+        const module = context?.module ? `[${context.module}]` : '';
+        const operation = context?.operation ? `[${context.operation}]` : '';
+
+        let formatted = `[${timestamp}] [${levelUpper}] ${module}${operation} ${message}`;
+
+        // Add context as JSON if there are additional fields
+        const extras = { ...context };
+        delete extras.module;
+        delete extras.operation;
+
+        if (Object.keys(extras).length > 0) {
+            formatted += ` ${JSON.stringify(extras)}`;
+        }
+
+        return formatted;
+    }
+
+    private log(level: LogLevel, message: string, context?: LogContext): void {
+        if (!this.shouldLog(level)) return;
+
+        const formatted = this.formatMessage(level, message, context);
+
+        // Always write to stderr (stdout is reserved for MCP protocol)
+        console.error(formatted);
+    }
+
+    debug(message: string, context?: LogContext): void {
+        this.log('debug', message, context);
+    }
+
+    info(message: string, context?: LogContext): void {
+        this.log('info', message, context);
+    }
+
+    notice(message: string, context?: LogContext): void {
+        this.log('notice', message, context);
+    }
+
+    warning(message: string, context?: LogContext): void {
+        this.log('warning', message, context);
+    }
+
+    error(message: string, context?: LogContext): void {
+        this.log('error', message, context);
+    }
+
+    critical(message: string, context?: LogContext): void {
+        this.log('critical', message, context);
+    }
+
+    setLevel(level: LogLevel): void {
+        this.minLevel = LOG_LEVELS[level];
+    }
+}
+
+// Get log level from environment
+const envLevel = (process.env['LOG_LEVEL'] ?? 'info') as LogLevel;
+
+export const logger = new Logger(envLevel);

package/src/vector/VectorSearchManager.ts

@@ -0,0 +1,277 @@
+/**
+ * Memory Journal MCP Server - Vector Search Manager
+ * 
+ * Semantic search using @xenova/transformers for embeddings
+ * and vectra for vector indexing.
+ */
+
+import { pipeline } from '@xenova/transformers';
+import { LocalIndex } from 'vectra';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import { logger } from '../utils/logger.js';
+import type { SqliteAdapter } from '../database/SqliteAdapter.js';
+import type { JournalEntry } from '../types/index.js';
+
+/** Default model for embeddings (same as Python version) */
+const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
+
+/** Embedding dimensions for all-MiniLM-L6-v2 */
+const EMBEDDING_DIMENSIONS = 384;
+
+/** Search result with similarity score */
+export interface SemanticSearchResult {
+    entryId: number;
+    score: number;
+    entry?: JournalEntry;
+}
+
+/**
+ * VectorSearchManager - Handles semantic search with local embeddings
+ */
+export class VectorSearchManager {
+    // Use a more flexible type since FeatureExtractionPipeline doesn't fully implement Pipeline
+    private embedder: ((text: string, options?: Record<string, unknown>) => Promise<unknown>) | null = null;
+    private index: LocalIndex | null = null;
+    private readonly indexPath: string;
+    private readonly modelName: string;
+    private initialized = false;
+    private initializing = false;
+
+    constructor(dbPath: string, modelName = DEFAULT_MODEL) {
+        // Store index in same directory as database
+        const dbDir = path.dirname(dbPath);
+        this.indexPath = path.join(dbDir, '.vectra_index');
+        this.modelName = modelName;
+    }
+
+    /**
+     * Check if vector search is initialized
+     */
+    isInitialized(): boolean {
+        return this.initialized;
+    }
+
+    /**
+     * Initialize the vector search manager (lazy loading)
+     */
+    async initialize(): Promise<void> {
+        if (this.initialized || this.initializing) return;
+
+        this.initializing = true;
+
+        try {
+            logger.info('Initializing vector search...', { module: 'VectorSearch' });
+
+            // Load embedding model (downloads on first use, ~23MB)
+            logger.info(`Loading embedding model: ${this.modelName}`, { module: 'VectorSearch' });
+            this.embedder = await pipeline('feature-extraction', this.modelName, {
+                quantized: true, // Use quantized model for faster inference
+            });
+            logger.info('Embedding model loaded', { module: 'VectorSearch' });
+
+            // Create or load vectra index
+            if (!fs.existsSync(this.indexPath)) {
+                fs.mkdirSync(this.indexPath, { recursive: true });
+            }
+
+            this.index = new LocalIndex(this.indexPath);
+
+            // Check if index exists
+            if (!await this.index.isIndexCreated()) {
+                await this.index.createIndex();
+                logger.info('Created new vector index', { module: 'VectorSearch' });
+            } else {
+                logger.info('Loaded existing vector index', { module: 'VectorSearch' });
+            }
+
+            this.initialized = true;
+            this.initializing = false;
+            logger.info('Vector search initialized successfully', { module: 'VectorSearch' });
+        } catch (error) {
+            this.initializing = false;
+            logger.error('Failed to initialize vector search', {
+                module: 'VectorSearch',
+                error: error instanceof Error ? error.message : String(error)
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Generate embedding for text
+     */
+    async generateEmbedding(text: string): Promise<number[]> {
+        if (!this.embedder) {
+            throw new Error('Vector search not initialized');
+        }
+
+        // Generate embedding using feature-extraction pipeline
+        // The pipeline returns a Tensor with a data property containing the embeddings
+        const output = await this.embedder(text, {
+            pooling: 'mean',
+            normalize: true,
+        }) as { data: ArrayLike<number> };
+
+        // Convert to number array
+        const embedding = Array.from(output.data);
+        return embedding;
+    }
+
+    /**
+     * Add an entry to the vector index (upsert - replaces if exists)
+     */
+    async addEntry(entryId: number, content: string): Promise<boolean> {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+
+        if (!this.index) {
+            return false;
+        }
+
+        try {
+            // Delete existing item first to support upsert behavior
+            try {
+                await this.index.deleteItem(String(entryId));
+            } catch {
+                // Item may not exist, ignore
+            }
+
+            // Generate embedding
+            const embedding = await this.generateEmbedding(content);
+
+            // Add to vectra index with entry ID as metadata
+            await this.index.insertItem({
+                id: String(entryId),
+                vector: embedding,
+                metadata: { entryId, contentPreview: content.slice(0, 100) }
+            });
+
+            logger.debug('Added entry to vector index', {
+                module: 'VectorSearch',
+                entityId: entryId
+            });
+
+            return true;
+        } catch (error) {
+            logger.error('Failed to add entry to vector index', {
+                module: 'VectorSearch',
+                entityId: entryId,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            return false;
+        }
+    }
+
+    /**
+     * Perform semantic search
+     */
+    async search(
+        query: string,
+        limit = 10,
+        similarityThreshold = 0.3
+    ): Promise<SemanticSearchResult[]> {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+
+        if (!this.index) {
+            return [];
+        }
+
+        try {
+            // Generate query embedding
+            const queryEmbedding = await this.generateEmbedding(query);
+
+            // Search vectra index
+            const results = await this.index.queryItems(queryEmbedding, limit * 2);
+
+            // Filter by threshold and map to our format
+            const filteredResults: SemanticSearchResult[] = results
+                .filter(r => r.score >= similarityThreshold)
+                .slice(0, limit)
+                .map(r => ({
+                    entryId: (r.item.metadata as { entryId: number }).entryId,
+                    score: r.score
+                }));
+
+            return filteredResults;
+        } catch (error) {
+            logger.error('Semantic search failed', {
+                module: 'VectorSearch',
+                error: error instanceof Error ? error.message : String(error)
+            });
+            return [];
+        }
+    }
+
+    /**
+     * Remove an entry from the vector index
+     */
+    async removeEntry(entryId: number): Promise<boolean> {
+        if (!this.index) return false;
+
+        try {
+            await this.index.deleteItem(String(entryId));
+            return true;
+        } catch {
+            // Item might not exist in index
+            return false;
+        }
+    }
+
+    /**
+     * Rebuild index from database entries
+     */
+    async rebuildIndex(db: SqliteAdapter): Promise<number> {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+
+        if (!this.index) {
+            return 0;
+        }
+
+        logger.info('Rebuilding vector index from database...', { module: 'VectorSearch' });
+
+        // Get all entries
+        const entries = db.getRecentEntries(10000); // Get up to 10k entries
+
+        let indexed = 0;
+        for (const entry of entries) {
+            // Delete existing item first to avoid "already exists" error
+            try {
+                await this.index.deleteItem(String(entry.id));
+            } catch {
+                // Item may not exist, ignore
+            }
+
+            const success = await this.addEntry(entry.id, entry.content);
+            if (success) indexed++;
+        }
+
+        logger.info(`Rebuilt vector index with ${String(indexed)} entries`, { module: 'VectorSearch' });
+        return indexed;
+    }
+
+    /**
+     * Get index statistics
+     */
+    async getStats(): Promise<{ itemCount: number; modelName: string; dimensions: number }> {
+        if (!this.index) {
+            return { itemCount: 0, modelName: this.modelName, dimensions: EMBEDDING_DIMENSIONS };
+        }
+
+        try {
+            const items = await this.index.listItems();
+            return {
+                itemCount: items.length,
+                modelName: this.modelName,
+                dimensions: EMBEDDING_DIMENSIONS
+            };
+        } catch {
+            return { itemCount: 0, modelName: this.modelName, dimensions: EMBEDDING_DIMENSIONS };
+        }
+    }
+}