morpheus-cli 0.9.5 → 0.9.7

package/dist/runtime/link-search.js

@@ -0,0 +1,298 @@
+import { LinkRepository } from './link-repository.js';
+import { ConfigManager } from '../config/manager.js';
+import { EmbeddingService } from './memory/embedding.service.js';
+/**
+ * LinkSearch - Hybrid search for Link documents
+ *
+ * Combines vector similarity search (80% weight) with BM25 text search (20% weight)
+ * for optimal retrieval of relevant document chunks.
+ */
+export class LinkSearch {
+    static instance = null;
+    repository;
+    db = null;
+    embeddingService = null;
+    constructor() {
+        this.repository = LinkRepository.getInstance();
+    }
+    static getInstance() {
+        if (!LinkSearch.instance) {
+            LinkSearch.instance = new LinkSearch();
+        }
+        return LinkSearch.instance;
+    }
+    static resetInstance() {
+        LinkSearch.instance = null;
+    }
+    async initialize() {
+        // Get the database from the repository
+        this.db = this.repository.db;
+        // Initialize embedding service
+        this.embeddingService = await EmbeddingService.getInstance();
+    }
+    /**
+     * Perform vector similarity search using sqlite-vec.
+     */
+    vectorSearch(queryEmbedding, limit) {
+        if (!this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const embeddingBlob = new Float32Array(queryEmbedding);
+        // Query vector similarity using cosine distance
+        const rows = this.db.prepare(`
+      SELECT
+        e.chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        vec_distance_cosine(e.embedding, ?) as distance
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed'
+      ORDER BY distance ASC
+      LIMIT ?
+    `).all(embeddingBlob, limit);
+        // Convert distance to similarity score (1 - distance for cosine)
+        return rows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: 1 - row.distance,
+        }));
+    }
+    /**
+     * Perform BM25 full-text search using FTS5.
+     */
+    bm25Search(query, limit) {
+        if (!this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        // Sanitize query: remove characters that could break FTS5 syntax (like ?, *, OR, etc)
+        // keeping only letters, numbers and spaces.
+        const escapedQuery = query
+            .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+            .replace(/\s+/g, ' ')
+            .trim();
+        // Return empty results if query is empty after sanitization
+        if (!escapedQuery) {
+            return [];
+        }
+        const rows = this.db.prepare(`
+      SELECT
+        c.id as chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        bm25(chunks_fts) as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON c.rowid = fts.rowid
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed'
+        AND chunks_fts MATCH ?
+      ORDER BY bm25_score ASC
+      LIMIT ?
+    `).all(escapedQuery, limit);
+        // BM25 returns negative scores for better matches, negate and normalize
+        return rows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: -row.bm25_score, // Negate since BM25 returns negative for better matches
+        }));
+    }
+    /**
+     * Normalize scores to 0-1 range using min-max scaling.
+     */
+    normalizeScores(results) {
+        if (results.length === 0)
+            return results;
+        const scores = results.map(r => r.score);
+        const min = Math.min(...scores);
+        const max = Math.max(...scores);
+        const range = max - min;
+        if (range === 0) {
+            // All scores are the same
+            return results.map(r => ({ ...r, score: 1 }));
+        }
+        return results.map(r => ({
+            ...r,
+            score: (r.score - min) / range,
+        }));
+    }
+    /**
+     * Perform hybrid search combining vector and BM25 results.
+     */
+    hybridSearch(queryEmbedding, queryText, limit, threshold) {
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const vectorWeight = config.vector_weight;
+        const bm25Weight = config.bm25_weight;
+        // Get results from both methods (fetch more for better merging)
+        const fetchLimit = limit * 3;
+        const vectorResults = this.vectorSearch(queryEmbedding, fetchLimit);
+        const bm25Results = this.bm25Search(queryText, fetchLimit);
+        // Normalize scores
+        const normalizedVector = this.normalizeScores(vectorResults);
+        const normalizedBM25 = this.normalizeScores(bm25Results);
+        // Create maps for quick lookup
+        const vectorMap = new Map(normalizedVector.map(r => [r.chunk_id, r]));
+        const bm25Map = new Map(normalizedBM25.map(r => [r.chunk_id, r]));
+        // Combine all unique chunk IDs
+        const allChunkIds = new Set([...vectorMap.keys(), ...bm25Map.keys()]);
+        // Calculate combined scores
+        const combined = [];
+        for (const chunkId of allChunkIds) {
+            const vResult = vectorMap.get(chunkId);
+            const bResult = bm25Map.get(chunkId);
+            const vectorScore = vResult?.score ?? 0;
+            const bm25Score = bResult?.score ?? 0;
+            // Weighted combination
+            const combinedScore = (vectorScore * vectorWeight) + (bm25Score * bm25Weight);
+            // Get the data from whichever result has it
+            const data = vResult || bResult;
+            if (!data)
+                continue;
+            combined.push({
+                chunk_id: chunkId,
+                document_id: data.document_id,
+                filename: data.filename,
+                position: data.position,
+                content: data.content,
+                score: combinedScore,
+                vector_score: vectorScore,
+                bm25_score: bm25Score,
+            });
+        }
+        // Sort by combined score and filter by threshold
+        const filtered = combined
+            .filter(r => r.score >= threshold)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+        return filtered;
+    }
+    /**
+     * Search with a text query (generates embedding internally).
+     */
+    async search(queryText, limit, threshold) {
+        if (!this.embeddingService) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxResults = limit ?? config.max_results;
+        const minThreshold = threshold ?? config.score_threshold;
+        // Generate embedding for the query
+        const queryEmbedding = await this.embeddingService.generate(queryText);
+        return this.hybridSearch(queryEmbedding, queryText, maxResults, minThreshold);
+    }
+    /**
+     * Search within a specific document by document_id.
+     * Runs vector + BM25 search filtered to chunks belonging to that document.
+     */
+    async searchInDocument(queryText, documentId, limit, threshold) {
+        if (!this.embeddingService || !this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxResults = limit ?? config.max_results;
+        const minThreshold = threshold ?? config.score_threshold;
+        const vectorWeight = config.vector_weight;
+        const bm25Weight = config.bm25_weight;
+        const fetchLimit = maxResults * 3;
+        // Generate embedding for the query
+        const queryEmbedding = await this.embeddingService.generate(queryText);
+        const embeddingBlob = new Float32Array(queryEmbedding);
+        // Vector search filtered by document
+        const vectorRows = this.db.prepare(`
+      SELECT
+        e.chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        vec_distance_cosine(e.embedding, ?) as distance
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed' AND c.document_id = ?
+      ORDER BY distance ASC
+      LIMIT ?
+    `).all(embeddingBlob, documentId, fetchLimit);
+        const vectorResults = vectorRows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: 1 - row.distance,
+        }));
+        // BM25 search filtered by document
+        const escapedQuery = queryText
+            .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+            .replace(/\s+/g, ' ')
+            .trim();
+        let bm25Results = [];
+        if (escapedQuery) {
+            const bm25Rows = this.db.prepare(`
+        SELECT
+          c.id as chunk_id,
+          c.document_id,
+          d.filename,
+          c.position,
+          c.content,
+          bm25(chunks_fts) as bm25_score
+        FROM chunks_fts fts
+        JOIN chunks c ON c.rowid = fts.rowid
+        JOIN documents d ON c.document_id = d.id
+        WHERE d.status = 'indexed'
+          AND c.document_id = ?
+          AND chunks_fts MATCH ?
+        ORDER BY bm25_score ASC
+        LIMIT ?
+      `).all(documentId, escapedQuery, fetchLimit);
+            bm25Results = bm25Rows.map(row => ({
+                chunk_id: row.chunk_id,
+                document_id: row.document_id,
+                filename: row.filename,
+                position: row.position,
+                content: row.content,
+                score: -row.bm25_score,
+            }));
+        }
+        // Normalize and combine
+        const normalizedVector = this.normalizeScores(vectorResults);
+        const normalizedBM25 = this.normalizeScores(bm25Results);
+        const vectorMap = new Map(normalizedVector.map(r => [r.chunk_id, r]));
+        const bm25Map = new Map(normalizedBM25.map(r => [r.chunk_id, r]));
+        const allChunkIds = new Set([...vectorMap.keys(), ...bm25Map.keys()]);
+        const combined = [];
+        for (const chunkId of allChunkIds) {
+            const vResult = vectorMap.get(chunkId);
+            const bResult = bm25Map.get(chunkId);
+            const vectorScore = vResult?.score ?? 0;
+            const bm25Score = bResult?.score ?? 0;
+            const data = vResult || bResult;
+            if (!data)
+                continue;
+            combined.push({
+                chunk_id: chunkId,
+                document_id: data.document_id,
+                filename: data.filename,
+                position: data.position,
+                content: data.content,
+                score: (vectorScore * vectorWeight) + (bm25Score * bm25Weight),
+                vector_score: vectorScore,
+                bm25_score: bm25Score,
+            });
+        }
+        return combined
+            .filter(r => r.score >= minThreshold)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, maxResults);
+    }
+}

package/dist/runtime/link-worker.js

@@ -0,0 +1,284 @@
+import { homedir } from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import { LinkRepository } from './link-repository.js';
+import { LinkSearch } from './link-search.js';
+import { hashFile, processDocument, isSupportedFormat } from './link-chunker.js';
+import { EmbeddingService } from './memory/embedding.service.js';
+import { ConfigManager } from '../config/manager.js';
+import { DisplayManager } from './display.js';
+/**
+ * LinkWorker - Background worker for document indexing
+ *
+ * Scans ~/.morpheus/docs folder, processes new/changed documents,
+ * generates embeddings, and removes deleted documents from the index.
+ */
+export class LinkWorker {
+    static instance = null;
+    repository;
+    search;
+    embeddingService = null;
+    intervalId = null;
+    isRunning = false;
+    display = DisplayManager.getInstance();
+    docsPath;
+    constructor() {
+        this.repository = LinkRepository.getInstance();
+        this.search = LinkSearch.getInstance();
+        this.docsPath = path.join(homedir(), '.morpheus', 'docs');
+    }
+    static getInstance() {
+        if (!LinkWorker.instance) {
+            LinkWorker.instance = new LinkWorker();
+        }
+        return LinkWorker.instance;
+    }
+    static setInstance(instance) {
+        LinkWorker.instance = instance;
+    }
+    static resetInstance() {
+        if (LinkWorker.instance) {
+            LinkWorker.instance.stop();
+        }
+        LinkWorker.instance = null;
+    }
+    /**
+     * Start the background worker with interval-based scanning.
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const intervalMs = config.scan_interval_ms;
+        this.isRunning = true;
+        this.display.log('LinkWorker started', { source: 'Link' });
+        // Run initial scan immediately
+        this.tick().catch(err => {
+            this.display.log(`LinkWorker initial scan failed: ${err.message}`, { source: 'Link', level: 'error' });
+        });
+        // Schedule periodic scans
+        this.intervalId = setInterval(() => {
+            this.tick().catch(err => {
+                this.display.log(`LinkWorker tick failed: ${err.message}`, { source: 'Link', level: 'error' });
+            });
+        }, intervalMs);
+    }
+    /**
+     * Stop the background worker.
+     */
+    stop() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        this.isRunning = false;
+        this.display.log('LinkWorker stopped', { source: 'Link' });
+    }
+    /**
+     * Update the scan interval (hot-reload).
+     */
+    updateInterval(intervalMs) {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = setInterval(() => {
+                this.tick().catch(err => {
+                    this.display.log(`LinkWorker tick failed: ${err.message}`, { source: 'Link', level: 'error' });
+                });
+            }, intervalMs);
+            this.display.log(`LinkWorker interval updated to ${intervalMs}ms`, { source: 'Link' });
+        }
+    }
+    /**
+     * Perform a single scan cycle.
+     */
+    async tick() {
+        // Ensure embedding service is initialized
+        if (!this.embeddingService) {
+            this.embeddingService = await EmbeddingService.getInstance();
+        }
+        // Ensure docs folder exists
+        await fs.ensureDir(this.docsPath);
+        const stats = {
+            indexed: 0,
+            removed: 0,
+            errors: 0,
+        };
+        try {
+            // Scan for new/changed documents
+            const files = await this.scanFolder();
+            this.display.log(`LinkWorker found ${files.length} files`, { source: 'Link', level: 'debug' });
+            // Process each file
+            for (const filePath of files) {
+                try {
+                    const result = await this.processDocument(filePath);
+                    if (result === 'indexed') {
+                        stats.indexed++;
+                    }
+                    else if (result === 'error') {
+                        stats.errors++;
+                    }
+                }
+                catch (err) {
+                    this.display.log(`Failed to process ${filePath}: ${err.message}`, { source: 'Link', level: 'error' });
+                    stats.errors++;
+                }
+            }
+            // Remove deleted documents
+            stats.removed = await this.removeDeletedDocuments(files);
+            if (stats.indexed > 0 || stats.removed > 0) {
+                this.display.log(`LinkWorker: indexed ${stats.indexed}, removed ${stats.removed}, errors ${stats.errors}`, { source: 'Link', level: 'info' });
+            }
+        }
+        catch (err) {
+            this.display.log(`LinkWorker tick error: ${err.message}`, { source: 'Link', level: 'error' });
+            stats.errors++;
+        }
+        return stats;
+    }
+    /**
+     * Scan the docs folder for supported files.
+     */
+    async scanFolder() {
+        const files = [];
+        const entries = await fs.readdir(this.docsPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isFile()) {
+                const filePath = path.join(this.docsPath, entry.name);
+                if (isSupportedFormat(filePath)) {
+                    files.push(filePath);
+                }
+            }
+        }
+        return files;
+    }
+    /**
+     * Process a single document: check hash, parse, chunk, embed.
+     */
+    async processDocument(filePath) {
+        const existingDoc = this.repository.getDocumentByPath(filePath);
+        // Calculate file hash
+        let fileHash;
+        try {
+            fileHash = await hashFile(filePath);
+        }
+        catch (err) {
+            // File might not be readable
+            if (existingDoc) {
+                this.repository.updateDocumentStatus(existingDoc.id, 'error', `Failed to read file: ${err.message}`);
+            }
+            return 'error';
+        }
+        // Check if document already indexed with same hash
+        if (existingDoc && existingDoc.file_hash === fileHash && existingDoc.status === 'indexed') {
+            return 'skipped';
+        }
+        // Get file stats
+        const stats = await fs.stat(filePath);
+        const fileSize = stats.size;
+        // Check max file size
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxSizeBytes = config.max_file_size_mb * 1024 * 1024;
+        if (fileSize > maxSizeBytes) {
+            if (existingDoc) {
+                this.repository.updateDocumentStatus(existingDoc.id, 'error', `File exceeds max size of ${config.max_file_size_mb}MB`);
+            }
+            return 'error';
+        }
+        // Create or update document record
+        const filename = path.basename(filePath);
+        let document;
+        if (existingDoc) {
+            // Update existing document - delete old chunks first
+            this.repository.deleteChunksByDocument(existingDoc.id);
+            this.repository.deleteEmbeddingsByDocument(existingDoc.id);
+            this.repository.updateDocumentStatus(existingDoc.id, 'indexing');
+            document = existingDoc;
+        }
+        else {
+            // Create new document
+            document = this.repository.createDocument({
+                filename,
+                file_path: filePath,
+                file_hash: fileHash,
+                file_size: fileSize,
+            });
+        }
+        try {
+            // Index the document
+            await this.indexDocument(document.id, filePath, fileHash);
+            // Update status to indexed
+            const chunks = this.repository.getChunksByDocument(document.id);
+            this.repository.updateDocumentChunkCount(document.id, chunks.length);
+            this.display.log(`Indexed document: ${filename} (${chunks.length} chunks)`, { source: 'Link', level: 'debug' });
+            return 'indexed';
+        }
+        catch (err) {
+            this.repository.updateDocumentStatus(document.id, 'error', err.message);
+            return 'error';
+        }
+    }
+    /**
+     * Index a document: parse, chunk, generate embeddings.
+     */
+    async indexDocument(documentId, filePath, fileHash) {
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const chunkSize = config.chunk_size;
+        // Parse and chunk the document
+        const processed = await processDocument(filePath, chunkSize);
+        // Verify hash matches (file might have changed during processing)
+        if (processed.hash !== fileHash) {
+            throw new Error('File changed during processing - hash mismatch');
+        }
+        // Create chunks in database
+        const chunkInputs = processed.chunks.map(chunk => ({
+            document_id: documentId,
+            position: chunk.position,
+            content: chunk.content,
+            char_start: chunk.char_start,
+            char_end: chunk.char_end,
+        }));
+        this.repository.createChunks(chunkInputs);
+        // Get the created chunks with IDs
+        const chunks = this.repository.getChunksByDocument(documentId);
+        // Generate embeddings for each chunk
+        await this.generateEmbeddings(chunks);
+    }
+    /**
+     * Generate embeddings for chunks using Sati's EmbeddingService.
+     */
+    async generateEmbeddings(chunks) {
+        if (!this.embeddingService) {
+            this.embeddingService = await EmbeddingService.getInstance();
+        }
+        const embeddings = [];
+        // Process in batches to avoid memory issues
+        const batchSize = 50;
+        for (let i = 0; i < chunks.length; i += batchSize) {
+            const batch = chunks.slice(i, i + batchSize);
+            const batchEmbeddings = await Promise.all(batch.map(async (chunk) => {
+                const embedding = await this.embeddingService.generate(chunk.content);
+                return { chunk_id: chunk.id, embedding };
+            }));
+            embeddings.push(...batchEmbeddings);
+        }
+        // Store embeddings in database
+        this.repository.createEmbeddings(embeddings);
+    }
+    /**
+     * Remove documents that no longer exist in the docs folder.
+     */
+    async removeDeletedDocuments(existingFiles) {
+        const existingPaths = new Set(existingFiles);
+        const documents = this.repository.listDocuments();
+        let removed = 0;
+        for (const doc of documents) {
+            if (!existingPaths.has(doc.file_path)) {
+                // Document file no longer exists - remove from index
+                this.repository.deleteDocument(doc.id);
+                removed++;
+                this.display.log(`Removed deleted document: ${doc.filename}`, { source: 'Link', level: 'debug' });
+            }
+        }
+        return removed;
+    }
+}