morpheus-cli 0.9.5 → 0.9.6

package/dist/runtime/link-repository.js

@@ -0,0 +1,301 @@
+import Database from 'better-sqlite3';
+import fs from 'fs-extra';
+import path from 'path';
+import { homedir } from 'os';
+import { randomUUID } from 'crypto';
+import loadVecExtension from './memory/sqlite-vec.js';
+import { DisplayManager } from './display.js';
+// ─── Repository ──────────────────────────────────────────────────────────────
+const EMBEDDING_DIM = 384;
+export class LinkRepository {
+    static instance = null;
+    db = null;
+    dbPath;
+    display = DisplayManager.getInstance();
+    constructor(dbPath) {
+        this.dbPath = dbPath || path.join(homedir(), '.morpheus', 'memory', 'link.db');
+    }
+    static getInstance(dbPath) {
+        if (!LinkRepository.instance) {
+            LinkRepository.instance = new LinkRepository(dbPath);
+        }
+        return LinkRepository.instance;
+    }
+    static resetInstance() {
+        if (LinkRepository.instance?.db) {
+            LinkRepository.instance.db.close();
+        }
+        LinkRepository.instance = null;
+    }
+    initialize() {
+        fs.ensureDirSync(path.dirname(this.dbPath));
+        this.db = new Database(this.dbPath, { timeout: 5000 });
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('foreign_keys = ON');
+        loadVecExtension(this.db);
+        this.createSchema();
+    }
+    createSchema() {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // Documents table
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id TEXT PRIMARY KEY,
+        filename TEXT NOT NULL,
+        file_path TEXT NOT NULL UNIQUE,
+        file_hash TEXT NOT NULL,
+        file_size INTEGER NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        error_message TEXT,
+        chunk_count INTEGER DEFAULT 0,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+    `);
+        // Chunks table
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        document_id TEXT NOT NULL,
+        position INTEGER NOT NULL,
+        content TEXT NOT NULL,
+        char_start INTEGER NOT NULL,
+        char_end INTEGER NOT NULL,
+        created_at TEXT NOT NULL,
+        UNIQUE(document_id, position),
+        FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
+      );
+    `);
+        // Embeddings table (sqlite-vec)
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding FLOAT[${EMBEDDING_DIM}]
+      );
+    `);
+        // FTS5 virtual table for BM25 search
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        content='chunks',
+        content_rowid='rowid',
+        tokenize='porter unicode61'
+      );
+    `);
+        // Indexes
+        this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_documents_status ON documents(status);
+      CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(file_hash);
+      CREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_id);
+    `);
+        // Triggers to keep FTS in sync
+        this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+        INSERT INTO chunks_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+      END;
+    `);
+    }
+    // ─── Document CRUD ────────────────────────────────────────────────────────
+    createDocument(input) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const id = randomUUID();
+        this.db.prepare(`
+      INSERT INTO documents (id, filename, file_path, file_hash, file_size, status, error_message, chunk_count, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, 'pending', NULL, 0, ?, ?)
+    `).run(id, input.filename, input.file_path, input.file_hash, input.file_size, now, now);
+        return this.getDocument(id);
+    }
+    getDocument(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE id = ?').get(id);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    getDocumentByPath(file_path) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE file_path = ?').get(file_path);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    getDocumentByHash(hash) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE file_hash = ?').get(hash);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    listDocuments(status) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        let query = 'SELECT * FROM documents';
+        const params = [];
+        if (status) {
+            query += ' WHERE status = ?';
+            params.push(status);
+        }
+        query += ' ORDER BY updated_at DESC';
+        const rows = this.db.prepare(query).all(...params);
+        return rows.map((r) => this.deserializeDocument(r));
+    }
+    updateDocumentStatus(id, status, error_message) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        this.db.prepare(`
+      UPDATE documents SET status = ?, error_message = ?, updated_at = ? WHERE id = ?
+    `).run(status, error_message ?? null, now, id);
+        return this.getDocument(id);
+    }
+    updateDocumentChunkCount(id, chunk_count) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        this.db.prepare(`
+      UPDATE documents SET chunk_count = ?, status = 'indexed', updated_at = ? WHERE id = ?
+    `).run(chunk_count, now, id);
+    }
+    deleteDocument(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // CASCADE will delete chunks and embeddings automatically
+        const result = this.db.prepare('DELETE FROM documents WHERE id = ?').run(id);
+        return result.changes > 0;
+    }
+    deleteDocumentByPath(file_path) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const result = this.db.prepare('DELETE FROM documents WHERE file_path = ?').run(file_path);
+        return result.changes > 0;
+    }
+    // ─── Chunk CRUD ───────────────────────────────────────────────────────────
+    createChunk(input) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const id = randomUUID();
+        this.db.prepare(`
+      INSERT INTO chunks (id, document_id, position, content, char_start, char_end, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(id, input.document_id, input.position, input.content, input.char_start, input.char_end, now);
+        return this.getChunk(id);
+    }
+    createChunks(inputs) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const stmt = this.db.prepare(`
+      INSERT INTO chunks (id, document_id, position, content, char_start, char_end, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+        const insertMany = this.db.transaction((items) => {
+            for (const input of items) {
+                stmt.run(randomUUID(), input.document_id, input.position, input.content, input.char_start, input.char_end, now);
+            }
+        });
+        insertMany(inputs);
+    }
+    getChunk(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM chunks WHERE id = ?').get(id);
+        return row ? this.deserializeChunk(row) : null;
+    }
+    getChunksByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const rows = this.db.prepare('SELECT * FROM chunks WHERE document_id = ? ORDER BY position').all(document_id);
+        return rows.map((r) => this.deserializeChunk(r));
+    }
+    deleteChunksByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        this.db.prepare('DELETE FROM chunks WHERE document_id = ?').run(document_id);
+    }
+    // ─── Embeddings ───────────────────────────────────────────────────────────
+    createEmbedding(chunk_id, embedding) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const embeddingBlob = new Float32Array(embedding);
+        this.db.prepare(`
+      INSERT INTO embeddings (chunk_id, embedding) VALUES (?, ?)
+    `).run(chunk_id, embeddingBlob);
+    }
+    createEmbeddings(items) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const stmt = this.db.prepare(`
+      INSERT INTO embeddings (chunk_id, embedding) VALUES (?, ?)
+    `);
+        const insertMany = this.db.transaction((items) => {
+            for (const item of items) {
+                const embeddingBlob = new Float32Array(item.embedding);
+                stmt.run(item.chunk_id, embeddingBlob);
+            }
+        });
+        insertMany(items);
+    }
+    deleteEmbeddingsByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // Get all chunk IDs for this document
+        const chunks = this.db.prepare('SELECT id FROM chunks WHERE document_id = ?').all(document_id);
+        const chunkIds = chunks.map(c => c.id);
+        if (chunkIds.length === 0)
+            return;
+        const placeholders = chunkIds.map(() => '?').join(',');
+        this.db.prepare(`DELETE FROM embeddings WHERE chunk_id IN (${placeholders})`).run(...chunkIds);
+    }
+    // ─── Stats ─────────────────────────────────────────────────────────────────
+    getStats() {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const documents_total = this.db.prepare('SELECT COUNT(*) as cnt FROM documents').get().cnt;
+        const documents_indexed = this.db.prepare("SELECT COUNT(*) as cnt FROM documents WHERE status = 'indexed'").get().cnt;
+        const chunks_total = this.db.prepare('SELECT COUNT(*) as cnt FROM chunks').get().cnt;
+        return { documents_total, documents_indexed, chunks_total };
+    }
+    // ─── Cleanup ───────────────────────────────────────────────────────────────
+    close() {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+    }
+    // ─── Deserializers ─────────────────────────────────────────────────────────
+    deserializeDocument(row) {
+        return {
+            id: row.id,
+            filename: row.filename,
+            file_path: row.file_path,
+            file_hash: row.file_hash,
+            file_size: row.file_size,
+            status: row.status,
+            error_message: row.error_message ?? null,
+            chunk_count: row.chunk_count ?? 0,
+            created_at: row.created_at,
+            updated_at: row.updated_at,
+        };
+    }
+    deserializeChunk(row) {
+        return {
+            id: row.id,
+            document_id: row.document_id,
+            position: row.position,
+            content: row.content,
+            char_start: row.char_start,
+            char_end: row.char_end,
+            created_at: row.created_at,
+        };
+    }
+}

package/dist/runtime/link-search.js

@@ -0,0 +1,298 @@
+import { LinkRepository } from './link-repository.js';
+import { ConfigManager } from '../config/manager.js';
+import { EmbeddingService } from './memory/embedding.service.js';
+/**
+ * LinkSearch - Hybrid search for Link documents
+ *
+ * Combines vector similarity search (80% weight) with BM25 text search (20% weight)
+ * for optimal retrieval of relevant document chunks.
+ */
+export class LinkSearch {
+    static instance = null;
+    repository;
+    db = null;
+    embeddingService = null;
+    constructor() {
+        this.repository = LinkRepository.getInstance();
+    }
+    static getInstance() {
+        if (!LinkSearch.instance) {
+            LinkSearch.instance = new LinkSearch();
+        }
+        return LinkSearch.instance;
+    }
+    static resetInstance() {
+        LinkSearch.instance = null;
+    }
+    async initialize() {
+        // Get the database from the repository
+        this.db = this.repository.db;
+        // Initialize embedding service
+        this.embeddingService = await EmbeddingService.getInstance();
+    }
+    /**
+     * Perform vector similarity search using sqlite-vec.
+     */
+    vectorSearch(queryEmbedding, limit) {
+        if (!this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const embeddingBlob = new Float32Array(queryEmbedding);
+        // Query vector similarity using cosine distance
+        const rows = this.db.prepare(`
+      SELECT
+        e.chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        vec_distance_cosine(e.embedding, ?) as distance
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed'
+      ORDER BY distance ASC
+      LIMIT ?
+    `).all(embeddingBlob, limit);
+        // Convert distance to similarity score (1 - distance for cosine)
+        return rows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: 1 - row.distance,
+        }));
+    }
+    /**
+     * Perform BM25 full-text search using FTS5.
+     */
+    bm25Search(query, limit) {
+        if (!this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        // Sanitize query: remove characters that could break FTS5 syntax (like ?, *, OR, etc)
+        // keeping only letters, numbers and spaces.
+        const escapedQuery = query
+            .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+            .replace(/\s+/g, ' ')
+            .trim();
+        // Return empty results if query is empty after sanitization
+        if (!escapedQuery) {
+            return [];
+        }
+        const rows = this.db.prepare(`
+      SELECT
+        c.id as chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        bm25(chunks_fts) as bm25_score
+      FROM chunks_fts fts
+      JOIN chunks c ON c.rowid = fts.rowid
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed'
+        AND chunks_fts MATCH ?
+      ORDER BY bm25_score ASC
+      LIMIT ?
+    `).all(escapedQuery, limit);
+        // BM25 returns negative scores for better matches, negate and normalize
+        return rows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: -row.bm25_score, // Negate since BM25 returns negative for better matches
+        }));
+    }
+    /**
+     * Normalize scores to 0-1 range using min-max scaling.
+     */
+    normalizeScores(results) {
+        if (results.length === 0)
+            return results;
+        const scores = results.map(r => r.score);
+        const min = Math.min(...scores);
+        const max = Math.max(...scores);
+        const range = max - min;
+        if (range === 0) {
+            // All scores are the same
+            return results.map(r => ({ ...r, score: 1 }));
+        }
+        return results.map(r => ({
+            ...r,
+            score: (r.score - min) / range,
+        }));
+    }
+    /**
+     * Perform hybrid search combining vector and BM25 results.
+     */
+    hybridSearch(queryEmbedding, queryText, limit, threshold) {
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const vectorWeight = config.vector_weight;
+        const bm25Weight = config.bm25_weight;
+        // Get results from both methods (fetch more for better merging)
+        const fetchLimit = limit * 3;
+        const vectorResults = this.vectorSearch(queryEmbedding, fetchLimit);
+        const bm25Results = this.bm25Search(queryText, fetchLimit);
+        // Normalize scores
+        const normalizedVector = this.normalizeScores(vectorResults);
+        const normalizedBM25 = this.normalizeScores(bm25Results);
+        // Create maps for quick lookup
+        const vectorMap = new Map(normalizedVector.map(r => [r.chunk_id, r]));
+        const bm25Map = new Map(normalizedBM25.map(r => [r.chunk_id, r]));
+        // Combine all unique chunk IDs
+        const allChunkIds = new Set([...vectorMap.keys(), ...bm25Map.keys()]);
+        // Calculate combined scores
+        const combined = [];
+        for (const chunkId of allChunkIds) {
+            const vResult = vectorMap.get(chunkId);
+            const bResult = bm25Map.get(chunkId);
+            const vectorScore = vResult?.score ?? 0;
+            const bm25Score = bResult?.score ?? 0;
+            // Weighted combination
+            const combinedScore = (vectorScore * vectorWeight) + (bm25Score * bm25Weight);
+            // Get the data from whichever result has it
+            const data = vResult || bResult;
+            if (!data)
+                continue;
+            combined.push({
+                chunk_id: chunkId,
+                document_id: data.document_id,
+                filename: data.filename,
+                position: data.position,
+                content: data.content,
+                score: combinedScore,
+                vector_score: vectorScore,
+                bm25_score: bm25Score,
+            });
+        }
+        // Sort by combined score and filter by threshold
+        const filtered = combined
+            .filter(r => r.score >= threshold)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+        return filtered;
+    }
+    /**
+     * Search with a text query (generates embedding internally).
+     */
+    async search(queryText, limit, threshold) {
+        if (!this.embeddingService) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxResults = limit ?? config.max_results;
+        const minThreshold = threshold ?? config.score_threshold;
+        // Generate embedding for the query
+        const queryEmbedding = await this.embeddingService.generate(queryText);
+        return this.hybridSearch(queryEmbedding, queryText, maxResults, minThreshold);
+    }
+    /**
+     * Search within a specific document by document_id.
+     * Runs vector + BM25 search filtered to chunks belonging to that document.
+     */
+    async searchInDocument(queryText, documentId, limit, threshold) {
+        if (!this.embeddingService || !this.db) {
+            throw new Error('LinkSearch not initialized');
+        }
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxResults = limit ?? config.max_results;
+        const minThreshold = threshold ?? config.score_threshold;
+        const vectorWeight = config.vector_weight;
+        const bm25Weight = config.bm25_weight;
+        const fetchLimit = maxResults * 3;
+        // Generate embedding for the query
+        const queryEmbedding = await this.embeddingService.generate(queryText);
+        const embeddingBlob = new Float32Array(queryEmbedding);
+        // Vector search filtered by document
+        const vectorRows = this.db.prepare(`
+      SELECT
+        e.chunk_id,
+        c.document_id,
+        d.filename,
+        c.position,
+        c.content,
+        vec_distance_cosine(e.embedding, ?) as distance
+      FROM embeddings e
+      JOIN chunks c ON e.chunk_id = c.id
+      JOIN documents d ON c.document_id = d.id
+      WHERE d.status = 'indexed' AND c.document_id = ?
+      ORDER BY distance ASC
+      LIMIT ?
+    `).all(embeddingBlob, documentId, fetchLimit);
+        const vectorResults = vectorRows.map(row => ({
+            chunk_id: row.chunk_id,
+            document_id: row.document_id,
+            filename: row.filename,
+            position: row.position,
+            content: row.content,
+            score: 1 - row.distance,
+        }));
+        // BM25 search filtered by document
+        const escapedQuery = queryText
+            .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+            .replace(/\s+/g, ' ')
+            .trim();
+        let bm25Results = [];
+        if (escapedQuery) {
+            const bm25Rows = this.db.prepare(`
+        SELECT
+          c.id as chunk_id,
+          c.document_id,
+          d.filename,
+          c.position,
+          c.content,
+          bm25(chunks_fts) as bm25_score
+        FROM chunks_fts fts
+        JOIN chunks c ON c.rowid = fts.rowid
+        JOIN documents d ON c.document_id = d.id
+        WHERE d.status = 'indexed'
+          AND c.document_id = ?
+          AND chunks_fts MATCH ?
+        ORDER BY bm25_score ASC
+        LIMIT ?
+      `).all(documentId, escapedQuery, fetchLimit);
+            bm25Results = bm25Rows.map(row => ({
+                chunk_id: row.chunk_id,
+                document_id: row.document_id,
+                filename: row.filename,
+                position: row.position,
+                content: row.content,
+                score: -row.bm25_score,
+            }));
+        }
+        // Normalize and combine
+        const normalizedVector = this.normalizeScores(vectorResults);
+        const normalizedBM25 = this.normalizeScores(bm25Results);
+        const vectorMap = new Map(normalizedVector.map(r => [r.chunk_id, r]));
+        const bm25Map = new Map(normalizedBM25.map(r => [r.chunk_id, r]));
+        const allChunkIds = new Set([...vectorMap.keys(), ...bm25Map.keys()]);
+        const combined = [];
+        for (const chunkId of allChunkIds) {
+            const vResult = vectorMap.get(chunkId);
+            const bResult = bm25Map.get(chunkId);
+            const vectorScore = vResult?.score ?? 0;
+            const bm25Score = bResult?.score ?? 0;
+            const data = vResult || bResult;
+            if (!data)
+                continue;
+            combined.push({
+                chunk_id: chunkId,
+                document_id: data.document_id,
+                filename: data.filename,
+                position: data.position,
+                content: data.content,
+                score: (vectorScore * vectorWeight) + (bm25Score * bm25Weight),
+                vector_score: vectorScore,
+                bm25_score: bm25Score,
+            });
+        }
+        return combined
+            .filter(r => r.score >= minThreshold)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, maxResults);
+    }
+}