morpheus-cli 0.9.5 → 0.9.7

package/dist/runtime/link-chunker.js

@@ -0,0 +1,214 @@
+import { createHash } from 'crypto';
+import fs from 'fs-extra';
+import path from 'path';
+import { PDFParse } from 'pdf-parse';
+import mammoth from 'mammoth';
+// ─── Hashing ─────────────────────────────────────────────────────────────────
+/**
+ * Calculate SHA-256 hash of file content.
+ */
+export function hashDocument(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Calculate SHA-256 hash of a file by path.
+ */
+export async function hashFile(filePath) {
+    const content = await fs.readFile(filePath);
+    return hashDocument(content);
+}
+/**
+ * Split text into chunks respecting sentence boundaries.
+ * @param text - The text to chunk
+ * @param chunkSize - Target size in characters (default: 500)
+ * @param minChunkSize - Minimum chunk size to avoid tiny chunks (default: 100)
+ */
+export function chunkText(text, chunkSize = 500, minChunkSize = 100) {
+    const chunks = [];
+    let position = 0;
+    let charPos = 0;
+    // Split by paragraphs first
+    const paragraphs = text.split(/\n\n+/);
+    let currentChunk = '';
+    let chunkStart = 0;
+    for (const paragraph of paragraphs) {
+        // If adding this paragraph exceeds chunk size
+        if (currentChunk.length + paragraph.length + 2 > chunkSize && currentChunk.length >= minChunkSize) {
+            // Save current chunk
+            chunks.push({
+                content: currentChunk.trim(),
+                position: position++,
+                char_start: chunkStart,
+                char_end: chunkStart + currentChunk.length,
+            });
+            currentChunk = paragraph;
+            chunkStart = charPos;
+        }
+        else if (paragraph.length > chunkSize) {
+            // Paragraph is too long, split by sentences
+            if (currentChunk.length > 0) {
+                // Save current chunk first
+                chunks.push({
+                    content: currentChunk.trim(),
+                    position: position++,
+                    char_start: chunkStart,
+                    char_end: chunkStart + currentChunk.length,
+                });
+                currentChunk = '';
+            }
+            const sentences = splitBySentences(paragraph);
+            let sentenceChunk = '';
+            let sentenceStart = charPos;
+            for (const sentence of sentences) {
+                if (sentenceChunk.length + sentence.length + 1 > chunkSize && sentenceChunk.length >= minChunkSize) {
+                    chunks.push({
+                        content: sentenceChunk.trim(),
+                        position: position++,
+                        char_start: sentenceStart,
+                        char_end: sentenceStart + sentenceChunk.length,
+                    });
+                    sentenceChunk = sentence;
+                    sentenceStart = charPos + (paragraph.indexOf(sentence) > 0 ? paragraph.indexOf(sentence) : 0);
+                }
+                else {
+                    sentenceChunk += (sentenceChunk ? ' ' : '') + sentence;
+                }
+            }
+            if (sentenceChunk.trim()) {
+                currentChunk = sentenceChunk;
+                chunkStart = sentenceStart;
+            }
+        }
+        else {
+            // Add paragraph to current chunk
+            currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
+            if (!currentChunk) {
+                chunkStart = charPos;
+            }
+        }
+        charPos += paragraph.length + 2; // +2 for paragraph separator
+    }
+    // Don't forget the last chunk
+    if (currentChunk.trim()) {
+        chunks.push({
+            content: currentChunk.trim(),
+            position,
+            char_start: chunkStart,
+            char_end: chunkStart + currentChunk.length,
+        });
+    }
+    return chunks;
+}
+/**
+ * Split text by sentences using common sentence delimiters.
+ */
+function splitBySentences(text) {
+    // Match sentences ending with . ! ? followed by space or end of string
+    const sentences = text.match(/[^.!?]*[.!?]+(?:\s+|$)/g) || [text];
+    return sentences.map(s => s.trim()).filter(Boolean);
+}
+/**
+ * Parse PDF file and extract text.
+ */
+export async function parsePDF(filePath) {
+    const dataBuffer = await fs.readFile(filePath);
+    const parser = new PDFParse({ data: dataBuffer });
+    const textResult = await parser.getText();
+    const text = textResult.text || '';
+    const infoResult = await parser.getInfo();
+    return {
+        text,
+        metadata: {
+            pageCount: infoResult.total,
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse DOCX file and extract text.
+ */
+export async function parseDOCX(filePath) {
+    const result = await mammoth.extractRawText({ path: filePath });
+    const text = result.value;
+    return {
+        text,
+        metadata: {
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse plain text file.
+ */
+export async function parseTXT(filePath) {
+    const text = await fs.readFile(filePath, 'utf-8');
+    return {
+        text,
+        metadata: {
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse Markdown file (treated as plain text for chunking).
+ */
+export async function parseMD(filePath) {
+    return parseTXT(filePath);
+}
+// ─── Supported Formats ───────────────────────────────────────────────────────
+const SUPPORTED_EXTENSIONS = ['.pdf', '.txt', '.md', '.docx'];
+/**
+ * Check if a file extension is supported.
+ */
+export function isSupportedFormat(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return SUPPORTED_EXTENSIONS.includes(ext);
+}
+/**
+ * Get the content type based on file extension.
+ */
+export function getContentType(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const contentTypes = {
+        '.pdf': 'application/pdf',
+        '.txt': 'text/plain',
+        '.md': 'text/markdown',
+        '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    };
+    return contentTypes[ext] || 'application/octet-stream';
+}
+/**
+ * Parse a document based on its file extension.
+ */
+export async function parseDocument(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    switch (ext) {
+        case '.pdf':
+            return parsePDF(filePath);
+        case '.docx':
+            return parseDOCX(filePath);
+        case '.txt':
+            return parseTXT(filePath);
+        case '.md':
+            return parseMD(filePath);
+        default:
+            throw new Error(`Unsupported file format: ${ext}`);
+    }
+}
+/**
+ * Process a document: parse, chunk, and return chunks with metadata.
+ */
+export async function processDocument(filePath, chunkSize = 500) {
+    // Parse document
+    const parsed = await parseDocument(filePath);
+    // Calculate hash from raw file bytes (must match hashFile used by the caller)
+    const hash = await hashFile(filePath);
+    // Chunk text
+    const chunks = chunkText(parsed.text, chunkSize);
+    return {
+        text: parsed.text,
+        chunks,
+        hash,
+        metadata: parsed.metadata,
+    };
+}

package/dist/runtime/link-repository.js

@@ -0,0 +1,301 @@
+import Database from 'better-sqlite3';
+import fs from 'fs-extra';
+import path from 'path';
+import { homedir } from 'os';
+import { randomUUID } from 'crypto';
+import loadVecExtension from './memory/sqlite-vec.js';
+import { DisplayManager } from './display.js';
+// ─── Repository ──────────────────────────────────────────────────────────────
+const EMBEDDING_DIM = 384;
+export class LinkRepository {
+    static instance = null;
+    db = null;
+    dbPath;
+    display = DisplayManager.getInstance();
+    constructor(dbPath) {
+        this.dbPath = dbPath || path.join(homedir(), '.morpheus', 'memory', 'link.db');
+    }
+    static getInstance(dbPath) {
+        if (!LinkRepository.instance) {
+            LinkRepository.instance = new LinkRepository(dbPath);
+        }
+        return LinkRepository.instance;
+    }
+    static resetInstance() {
+        if (LinkRepository.instance?.db) {
+            LinkRepository.instance.db.close();
+        }
+        LinkRepository.instance = null;
+    }
+    initialize() {
+        fs.ensureDirSync(path.dirname(this.dbPath));
+        this.db = new Database(this.dbPath, { timeout: 5000 });
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('foreign_keys = ON');
+        loadVecExtension(this.db);
+        this.createSchema();
+    }
+    createSchema() {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // Documents table
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id TEXT PRIMARY KEY,
+        filename TEXT NOT NULL,
+        file_path TEXT NOT NULL UNIQUE,
+        file_hash TEXT NOT NULL,
+        file_size INTEGER NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        error_message TEXT,
+        chunk_count INTEGER DEFAULT 0,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+    `);
+        // Chunks table
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id TEXT PRIMARY KEY,
+        document_id TEXT NOT NULL,
+        position INTEGER NOT NULL,
+        content TEXT NOT NULL,
+        char_start INTEGER NOT NULL,
+        char_end INTEGER NOT NULL,
+        created_at TEXT NOT NULL,
+        UNIQUE(document_id, position),
+        FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
+      );
+    `);
+        // Embeddings table (sqlite-vec)
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding FLOAT[${EMBEDDING_DIM}]
+      );
+    `);
+        // FTS5 virtual table for BM25 search
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+        content,
+        content='chunks',
+        content_rowid='rowid',
+        tokenize='porter unicode61'
+      );
+    `);
+        // Indexes
+        this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_documents_status ON documents(status);
+      CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(file_hash);
+      CREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_id);
+    `);
+        // Triggers to keep FTS in sync
+        this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO chunks_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
+        INSERT INTO chunks_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
+      END;
+    `);
+    }
+    // ─── Document CRUD ────────────────────────────────────────────────────────
+    createDocument(input) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const id = randomUUID();
+        this.db.prepare(`
+      INSERT INTO documents (id, filename, file_path, file_hash, file_size, status, error_message, chunk_count, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, 'pending', NULL, 0, ?, ?)
+    `).run(id, input.filename, input.file_path, input.file_hash, input.file_size, now, now);
+        return this.getDocument(id);
+    }
+    getDocument(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE id = ?').get(id);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    getDocumentByPath(file_path) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE file_path = ?').get(file_path);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    getDocumentByHash(hash) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM documents WHERE file_hash = ?').get(hash);
+        return row ? this.deserializeDocument(row) : null;
+    }
+    listDocuments(status) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        let query = 'SELECT * FROM documents';
+        const params = [];
+        if (status) {
+            query += ' WHERE status = ?';
+            params.push(status);
+        }
+        query += ' ORDER BY updated_at DESC';
+        const rows = this.db.prepare(query).all(...params);
+        return rows.map((r) => this.deserializeDocument(r));
+    }
+    updateDocumentStatus(id, status, error_message) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        this.db.prepare(`
+      UPDATE documents SET status = ?, error_message = ?, updated_at = ? WHERE id = ?
+    `).run(status, error_message ?? null, now, id);
+        return this.getDocument(id);
+    }
+    updateDocumentChunkCount(id, chunk_count) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        this.db.prepare(`
+      UPDATE documents SET chunk_count = ?, status = 'indexed', updated_at = ? WHERE id = ?
+    `).run(chunk_count, now, id);
+    }
+    deleteDocument(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // CASCADE will delete chunks and embeddings automatically
+        const result = this.db.prepare('DELETE FROM documents WHERE id = ?').run(id);
+        return result.changes > 0;
+    }
+    deleteDocumentByPath(file_path) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const result = this.db.prepare('DELETE FROM documents WHERE file_path = ?').run(file_path);
+        return result.changes > 0;
+    }
+    // ─── Chunk CRUD ───────────────────────────────────────────────────────────
+    createChunk(input) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const id = randomUUID();
+        this.db.prepare(`
+      INSERT INTO chunks (id, document_id, position, content, char_start, char_end, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(id, input.document_id, input.position, input.content, input.char_start, input.char_end, now);
+        return this.getChunk(id);
+    }
+    createChunks(inputs) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const now = new Date().toISOString();
+        const stmt = this.db.prepare(`
+      INSERT INTO chunks (id, document_id, position, content, char_start, char_end, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+        const insertMany = this.db.transaction((items) => {
+            for (const input of items) {
+                stmt.run(randomUUID(), input.document_id, input.position, input.content, input.char_start, input.char_end, now);
+            }
+        });
+        insertMany(inputs);
+    }
+    getChunk(id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const row = this.db.prepare('SELECT * FROM chunks WHERE id = ?').get(id);
+        return row ? this.deserializeChunk(row) : null;
+    }
+    getChunksByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const rows = this.db.prepare('SELECT * FROM chunks WHERE document_id = ? ORDER BY position').all(document_id);
+        return rows.map((r) => this.deserializeChunk(r));
+    }
+    deleteChunksByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        this.db.prepare('DELETE FROM chunks WHERE document_id = ?').run(document_id);
+    }
+    // ─── Embeddings ───────────────────────────────────────────────────────────
+    createEmbedding(chunk_id, embedding) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const embeddingBlob = new Float32Array(embedding);
+        this.db.prepare(`
+      INSERT INTO embeddings (chunk_id, embedding) VALUES (?, ?)
+    `).run(chunk_id, embeddingBlob);
+    }
+    createEmbeddings(items) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const stmt = this.db.prepare(`
+      INSERT INTO embeddings (chunk_id, embedding) VALUES (?, ?)
+    `);
+        const insertMany = this.db.transaction((items) => {
+            for (const item of items) {
+                const embeddingBlob = new Float32Array(item.embedding);
+                stmt.run(item.chunk_id, embeddingBlob);
+            }
+        });
+        insertMany(items);
+    }
+    deleteEmbeddingsByDocument(document_id) {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        // Get all chunk IDs for this document
+        const chunks = this.db.prepare('SELECT id FROM chunks WHERE document_id = ?').all(document_id);
+        const chunkIds = chunks.map(c => c.id);
+        if (chunkIds.length === 0)
+            return;
+        const placeholders = chunkIds.map(() => '?').join(',');
+        this.db.prepare(`DELETE FROM embeddings WHERE chunk_id IN (${placeholders})`).run(...chunkIds);
+    }
+    // ─── Stats ─────────────────────────────────────────────────────────────────
+    getStats() {
+        if (!this.db)
+            throw new Error('DB not initialized');
+        const documents_total = this.db.prepare('SELECT COUNT(*) as cnt FROM documents').get().cnt;
+        const documents_indexed = this.db.prepare("SELECT COUNT(*) as cnt FROM documents WHERE status = 'indexed'").get().cnt;
+        const chunks_total = this.db.prepare('SELECT COUNT(*) as cnt FROM chunks').get().cnt;
+        return { documents_total, documents_indexed, chunks_total };
+    }
+    // ─── Cleanup ───────────────────────────────────────────────────────────────
+    close() {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+    }
+    // ─── Deserializers ─────────────────────────────────────────────────────────
+    deserializeDocument(row) {
+        return {
+            id: row.id,
+            filename: row.filename,
+            file_path: row.file_path,
+            file_hash: row.file_hash,
+            file_size: row.file_size,
+            status: row.status,
+            error_message: row.error_message ?? null,
+            chunk_count: row.chunk_count ?? 0,
+            created_at: row.created_at,
+            updated_at: row.updated_at,
+        };
+    }
+    deserializeChunk(row) {
+        return {
+            id: row.id,
+            document_id: row.document_id,
+            position: row.position,
+            content: row.content,
+            char_start: row.char_start,
+            char_end: row.char_end,
+            created_at: row.created_at,
+        };
+    }
+}