morpheus-cli 0.9.5 → 0.9.6

package/dist/http/routers/link.js

@@ -0,0 +1,239 @@
+import { Router } from 'express';
+import multer from 'multer';
+import path from 'path';
+import fs from 'fs-extra';
+import { homedir } from 'os';
+import { LinkRepository } from '../../runtime/link-repository.js';
+import { LinkWorker } from '../../runtime/link-worker.js';
+import { ConfigManager } from '../../config/manager.js';
+const DOCS_PATH = path.join(homedir(), '.morpheus', 'docs');
+// Configure multer for file uploads
+const storage = multer.diskStorage({
+    destination: async (req, file, cb) => {
+        await fs.ensureDir(DOCS_PATH);
+        cb(null, DOCS_PATH);
+    },
+    filename: (req, file, cb) => {
+        // Multer decodes originalname as Latin1 per HTTP spec.
+        // Re-encode to get the raw bytes and decode as UTF-8.
+        const fixedName = Buffer.from(file.originalname, 'latin1').toString('utf-8');
+        cb(null, fixedName);
+    },
+});
+const upload = multer({
+    storage,
+    limits: {
+        fileSize: 50 * 1024 * 1024, // 50MB default, will check config
+    },
+    fileFilter: (req, file, cb) => {
+        const name = Buffer.from(file.originalname, 'latin1').toString('utf-8');
+        const ext = path.extname(name).toLowerCase();
+        const allowed = ['.pdf', '.txt', '.md', '.docx'];
+        if (allowed.includes(ext)) {
+            cb(null, true);
+        }
+        else {
+            cb(new Error(`Unsupported file type: ${ext}. Allowed: ${allowed.join(', ')}`));
+        }
+    },
+});
+/**
+ * Create the Link router for document management.
+ */
+export function createLinkRouter() {
+    const router = Router();
+    const repository = LinkRepository.getInstance();
+    const worker = LinkWorker.getInstance();
+    // GET /api/link/documents - List all documents
+    router.get('/documents', (req, res) => {
+        try {
+            const status = req.query.status;
+            const documents = repository.listDocuments(status);
+            const stats = repository.getStats();
+            res.json({
+                documents,
+                stats,
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // GET /api/link/documents/:id - Get single document
+    router.get('/documents/:id', (req, res) => {
+        try {
+            const document = repository.getDocument(req.params.id);
+            if (!document) {
+                return res.status(404).json({ error: 'Document not found' });
+            }
+            // Also fetch chunks
+            const chunks = repository.getChunksByDocument(req.params.id);
+            res.json({ document, chunks });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // POST /api/link/documents/upload - Upload a new document
+    router.post('/documents/upload', async (req, res) => {
+        try {
+            const config = ConfigManager.getInstance().getLinkConfig();
+            const maxSizeMB = config.max_file_size_mb;
+            // Configure multer with config max size
+            const uploadWithConfig = multer({
+                storage,
+                limits: { fileSize: maxSizeMB * 1024 * 1024 },
+                fileFilter: (req, file, cb) => {
+                    const name = Buffer.from(file.originalname, 'latin1').toString('utf-8');
+                    const ext = path.extname(name).toLowerCase();
+                    const allowed = ['.pdf', '.txt', '.md', '.docx'];
+                    if (allowed.includes(ext)) {
+                        cb(null, true);
+                    }
+                    else {
+                        cb(new Error(`Unsupported file type: ${ext}`));
+                    }
+                },
+            });
+            // Handle upload
+            await new Promise((resolve, reject) => {
+                uploadWithConfig.single('file')(req, res, (err) => {
+                    if (err)
+                        reject(err);
+                    else
+                        resolve();
+                });
+            });
+            if (!req.file) {
+                return res.status(400).json({ error: 'No file uploaded' });
+            }
+            // Trigger immediate scan
+            const result = await worker.tick();
+            res.json({
+                message: 'File uploaded successfully',
+                filename: Buffer.from(req.file.originalname, 'latin1').toString('utf-8'),
+                path: req.file.path,
+                indexed: result.indexed,
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // DELETE /api/link/documents/:id - Delete a document
+    router.delete('/documents/:id', async (req, res) => {
+        try {
+            const document = repository.getDocument(req.params.id);
+            if (!document) {
+                return res.status(404).json({ error: 'Document not found' });
+            }
+            // Delete from repository (CASCADE removes chunks and embeddings)
+            const deleted = repository.deleteDocument(req.params.id);
+            // Also delete file from disk
+            try {
+                await fs.unlink(document.file_path);
+            }
+            catch {
+                // File may not exist, ignore
+            }
+            res.json({ message: 'Document deleted', deleted });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // POST /api/link/documents/:id/reindex - Force reindex a document
+    router.post('/documents/:id/reindex', async (req, res) => {
+        try {
+            const document = repository.getDocument(req.params.id);
+            if (!document) {
+                return res.status(404).json({ error: 'Document not found' });
+            }
+            // Check if file still exists
+            const exists = await fs.pathExists(document.file_path);
+            if (!exists) {
+                return res.status(400).json({ error: 'Document file no longer exists' });
+            }
+            // Reset status to pending and trigger processing
+            repository.updateDocumentStatus(req.params.id, 'pending');
+            // Process the document
+            const result = await worker.processDocument(document.file_path);
+            res.json({
+                message: 'Document reindexed',
+                result,
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // GET /api/link/config - Get Link configuration
+    router.get('/config', (req, res) => {
+        try {
+            const config = ConfigManager.getInstance().getLinkConfig();
+            res.json(config);
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // POST /api/link/config - Update Link configuration (partial update)
+    router.post('/config', async (req, res) => {
+        try {
+            const configManager = ConfigManager.getInstance();
+            const currentConfig = configManager.get();
+            const currentLinkConfig = configManager.getLinkConfig();
+            const updates = req.body;
+            // Merge updates with current config (ensuring all required fields are present)
+            const newLinkConfig = {
+                ...currentLinkConfig,
+                ...updates,
+            };
+            // Save to zaion.yaml
+            await configManager.save({
+                ...currentConfig,
+                link: newLinkConfig,
+            });
+            // Update worker interval if changed
+            if (updates.scan_interval_ms) {
+                worker.updateInterval(updates.scan_interval_ms);
+            }
+            res.json({
+                message: 'Configuration updated',
+                config: configManager.getLinkConfig(),
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // POST /api/link/worker/scan - Trigger manual scan
+    router.post('/worker/scan', async (req, res) => {
+        try {
+            const result = await worker.tick();
+            res.json({
+                message: 'Scan completed',
+                ...result,
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // GET /api/link/worker/status - Get worker status
+    router.get('/worker/status', (req, res) => {
+        try {
+            const config = ConfigManager.getInstance().getLinkConfig();
+            const stats = repository.getStats();
+            res.json({
+                running: true, // Worker is always running when daemon is up
+                scan_interval_ms: config.scan_interval_ms,
+                ...stats,
+            });
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    return router;
+}

package/dist/http/routers/skills.js

@@ -4,7 +4,7 @@ import extract from 'extract-zip';
 import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
-import { SkillRegistry, updateSkillDelegateDescription } from '../../runtime/skills/index.js';
+import { SkillRegistry } from '../../runtime/skills/index.js';
 import { DisplayManager } from '../../runtime/display.js';
 import { PATHS } from '../../config/paths.js';
 import { SkillMetadataSchema } from '../../runtime/skills/schema.js';
@@ -119,8 +119,6 @@ export function createSkillsRouter() {
         try {
             const registry = SkillRegistry.getInstance();
             const result = await registry.reload();
-            // Update skill_delegate tool description with new skills
-            updateSkillDelegateDescription();
             display.log(`Skills reloaded: ${result.skills.length} loaded, ${result.errors.length} errors`, {
                 source: 'SkillsAPI',
             });
@@ -207,7 +205,6 @@ export function createSkillsRouter() {
             // Reload skills
             const registry = SkillRegistry.getInstance();
             await registry.reload();
-            updateSkillDelegateDescription();
             display.log(`Skill "${metadata.name}" uploaded successfully`, { source: 'SkillsAPI' });
             res.json({
                 success: true,
@@ -258,8 +255,6 @@ export function createSkillsRouter() {
             if (!success) {
                 return res.status(404).json({ error: `Skill "${name}" not found` });
             }
-            // Update skill_delegate tool description
-            updateSkillDelegateDescription();
             display.log(`Skill "${name}" enabled`, { source: 'SkillsAPI' });
             res.json({ success: true, name, enabled: true });
         }
@@ -277,8 +272,6 @@ export function createSkillsRouter() {
             if (!success) {
                 return res.status(404).json({ error: `Skill "${name}" not found` });
             }
-            // Update skill_delegate tool description
-            updateSkillDelegateDescription();
             display.log(`Skill "${name}" disabled`, { source: 'SkillsAPI' });
             res.json({ success: true, name, enabled: false });
         }

package/dist/runtime/apoc.js

@@ -259,7 +259,7 @@ ${context ? `CONTEXT FROM ORACLE:\n${context}` : ""}
         try {
             const inputCount = messages.length;
             const startMs = Date.now();
-            const response = await this.agent.invoke({ messages }, { recursionLimit: 50 });
+            const response = await this.agent.invoke({ messages }, { recursionLimit: 10 });
             const durationMs = Date.now() - startMs;
             const apocConfig = this.config.apoc || this.config.llm;
             const lastMessage = response.messages[response.messages.length - 1];

package/dist/runtime/audit/repository.js

@@ -188,7 +188,7 @@ export class AuditRepository {
         SUM(CASE WHEN ae.event_type = 'llm_call'        THEN 1 ELSE 0 END) as llmCallCount,
         SUM(CASE WHEN ae.event_type = 'tool_call'       THEN 1 ELSE 0 END) as toolCallCount,
         SUM(CASE WHEN ae.event_type = 'mcp_tool'        THEN 1 ELSE 0 END) as mcpToolCount,
-        SUM(CASE WHEN ae.event_type = 'skill_executed'  THEN 1 ELSE 0 END) as skillCount,
+        SUM(CASE WHEN ae.event_type = 'skill_loaded'  THEN 1 ELSE 0 END) as skillCount,
         SUM(CASE WHEN ae.event_type = 'memory_recovery' THEN 1 ELSE 0 END) as memoryRecoveryCount,
         SUM(CASE WHEN ae.event_type = 'memory_persist'  THEN 1 ELSE 0 END) as memoryPersistCount,
         SUM(CASE WHEN ae.event_type = 'chronos_job'     THEN 1 ELSE 0 END) as chronosJobCount,

package/dist/runtime/link-chunker.js

@@ -0,0 +1,214 @@
+import { createHash } from 'crypto';
+import fs from 'fs-extra';
+import path from 'path';
+import { PDFParse } from 'pdf-parse';
+import mammoth from 'mammoth';
+// ─── Hashing ─────────────────────────────────────────────────────────────────
+/**
+ * Calculate SHA-256 hash of file content.
+ */
+export function hashDocument(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Calculate SHA-256 hash of a file by path.
+ */
+export async function hashFile(filePath) {
+    const content = await fs.readFile(filePath);
+    return hashDocument(content);
+}
+/**
+ * Split text into chunks respecting sentence boundaries.
+ * @param text - The text to chunk
+ * @param chunkSize - Target size in characters (default: 500)
+ * @param minChunkSize - Minimum chunk size to avoid tiny chunks (default: 100)
+ */
+export function chunkText(text, chunkSize = 500, minChunkSize = 100) {
+    const chunks = [];
+    let position = 0;
+    let charPos = 0;
+    // Split by paragraphs first
+    const paragraphs = text.split(/\n\n+/);
+    let currentChunk = '';
+    let chunkStart = 0;
+    for (const paragraph of paragraphs) {
+        // If adding this paragraph exceeds chunk size
+        if (currentChunk.length + paragraph.length + 2 > chunkSize && currentChunk.length >= minChunkSize) {
+            // Save current chunk
+            chunks.push({
+                content: currentChunk.trim(),
+                position: position++,
+                char_start: chunkStart,
+                char_end: chunkStart + currentChunk.length,
+            });
+            currentChunk = paragraph;
+            chunkStart = charPos;
+        }
+        else if (paragraph.length > chunkSize) {
+            // Paragraph is too long, split by sentences
+            if (currentChunk.length > 0) {
+                // Save current chunk first
+                chunks.push({
+                    content: currentChunk.trim(),
+                    position: position++,
+                    char_start: chunkStart,
+                    char_end: chunkStart + currentChunk.length,
+                });
+                currentChunk = '';
+            }
+            const sentences = splitBySentences(paragraph);
+            let sentenceChunk = '';
+            let sentenceStart = charPos;
+            for (const sentence of sentences) {
+                if (sentenceChunk.length + sentence.length + 1 > chunkSize && sentenceChunk.length >= minChunkSize) {
+                    chunks.push({
+                        content: sentenceChunk.trim(),
+                        position: position++,
+                        char_start: sentenceStart,
+                        char_end: sentenceStart + sentenceChunk.length,
+                    });
+                    sentenceChunk = sentence;
+                    sentenceStart = charPos + (paragraph.indexOf(sentence) > 0 ? paragraph.indexOf(sentence) : 0);
+                }
+                else {
+                    sentenceChunk += (sentenceChunk ? ' ' : '') + sentence;
+                }
+            }
+            if (sentenceChunk.trim()) {
+                currentChunk = sentenceChunk;
+                chunkStart = sentenceStart;
+            }
+        }
+        else {
+            // Add paragraph to current chunk
+            currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
+            if (!currentChunk) {
+                chunkStart = charPos;
+            }
+        }
+        charPos += paragraph.length + 2; // +2 for paragraph separator
+    }
+    // Don't forget the last chunk
+    if (currentChunk.trim()) {
+        chunks.push({
+            content: currentChunk.trim(),
+            position,
+            char_start: chunkStart,
+            char_end: chunkStart + currentChunk.length,
+        });
+    }
+    return chunks;
+}
+/**
+ * Split text by sentences using common sentence delimiters.
+ */
+function splitBySentences(text) {
+    // Match sentences ending with . ! ? followed by space or end of string
+    const sentences = text.match(/[^.!?]*[.!?]+(?:\s+|$)/g) || [text];
+    return sentences.map(s => s.trim()).filter(Boolean);
+}
+/**
+ * Parse PDF file and extract text.
+ */
+export async function parsePDF(filePath) {
+    const dataBuffer = await fs.readFile(filePath);
+    const parser = new PDFParse({ data: dataBuffer });
+    const textResult = await parser.getText();
+    const text = textResult.text || '';
+    const infoResult = await parser.getInfo();
+    return {
+        text,
+        metadata: {
+            pageCount: infoResult.total,
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse DOCX file and extract text.
+ */
+export async function parseDOCX(filePath) {
+    const result = await mammoth.extractRawText({ path: filePath });
+    const text = result.value;
+    return {
+        text,
+        metadata: {
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse plain text file.
+ */
+export async function parseTXT(filePath) {
+    const text = await fs.readFile(filePath, 'utf-8');
+    return {
+        text,
+        metadata: {
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+        },
+    };
+}
+/**
+ * Parse Markdown file (treated as plain text for chunking).
+ */
+export async function parseMD(filePath) {
+    return parseTXT(filePath);
+}
+// ─── Supported Formats ───────────────────────────────────────────────────────
+const SUPPORTED_EXTENSIONS = ['.pdf', '.txt', '.md', '.docx'];
+/**
+ * Check if a file extension is supported.
+ */
+export function isSupportedFormat(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return SUPPORTED_EXTENSIONS.includes(ext);
+}
+/**
+ * Get the content type based on file extension.
+ */
+export function getContentType(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const contentTypes = {
+        '.pdf': 'application/pdf',
+        '.txt': 'text/plain',
+        '.md': 'text/markdown',
+        '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    };
+    return contentTypes[ext] || 'application/octet-stream';
+}
+/**
+ * Parse a document based on its file extension.
+ */
+export async function parseDocument(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    switch (ext) {
+        case '.pdf':
+            return parsePDF(filePath);
+        case '.docx':
+            return parseDOCX(filePath);
+        case '.txt':
+            return parseTXT(filePath);
+        case '.md':
+            return parseMD(filePath);
+        default:
+            throw new Error(`Unsupported file format: ${ext}`);
+    }
+}
+/**
+ * Process a document: parse, chunk, and return chunks with metadata.
+ */
+export async function processDocument(filePath, chunkSize = 500) {
+    // Parse document
+    const parsed = await parseDocument(filePath);
+    // Calculate hash from raw file bytes (must match hashFile used by the caller)
+    const hash = await hashFile(filePath);
+    // Chunk text
+    const chunks = chunkText(parsed.text, chunkSize);
+    return {
+        text: parsed.text,
+        chunks,
+        hash,
+        metadata: parsed.metadata,
+    };
+}