morpheus-cli 0.9.5 → 0.9.6

package/dist/runtime/link-worker.js

@@ -0,0 +1,284 @@
+import { homedir } from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import { LinkRepository } from './link-repository.js';
+import { LinkSearch } from './link-search.js';
+import { hashFile, processDocument, isSupportedFormat } from './link-chunker.js';
+import { EmbeddingService } from './memory/embedding.service.js';
+import { ConfigManager } from '../config/manager.js';
+import { DisplayManager } from './display.js';
+/**
+ * LinkWorker - Background worker for document indexing
+ *
+ * Scans ~/.morpheus/docs folder, processes new/changed documents,
+ * generates embeddings, and removes deleted documents from the index.
+ */
+export class LinkWorker {
+    static instance = null;
+    repository;
+    search;
+    embeddingService = null;
+    intervalId = null;
+    isRunning = false;
+    display = DisplayManager.getInstance();
+    docsPath;
+    constructor() {
+        this.repository = LinkRepository.getInstance();
+        this.search = LinkSearch.getInstance();
+        this.docsPath = path.join(homedir(), '.morpheus', 'docs');
+    }
+    static getInstance() {
+        if (!LinkWorker.instance) {
+            LinkWorker.instance = new LinkWorker();
+        }
+        return LinkWorker.instance;
+    }
+    static setInstance(instance) {
+        LinkWorker.instance = instance;
+    }
+    static resetInstance() {
+        if (LinkWorker.instance) {
+            LinkWorker.instance.stop();
+        }
+        LinkWorker.instance = null;
+    }
+    /**
+     * Start the background worker with interval-based scanning.
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const intervalMs = config.scan_interval_ms;
+        this.isRunning = true;
+        this.display.log('LinkWorker started', { source: 'Link' });
+        // Run initial scan immediately
+        this.tick().catch(err => {
+            this.display.log(`LinkWorker initial scan failed: ${err.message}`, { source: 'Link', level: 'error' });
+        });
+        // Schedule periodic scans
+        this.intervalId = setInterval(() => {
+            this.tick().catch(err => {
+                this.display.log(`LinkWorker tick failed: ${err.message}`, { source: 'Link', level: 'error' });
+            });
+        }, intervalMs);
+    }
+    /**
+     * Stop the background worker.
+     */
+    stop() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        this.isRunning = false;
+        this.display.log('LinkWorker stopped', { source: 'Link' });
+    }
+    /**
+     * Update the scan interval (hot-reload).
+     */
+    updateInterval(intervalMs) {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = setInterval(() => {
+                this.tick().catch(err => {
+                    this.display.log(`LinkWorker tick failed: ${err.message}`, { source: 'Link', level: 'error' });
+                });
+            }, intervalMs);
+            this.display.log(`LinkWorker interval updated to ${intervalMs}ms`, { source: 'Link' });
+        }
+    }
+    /**
+     * Perform a single scan cycle.
+     */
+    async tick() {
+        // Ensure embedding service is initialized
+        if (!this.embeddingService) {
+            this.embeddingService = await EmbeddingService.getInstance();
+        }
+        // Ensure docs folder exists
+        await fs.ensureDir(this.docsPath);
+        const stats = {
+            indexed: 0,
+            removed: 0,
+            errors: 0,
+        };
+        try {
+            // Scan for new/changed documents
+            const files = await this.scanFolder();
+            this.display.log(`LinkWorker found ${files.length} files`, { source: 'Link', level: 'debug' });
+            // Process each file
+            for (const filePath of files) {
+                try {
+                    const result = await this.processDocument(filePath);
+                    if (result === 'indexed') {
+                        stats.indexed++;
+                    }
+                    else if (result === 'error') {
+                        stats.errors++;
+                    }
+                }
+                catch (err) {
+                    this.display.log(`Failed to process ${filePath}: ${err.message}`, { source: 'Link', level: 'error' });
+                    stats.errors++;
+                }
+            }
+            // Remove deleted documents
+            stats.removed = await this.removeDeletedDocuments(files);
+            if (stats.indexed > 0 || stats.removed > 0) {
+                this.display.log(`LinkWorker: indexed ${stats.indexed}, removed ${stats.removed}, errors ${stats.errors}`, { source: 'Link', level: 'info' });
+            }
+        }
+        catch (err) {
+            this.display.log(`LinkWorker tick error: ${err.message}`, { source: 'Link', level: 'error' });
+            stats.errors++;
+        }
+        return stats;
+    }
+    /**
+     * Scan the docs folder for supported files.
+     */
+    async scanFolder() {
+        const files = [];
+        const entries = await fs.readdir(this.docsPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isFile()) {
+                const filePath = path.join(this.docsPath, entry.name);
+                if (isSupportedFormat(filePath)) {
+                    files.push(filePath);
+                }
+            }
+        }
+        return files;
+    }
+    /**
+     * Process a single document: check hash, parse, chunk, embed.
+     */
+    async processDocument(filePath) {
+        const existingDoc = this.repository.getDocumentByPath(filePath);
+        // Calculate file hash
+        let fileHash;
+        try {
+            fileHash = await hashFile(filePath);
+        }
+        catch (err) {
+            // File might not be readable
+            if (existingDoc) {
+                this.repository.updateDocumentStatus(existingDoc.id, 'error', `Failed to read file: ${err.message}`);
+            }
+            return 'error';
+        }
+        // Check if document already indexed with same hash
+        if (existingDoc && existingDoc.file_hash === fileHash && existingDoc.status === 'indexed') {
+            return 'skipped';
+        }
+        // Get file stats
+        const stats = await fs.stat(filePath);
+        const fileSize = stats.size;
+        // Check max file size
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const maxSizeBytes = config.max_file_size_mb * 1024 * 1024;
+        if (fileSize > maxSizeBytes) {
+            if (existingDoc) {
+                this.repository.updateDocumentStatus(existingDoc.id, 'error', `File exceeds max size of ${config.max_file_size_mb}MB`);
+            }
+            return 'error';
+        }
+        // Create or update document record
+        const filename = path.basename(filePath);
+        let document;
+        if (existingDoc) {
+            // Update existing document - delete old chunks first
+            this.repository.deleteChunksByDocument(existingDoc.id);
+            this.repository.deleteEmbeddingsByDocument(existingDoc.id);
+            this.repository.updateDocumentStatus(existingDoc.id, 'indexing');
+            document = existingDoc;
+        }
+        else {
+            // Create new document
+            document = this.repository.createDocument({
+                filename,
+                file_path: filePath,
+                file_hash: fileHash,
+                file_size: fileSize,
+            });
+        }
+        try {
+            // Index the document
+            await this.indexDocument(document.id, filePath, fileHash);
+            // Update status to indexed
+            const chunks = this.repository.getChunksByDocument(document.id);
+            this.repository.updateDocumentChunkCount(document.id, chunks.length);
+            this.display.log(`Indexed document: ${filename} (${chunks.length} chunks)`, { source: 'Link', level: 'debug' });
+            return 'indexed';
+        }
+        catch (err) {
+            this.repository.updateDocumentStatus(document.id, 'error', err.message);
+            return 'error';
+        }
+    }
+    /**
+     * Index a document: parse, chunk, generate embeddings.
+     */
+    async indexDocument(documentId, filePath, fileHash) {
+        const config = ConfigManager.getInstance().getLinkConfig();
+        const chunkSize = config.chunk_size;
+        // Parse and chunk the document
+        const processed = await processDocument(filePath, chunkSize);
+        // Verify hash matches (file might have changed during processing)
+        if (processed.hash !== fileHash) {
+            throw new Error('File changed during processing - hash mismatch');
+        }
+        // Create chunks in database
+        const chunkInputs = processed.chunks.map(chunk => ({
+            document_id: documentId,
+            position: chunk.position,
+            content: chunk.content,
+            char_start: chunk.char_start,
+            char_end: chunk.char_end,
+        }));
+        this.repository.createChunks(chunkInputs);
+        // Get the created chunks with IDs
+        const chunks = this.repository.getChunksByDocument(documentId);
+        // Generate embeddings for each chunk
+        await this.generateEmbeddings(chunks);
+    }
+    /**
+     * Generate embeddings for chunks using Sati's EmbeddingService.
+     */
+    async generateEmbeddings(chunks) {
+        if (!this.embeddingService) {
+            this.embeddingService = await EmbeddingService.getInstance();
+        }
+        const embeddings = [];
+        // Process in batches to avoid memory issues
+        const batchSize = 50;
+        for (let i = 0; i < chunks.length; i += batchSize) {
+            const batch = chunks.slice(i, i + batchSize);
+            const batchEmbeddings = await Promise.all(batch.map(async (chunk) => {
+                const embedding = await this.embeddingService.generate(chunk.content);
+                return { chunk_id: chunk.id, embedding };
+            }));
+            embeddings.push(...batchEmbeddings);
+        }
+        // Store embeddings in database
+        this.repository.createEmbeddings(embeddings);
+    }
+    /**
+     * Remove documents that no longer exist in the docs folder.
+     */
+    async removeDeletedDocuments(existingFiles) {
+        const existingPaths = new Set(existingFiles);
+        const documents = this.repository.listDocuments();
+        let removed = 0;
+        for (const doc of documents) {
+            if (!existingPaths.has(doc.file_path)) {
+                // Document file no longer exists - remove from index
+                this.repository.deleteDocument(doc.id);
+                removed++;
+                this.display.log(`Removed deleted document: ${doc.filename}`, { source: 'Link', level: 'debug' });
+            }
+        }
+        return removed;
+    }
+}

package/dist/runtime/link.js

@@ -0,0 +1,295 @@
+import { HumanMessage, SystemMessage, AIMessage } from "@langchain/core/messages";
+import { z } from "zod";
+import { DynamicStructuredTool } from "@langchain/core/tools";
+import { ConfigManager } from '../config/manager.js';
+import { LinkRepository } from './link-repository.js';
+import { LinkSearch } from './link-search.js';
+import { ProviderFactory } from './providers/factory.js';
+import { ProviderError } from './errors.js';
+import { DisplayManager } from './display.js';
+import { TaskRequestContext } from './tasks/context.js';
+import { extractRawUsage, persistAgentMessage, buildAgentResult, emitToolAuditEvents } from './subagent-utils.js';
+import { buildDelegationTool } from './tools/delegation-utils.js';
+const LINK_BASE_DESCRIPTION = `Delegate to Link, the documentation specialist subagent.
+
+Link has access to indexed user documents (PDFs, Markdown, TXT, DOCX) stored in ~/.morpheus/docs.
+It uses an LLM to search, reason over, and synthesize answers from document content.
+
+Use this tool when the user asks about information that might be in their uploaded documents.
+Input should be a natural language query or question about the user's documentation.`;
+function buildDocumentCatalogSection(repository) {
+    try {
+        const docs = repository.listDocuments('indexed');
+        if (docs.length === 0) {
+            return '\n\nIndexed documents: none currently indexed.';
+        }
+        const lines = docs.map((d) => `- ${d.filename} (${d.chunk_count} chunks)`);
+        return `\n\nIndexed documents:\n${lines.join('\n')}`;
+    }
+    catch {
+        return '\n\nIndexed documents: unable to retrieve list.';
+    }
+}
+/**
+ * Link - Documentation Specialist Subagent
+ *
+ * Provides RAG (Retrieval-Augmented Generation) capabilities over user documents.
+ * Uses a ReactAgent with an LLM to reason over search results and synthesize answers.
+ */
+export class Link {
+    static instance = null;
+    static currentSessionId = undefined;
+    static _delegateTool = null;
+    config;
+    agentConfig;
+    repository;
+    search;
+    agent;
+    display = DisplayManager.getInstance();
+    constructor(config) {
+        this.config = config;
+        this.agentConfig = ConfigManager.getInstance().getLinkConfig();
+        this.repository = LinkRepository.getInstance();
+        this.search = LinkSearch.getInstance();
+    }
+    static getInstance(config) {
+        if (!Link.instance) {
+            if (!config) {
+                config = ConfigManager.getInstance().get();
+            }
+            Link.instance = new Link(config);
+        }
+        return Link.instance;
+    }
+    static resetInstance() {
+        Link.instance = null;
+        Link._delegateTool = null;
+    }
+    static setSessionId(id) {
+        Link.currentSessionId = id;
+    }
+    /**
+     * Build the internal search tool that the ReactAgent will use.
+     */
+    buildTools() {
+        const search = this.search;
+        const repository = this.repository;
+        const agentConfig = this.agentConfig;
+        const searchTool = new DynamicStructuredTool({
+            name: 'link_search_documents',
+            description: 'Search ALL indexed user documents using hybrid vector + keyword search. Returns the most relevant document chunks for a given query. Use this for broad searches when you don\'t know which document contains the answer.',
+            schema: z.object({
+                query: z.string().describe('The search query to find relevant document passages'),
+                limit: z.number().optional().describe('Maximum number of results to return (default: max_results from config)'),
+            }),
+            func: async ({ query, limit }) => {
+                const maxResults = limit ?? agentConfig.max_results;
+                const threshold = agentConfig.score_threshold;
+                const results = await search.search(query, maxResults, threshold);
+                if (results.length === 0) {
+                    return `No relevant documents found for query: "${query}"`;
+                }
+                const formatted = results
+                    .map((r, i) => `[${i + 1}] Source: ${r.filename} (chunk ${r.position}, score: ${r.score.toFixed(3)})\n${r.content}`)
+                    .join('\n\n---\n\n');
+                return `Found ${results.length} relevant passages:\n\n${formatted}`;
+            },
+        });
+        const listDocumentsTool = new DynamicStructuredTool({
+            name: 'link_list_documents',
+            description: 'List indexed documents. Use this to find documents by filename before searching within a specific one. Supports optional name filter (case-insensitive partial match).',
+            schema: z.object({
+                name_filter: z.string().optional().describe('Optional partial filename to filter by (case-insensitive). E.g. "CV", "contrato", "readme"'),
+            }),
+            func: async ({ name_filter }) => {
+                const docs = repository.listDocuments('indexed');
+                if (docs.length === 0) {
+                    return 'No indexed documents found.';
+                }
+                let filtered = docs;
+                if (name_filter) {
+                    const lower = name_filter.toLowerCase();
+                    filtered = docs.filter(d => d.filename.toLowerCase().includes(lower));
+                }
+                if (filtered.length === 0) {
+                    const allNames = docs.map(d => `- ${d.filename}`).join('\n');
+                    return `No documents matching "${name_filter}". Available documents:\n${allNames}`;
+                }
+                const lines = filtered.map(d => `- [${d.id}] ${d.filename} (${d.chunk_count} chunks)`);
+                return `Found ${filtered.length} document(s):\n${lines.join('\n')}`;
+            },
+        });
+        const searchInDocumentTool = new DynamicStructuredTool({
+            name: 'link_search_in_document',
+            description: 'Search within a SPECIFIC document by its ID. Use this when you know which document to search (e.g. after using link_list_documents to find it). More precise than link_search_documents for targeted queries.',
+            schema: z.object({
+                document_id: z.string().describe('The document ID to search within (get this from link_list_documents)'),
+                query: z.string().describe('The search query to find relevant passages within this document'),
+                limit: z.number().optional().describe('Maximum number of results (default: max_results from config)'),
+            }),
+            func: async ({ document_id, query, limit }) => {
+                const doc = repository.getDocument(document_id);
+                if (!doc) {
+                    return `Document not found: ${document_id}`;
+                }
+                const maxResults = limit ?? agentConfig.max_results;
+                const threshold = agentConfig.score_threshold;
+                const results = await search.searchInDocument(query, document_id, maxResults, threshold);
+                if (results.length === 0) {
+                    return `No relevant passages found in "${doc.filename}" for query: "${query}"`;
+                }
+                const formatted = results
+                    .map((r, i) => `[${i + 1}] (chunk ${r.position}, score: ${r.score.toFixed(3)})\n${r.content}`)
+                    .join('\n\n---\n\n');
+                return `Found ${results.length} passages in "${doc.filename}":\n\n${formatted}`;
+            },
+        });
+        return [listDocumentsTool, searchTool, searchInDocumentTool];
+    }
+    async initialize() {
+        this.repository.initialize();
+        await this.search.initialize();
+        const linkConfig = this.agentConfig;
+        const personality = linkConfig.personality || 'documentation_specialist';
+        const tools = this.buildTools();
+        // Update delegate tool description with current document catalog
+        if (Link._delegateTool) {
+            const full = `${LINK_BASE_DESCRIPTION}${buildDocumentCatalogSection(this.repository)}`;
+            Link._delegateTool.description = full;
+        }
+        this.display.log(`Link initialized with personality: ${personality}.`, { source: 'Link' });
+        try {
+            this.agent = await ProviderFactory.create(linkConfig, tools);
+        }
+        catch (err) {
+            throw new ProviderError(linkConfig.provider, err, 'Link subagent initialization failed');
+        }
+    }
+    /**
+     * Search documents for relevant information (used internally by search tool and HTTP API).
+     */
+    async searchDocuments(query, limit) {
+        const maxResults = limit ?? this.agentConfig.max_results;
+        const threshold = this.agentConfig.score_threshold;
+        const results = await this.search.search(query, maxResults, threshold);
+        return {
+            results: results.map(r => ({
+                chunk_id: r.chunk_id,
+                content: r.content,
+                document_id: r.document_id,
+                filename: r.filename,
+                position: r.position,
+                score: r.score,
+            })),
+            total: results.length,
+        };
+    }
+    /**
+     * Execute a query using the LLM-powered ReactAgent.
+     */
+    async execute(task, context, sessionId, taskContext) {
+        const linkConfig = this.agentConfig;
+        if (!this.agent) {
+            await this.initialize();
+        }
+        this.display.log(`Executing delegated task in Link: ${task.slice(0, 80)}...`, {
+            source: 'Link',
+        });
+        const personality = linkConfig.personality || 'documentation_specialist';
+        const systemMessage = new SystemMessage(`
+You are Link, ${personality === 'documentation_specialist' ? 'a documentation specialist and knowledge synthesizer' : personality}, a subagent in Morpheus.
+
+You have access to the user's indexed documents via the link_search_documents tool.
+
+Rules:
+1. ALWAYS search the documents before answering. Never answer from general knowledge alone when documents may contain relevant information.
+2. Synthesize search results into a clear, natural response. Do not just dump raw chunks.
+3. Cite sources by filename when referencing specific information (e.g., "According to readme.md, ...").
+4. If no relevant documents are found, clearly state that no matching documentation was found.
+5. NEVER fabricate or invent document content. Only report what the search actually returns.
+6. If the query is ambiguous, search with multiple relevant terms to maximize coverage.
+7. Keep responses concise and focused on the user's question.
+8. Respond in the language requested by the user. If not explicit, use the dominant language of the task/context.
+
+## Tool Selection Strategy
+- When the user refers to a SPECIFIC document (by name or partial name like "meu currículo", "CV", "contrato"):
+  1. First call **link_list_documents** with a name filter to find the document ID.
+  2. Then call **link_search_in_document** with that document ID for a targeted search.
+- When the user asks a general question without referencing a specific document:
+  - Use **link_search_documents** for a broad search across all documents.
+- When unsure which document contains the answer, start with **link_search_documents**, then narrow down with **link_search_in_document** if results point to a specific file.
+
+${context ? `Context:\n${context}` : ''}
+    `);
+        const userMessage = new HumanMessage(task);
+        const messages = [systemMessage, userMessage];
+        try {
+            const invokeContext = {
+                origin_channel: taskContext?.origin_channel ?? 'api',
+                session_id: taskContext?.session_id ?? sessionId ?? 'default',
+                origin_message_id: taskContext?.origin_message_id,
+                origin_user_id: taskContext?.origin_user_id,
+            };
+            const inputCount = messages.length;
+            const startMs = Date.now();
+            const response = await TaskRequestContext.run(invokeContext, () => this.agent.invoke({ messages }, { recursionLimit: 25 }));
+            const durationMs = Date.now() - startMs;
+            const lastMessage = response.messages[response.messages.length - 1];
+            const content = typeof lastMessage.content === 'string'
+                ? lastMessage.content
+                : JSON.stringify(lastMessage.content);
+            const rawUsage = extractRawUsage(lastMessage);
+            const stepCount = response.messages.filter((m) => m instanceof AIMessage).length;
+            const targetSession = sessionId ?? Link.currentSessionId ?? 'link';
+            await persistAgentMessage('link', content, linkConfig, targetSession, rawUsage, durationMs);
+            emitToolAuditEvents(response.messages.slice(inputCount), targetSession, 'link');
+            this.display.log('Link task completed.', { source: 'Link' });
+            return buildAgentResult(content, linkConfig, rawUsage, durationMs, stepCount);
+        }
+        catch (err) {
+            throw new ProviderError(linkConfig.provider, err, 'Link task execution failed');
+        }
+    }
+    /**
+     * Create the delegation tool for Oracle.
+     */
+    createDelegateTool() {
+        if (!Link._delegateTool) {
+            Link._delegateTool = buildDelegationTool({
+                name: 'link_delegate',
+                description: LINK_BASE_DESCRIPTION,
+                agentKey: 'link',
+                agentLabel: 'Link',
+                auditAgent: 'link',
+                isSync: () => ConfigManager.getInstance().getLinkConfig().execution_mode === 'sync',
+                notifyText: '📚 Link is searching your documentation...',
+                executeSync: (task, context, sessionId, ctx) => Link.getInstance().execute(task, context, sessionId, {
+                    origin_channel: ctx?.origin_channel ?? 'api',
+                    session_id: sessionId,
+                    origin_message_id: ctx?.origin_message_id,
+                    origin_user_id: ctx?.origin_user_id,
+                }),
+            });
+        }
+        return Link._delegateTool;
+    }
+    /**
+     * Refresh the delegate tool description with current document catalog.
+     */
+    static async refreshDelegateCatalog() {
+        if (Link._delegateTool) {
+            try {
+                const repository = LinkRepository.getInstance();
+                const full = `${LINK_BASE_DESCRIPTION}${buildDocumentCatalogSection(repository)}`;
+                Link._delegateTool.description = full;
+            }
+            catch { /* non-critical */ }
+        }
+    }
+    async reload() {
+        this.config = ConfigManager.getInstance().get();
+        this.agentConfig = ConfigManager.getInstance().getLinkConfig();
+        this.agent = undefined;
+        await this.initialize();
+    }
+}

package/dist/runtime/memory/sati/service.js

@@ -97,7 +97,7 @@ export class SatiService {
                 console.warn('[SatiService] Failed to persist input log:', e);
             }
             const satiStartMs = Date.now();
-            const response = await agent.invoke({ messages }, { recursionLimit: 50 });
+            const response = await agent.invoke({ messages }, { recursionLimit: 10 });
             const satiDurationMs = Date.now() - satiStartMs;
             const lastMessage = response.messages[response.messages.length - 1];
             let content = lastMessage.content.toString();

package/dist/runtime/neo.js

@@ -137,7 +137,7 @@ ${context ? `Context:\n${context}` : ""}
             };
             const inputCount = messages.length;
             const startMs = Date.now();
-            const response = await TaskRequestContext.run(invokeContext, () => this.agent.invoke({ messages }, { recursionLimit: 50 }));
+            const response = await TaskRequestContext.run(invokeContext, () => this.agent.invoke({ messages }, { recursionLimit: 10 }));
             const durationMs = Date.now() - startMs;
             const lastMessage = response.messages[response.messages.length - 1];
             const content = typeof lastMessage.content === "string"