morpheus-cli 0.1.5 → 0.1.6

package/dist/config/schemas.js

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { DEFAULT_CONFIG } from '../types/config.js';
 export const AudioConfigSchema = z.object({
+    provider: z.enum(['google']).default(DEFAULT_CONFIG.audio.provider),
     enabled: z.boolean().default(DEFAULT_CONFIG.audio.enabled),
     apiKey: z.string().optional(),
     maxDurationSeconds: z.number().default(DEFAULT_CONFIG.audio.maxDurationSeconds),
@@ -16,6 +17,7 @@ export const ConfigSchema = z.object({
         provider: z.enum(['openai', 'anthropic', 'ollama', 'gemini']).default(DEFAULT_CONFIG.llm.provider),
         model: z.string().min(1).default(DEFAULT_CONFIG.llm.model),
         temperature: z.number().min(0).max(1).default(DEFAULT_CONFIG.llm.temperature),
+        max_tokens: z.number().int().positive().optional(),
         api_key: z.string().optional(),
     }).default(DEFAULT_CONFIG.llm),
     audio: AudioConfigSchema.default(DEFAULT_CONFIG.audio),
@@ -43,3 +45,14 @@ export const ConfigSchema = z.object({
         retention: z.string().default(DEFAULT_CONFIG.logging.retention),
     }).default(DEFAULT_CONFIG.logging),
 });
+export const MCPServerConfigSchema = z.object({
+    transport: z.enum(['stdio', 'http']),
+    command: z.string().min(1, 'Command is required'),
+    args: z.array(z.string()).optional().default([]),
+    env: z.record(z.string(), z.string()).optional().default({}),
+    _comment: z.string().optional(),
+});
+export const MCPConfigFileSchema = z.record(z.string(), z.union([
+    MCPServerConfigSchema,
+    z.string(),
+]));

package/dist/http/__tests__/auth.test.js

@@ -0,0 +1,53 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import request from 'supertest';
+import express from 'express';
+import bodyParser from 'body-parser';
+import { authMiddleware } from '../middleware/auth.js';
+import { AUTH_HEADER } from '../../types/auth.js';
+import { DisplayManager } from '../../runtime/display.js';
+vi.mock('../../runtime/display.js');
+describe('Auth Middleware', () => {
+    let app;
+    let mockDisplayManager;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        mockDisplayManager = {
+            log: vi.fn(),
+        };
+        DisplayManager.getInstance.mockReturnValue(mockDisplayManager);
+        app = express();
+        app.use(bodyParser.json());
+        app.use(authMiddleware);
+        app.get('/test', (req, res) => res.status(200).json({ success: true }));
+        // Reset env var
+        delete process.env.THE_ARCHITECT_PASS;
+    });
+    it('should allow access when THE_ARCHITECT_PASS is not set', async () => {
+        const res = await request(app).get('/test');
+        expect(res.status).toBe(200);
+        expect(res.body.success).toBe(true);
+    });
+    it('should block access when THE_ARCHITECT_PASS is set and header is missing', async () => {
+        process.env.THE_ARCHITECT_PASS = 'secret123';
+        const res = await request(app).get('/test');
+        expect(res.status).toBe(401);
+        expect(res.body.error).toBe('Unauthorized');
+        expect(mockDisplayManager.log).toHaveBeenCalledWith(expect.stringContaining('Unauthorized'), expect.objectContaining({ source: 'http', level: 'warning' }));
+    });
+    it('should block access when THE_ARCHITECT_PASS is set and header is incorrect', async () => {
+        process.env.THE_ARCHITECT_PASS = 'secret123';
+        const res = await request(app)
+            .get('/test')
+            .set(AUTH_HEADER, 'wrongpass');
+        expect(res.status).toBe(401);
+        expect(mockDisplayManager.log).toHaveBeenCalled();
+    });
+    it('should allow access when THE_ARCHITECT_PASS is set and header matches', async () => {
+        process.env.THE_ARCHITECT_PASS = 'secret123';
+        const res = await request(app)
+            .get('/test')
+            .set(AUTH_HEADER, 'secret123');
+        expect(res.status).toBe(200);
+        expect(res.body.success).toBe(true);
+    });
+});

package/dist/http/api.js

@@ -4,6 +4,7 @@ import { PATHS } from '../config/paths.js';
 import { DisplayManager } from '../runtime/display.js';
 import fs from 'fs-extra';
 import path from 'path';
+import { SQLiteChatMessageHistory } from '../runtime/memory/sqlite.js';
 async function readLastLines(filePath, n) {
     try {
         const content = await fs.readFile(filePath, 'utf8');
@@ -39,6 +40,17 @@ export function createApiRouter() {
     router.get('/config', (req, res) => {
         res.json(configManager.get());
     });
+    router.get('/stats/usage', async (req, res) => {
+        try {
+            const history = new SQLiteChatMessageHistory({ sessionId: 'api-reader' });
+            const stats = await history.getGlobalUsageStats();
+            history.close();
+            res.json(stats);
+        }
+        catch (error) {
+            res.status(500).json({ error: error.message });
+        }
+    });
     // Calculate diff between two objects
     const getDiff = (obj1, obj2, prefix = '') => {
         const changes = [];

package/dist/http/middleware/auth.js

@@ -0,0 +1,23 @@
+import { AUTH_HEADER } from '../../types/auth.js';
+import { DisplayManager } from '../../runtime/display.js';
+/**
+ * Middleware to protect API routes with a password from THE_ARCHITECT_PASS env var.
+ * If the env var is not set, authentication is skipped (open mode).
+ */
+export const authMiddleware = (req, res, next) => {
+    const architectPass = process.env.THE_ARCHITECT_PASS;
+    // If password is not configured, allow all requests
+    if (!architectPass || architectPass.trim() === '') {
+        return next();
+    }
+    const providedPass = req.headers[AUTH_HEADER];
+    if (providedPass === architectPass) {
+        return next();
+    }
+    const display = DisplayManager.getInstance();
+    display.log(`Unauthorized access attempt to ${req.path}`, { source: 'http', level: 'warning' });
+    return res.status(401).json({
+        error: 'Unauthorized',
+        code: 'UNAUTHORIZED'
+    });
+};

package/dist/http/server.js

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'url';
 import { ConfigManager } from '../config/manager.js';
 import { DisplayManager } from '../runtime/display.js';
 import { createApiRouter } from './api.js';
+import { authMiddleware } from './middleware/auth.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 export class HttpServer {
@@ -21,7 +22,7 @@ export class HttpServer {
         this.app.use(bodyParser.json());
     }
     setupRoutes() {
-        this.app.use('/api', createApiRouter());
+        this.app.use('/api', authMiddleware, createApiRouter());
         // Serve static frontend from compiled output
         const uiPath = path.resolve(__dirname, '../ui');
         this.app.use(express.static(uiPath));

package/dist/runtime/__tests__/manual_start_verify.js

@@ -12,6 +12,7 @@ const mockConfig = {
     ui: { enabled: false, port: 3333 },
     logging: { enabled: false, level: 'info', retention: '1d' },
     audio: {
+        provider: 'google',
         enabled: false,
         maxDurationSeconds: 60,
         supportedMimeTypes: ['audio/ogg']

package/dist/runtime/agent.js

@@ -1,4 +1,4 @@
-import { HumanMessage, SystemMessage, AIMessage } from "@langchain/core/messages";
+import { HumanMessage, SystemMessage } from "@langchain/core/messages";
 import { ProviderFactory } from "./providers/factory.js";
 import { ToolsFactory } from "./tools/factory.js";
 import { ConfigManager } from "../config/manager.js";
@@ -42,7 +42,7 @@ export class Agent {
             throw new ProviderError(this.config.llm.provider || 'unknown', err, "Agent initialization failed");
         }
     }
-    async chat(message) {
+    async chat(message, extraUsage) {
         if (!this.provider) {
             throw new Error("Agent not initialized. Call initialize() first.");
         }
@@ -52,7 +52,70 @@ export class Agent {
         try {
             this.display.log('Processing message...', { source: 'Agent' });
             const userMessage = new HumanMessage(message);
-            const systemMessage = new SystemMessage(`You are ${this.config.agent.name}, ${this.config.agent.personality}. You are a personal dev assistent.`);
+            // Attach extra usage (e.g. from Audio) to the user message to be persisted
+            if (extraUsage) {
+                userMessage.usage_metadata = extraUsage;
+            }
+            const systemMessage = new SystemMessage(`You are ${this.config.agent.name}, ${this.config.agent.personality},a local AI operator responsible for orchestrating tools, MCPs, and language models to solve the user’s request accurately and reliably.
+
+          Your primary responsibility is NOT to answer from memory when external tools are available.
+
+          You must follow these rules strictly:
+
+          1. Tool Evaluation First
+          Before generating a final answer, always evaluate whether any available tool or MCP is capable of providing a more accurate, up-to-date, or authoritative response.
+
+          If a tool can provide the answer, you MUST call the tool.
+
+          2. No Historical Assumptions for Dynamic Data
+          If the user asks something that:
+          - may change over time
+          - depends on system state
+          - depends on filesystem
+          - depends on external APIs
+          - was previously asked in the conversation
+
+          You MUST NOT reuse previous outputs as final truth.
+
+          Instead:
+          - Re-evaluate available tools
+          - Re-execute the relevant tool
+          - Provide a fresh result
+
+          Even if the user already asked the same question before, you must treat the request as requiring a new verification.
+
+          3. History Is Context, Not Source of Truth
+          Conversation history may help with context, but it must not replace real-time verification via tools when tools are available.
+
+          Never assume:
+          - System state
+          - File contents
+          - Database values
+          - API responses
+          based only on previous messages.
+
+          4. Tool Priority Over Language Guessing
+          If a tool can compute, fetch, inspect, or verify something, prefer tool usage over generating a speculative answer.
+
+          Never hallucinate values that could be retrieved through a tool.
+
+          5. Freshness Principle
+          Repeated user queries require fresh validation.
+          Do not respond with:
+          "As I said before..."
+          Instead, perform a new tool check if applicable.
+
+          6. Final Answer Policy
+          Only provide a direct natural language answer if:
+          - No tool is relevant
+          - Tools are unavailable
+          - The question is conceptual or explanatory
+
+          Otherwise, use tools first.
+
+          You are an operator, not a guesser.
+          Accuracy is more important than speed.
+      `);
             // Load existing history from database
             const previousMessages = await this.history.getMessages();
             const messages = [
@@ -61,12 +124,21 @@ export class Agent {
                 userMessage
             ];
             const response = await this.provider.invoke({ messages });
-            // console.log('Agent response:', response);
-            // Persist messages to database
+            // Identify new messages generated during the interaction
+            // The `messages` array passed to invoke had length `messages.length`
+            // The `response.messages` contains the full state.
+            // New messages start after the inputs.
+            const startNewMessagesIndex = messages.length;
+            const newGeneratedMessages = response.messages.slice(startNewMessagesIndex);
+            // Persist User Message first
             await this.history.addMessage(userMessage);
-            await this.history.addMessage(new AIMessage(response.messages[response.messages.length - 1].text));
+            // Persist all new intermediate tool calls and responses
+            for (const msg of newGeneratedMessages) {
+                await this.history.addMessage(msg);
+            }
             this.display.log('Response generated.', { source: 'Agent' });
-            return response.messages[response.messages.length - 1].text;
+            const lastMessage = response.messages[response.messages.length - 1];
+            return (typeof lastMessage.content === 'string') ? lastMessage.content : JSON.stringify(lastMessage.content);
         }
         catch (err) {
             throw new ProviderError(this.config.llm.provider, err, "Chat request failed");

package/dist/runtime/audio-agent.js

@@ -32,7 +32,17 @@ export class AudioAgent {
             if (!text) {
                 throw new Error('No transcription generated');
             }
-            return text;
+            // Extract usage metadata
+            const usage = response.usageMetadata;
+            const usageMetadata = {
+                input_tokens: usage?.promptTokenCount ?? 0,
+                output_tokens: usage?.candidatesTokenCount ?? 0,
+                total_tokens: usage?.totalTokenCount ?? 0,
+                input_token_details: {
+                    cache_read: usage?.cachedContentTokenCount ?? 0
+                }
+            };
+            return { text, usage: usageMetadata };
         }
         catch (error) {
             // Wrap error for clarity

package/dist/runtime/display.js

@@ -83,6 +83,18 @@ export class DisplayManager {
             else if (options.source === 'AgentAudio') {
                 color = chalk.hex('#b902b9');
             }
+            else if (options.source === 'ToolsFactory') {
+                color = chalk.hex('#806d00');
+            }
+            else if (options.source === 'MCPServer') {
+                color = chalk.hex('#be4b1d');
+            }
+            else if (options.source === 'ToolCall') {
+                color = chalk.hex('#e5ff00');
+            }
+            else if (options.source === 'Config') {
+                color = chalk.hex('#00c3ff');
+            }
             prefix = color(`[${options.source}] `);
         }
         let formattedMessage = message;

package/dist/runtime/memory/sqlite.js

@@ -1,5 +1,5 @@
 import { BaseListChatMessageHistory } from "@langchain/core/chat_history";
-import { HumanMessage, AIMessage, SystemMessage } from "@langchain/core/messages";
+import { HumanMessage, AIMessage, SystemMessage, ToolMessage } from "@langchain/core/messages";
 import Database from "better-sqlite3";
 import * as fs from "fs-extra";
 import * as path from "path";
@@ -82,37 +82,115 @@ export class SQLiteChatMessageHistory extends BaseListChatMessageHistory {
           session_id TEXT NOT NULL,
           type TEXT NOT NULL,
           content TEXT NOT NULL,
-          created_at INTEGER NOT NULL
+          created_at INTEGER NOT NULL,
+          input_tokens INTEGER,
+          output_tokens INTEGER,
+          total_tokens INTEGER,
+          cache_read_tokens INTEGER
         );
         
         CREATE INDEX IF NOT EXISTS idx_messages_session_id 
         ON messages(session_id);
       `);
+            this.migrateTable();
         }
         catch (error) {
             throw new Error(`Failed to create messages table: ${error}`);
         }
     }
+    /**
+     * Checks for missing columns and adds them if necessary.
+     */
+    migrateTable() {
+        try {
+            const tableInfo = this.db.pragma('table_info(messages)');
+            const columns = new Set(tableInfo.map(c => c.name));
+            const newColumns = [
+                'input_tokens',
+                'output_tokens',
+                'total_tokens',
+                'cache_read_tokens'
+            ];
+            for (const col of newColumns) {
+                if (!columns.has(col)) {
+                    try {
+                        this.db.exec(`ALTER TABLE messages ADD COLUMN ${col} INTEGER`);
+                    }
+                    catch (e) {
+                        // Ignore error if column already exists (race condition or check failed)
+                        console.warn(`[SQLite] Failed to add column ${col}: ${e}`);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`[SQLite] Migration check failed: ${error}`);
+        }
+    }
     /**
      * Retrieves all messages for the current session from the database.
      * @returns Promise resolving to an array of BaseMessage objects
      */
     async getMessages() {
         try {
-            // Esta query é válida para SQLite: seleciona os campos type e content da tabela messages filtrando por session_id, ordenando por id e limitando o número de resultados.
-            const stmt = this.db.prepare("SELECT type, content FROM messages WHERE session_id = ? ORDER BY id ASC LIMIT ?");
+            // Fetch new columns
+            const stmt = this.db.prepare("SELECT type, content, input_tokens, output_tokens, total_tokens, cache_read_tokens FROM messages WHERE session_id = ? ORDER BY id ASC LIMIT ?");
             const rows = stmt.all(this.sessionId, this.limit);
             return rows.map((row) => {
+                let msg;
+                // Reconstruct usage metadata if present
+                const usage_metadata = row.total_tokens != null ? {
+                    input_tokens: row.input_tokens || 0,
+                    output_tokens: row.output_tokens || 0,
+                    total_tokens: row.total_tokens || 0,
+                    input_token_details: row.cache_read_tokens ? { cache_read: row.cache_read_tokens } : undefined
+                } : undefined;
                 switch (row.type) {
                     case "human":
-                        return new HumanMessage(row.content);
+                        msg = new HumanMessage(row.content);
+                        break;
                     case "ai":
-                        return new AIMessage(row.content);
+                        try {
+                            // Attempt to parse structured content (for tool calls)
+                            const parsed = JSON.parse(row.content);
+                            if (parsed && typeof parsed === 'object' && Array.isArray(parsed.tool_calls)) {
+                                msg = new AIMessage({
+                                    content: parsed.text || "",
+                                    tool_calls: parsed.tool_calls
+                                });
+                            }
+                            else {
+                                msg = new AIMessage(row.content);
+                            }
+                        }
+                        catch {
+                            // Fallback for legacy text-only messages
+                            msg = new AIMessage(row.content);
+                        }
+                        break;
                     case "system":
-                        return new SystemMessage(row.content);
+                        msg = new SystemMessage(row.content);
+                        break;
+                    case "tool":
+                        try {
+                            const parsed = JSON.parse(row.content);
+                            msg = new ToolMessage({
+                                content: parsed.content,
+                                tool_call_id: parsed.tool_call_id || 'unknown',
+                                name: parsed.name
+                            });
+                        }
+                        catch {
+                            msg = new ToolMessage({ content: row.content, tool_call_id: 'unknown' });
+                        }
+                        break;
                     default:
                         throw new Error(`Unknown message type: ${row.type}`);
                 }
+                if (usage_metadata) {
+                    msg.usage_metadata = usage_metadata;
+                }
+                return msg;
             });
         }
         catch (error) {
@@ -139,14 +217,50 @@ export class SQLiteChatMessageHistory extends BaseListChatMessageHistory {
             else if (message instanceof SystemMessage) {
                 type = "system";
             }
+            else if (message instanceof ToolMessage) {
+                type = "tool";
+            }
             else {
                 throw new Error(`Unsupported message type: ${message.constructor.name}`);
             }
             const content = typeof message.content === "string"
                 ? message.content
                 : JSON.stringify(message.content);
-            const stmt = this.db.prepare("INSERT INTO messages (session_id, type, content, created_at) VALUES (?, ?, ?, ?)");
-            stmt.run(this.sessionId, type, content, Date.now());
+            // Extract usage metadata
+            // 1. Try generic usage_metadata (LangChain standard)
+            // 2. Try extraUsage (passed via some adapters) - attached to additional_kwargs usually, but we might pass it differently
+            // The Spec says we might pass it to chat(), but addMessage receives a BaseMessage. 
+            // So we should expect usage to be on the message object properties.
+            const anyMsg = message;
+            const usage = anyMsg.usage_metadata || anyMsg.response_metadata?.usage || anyMsg.response_metadata?.tokenUsage || anyMsg.usage;
+            const inputTokens = usage?.input_tokens ?? null;
+            const outputTokens = usage?.output_tokens ?? null;
+            const totalTokens = usage?.total_tokens ?? null;
+            const cacheReadTokens = usage?.input_token_details?.cache_read ?? usage?.cache_read_tokens ?? null;
+            // Handle special content serialization for Tools
+            let finalContent = "";
+            if (type === 'ai' && (message.tool_calls?.length ?? 0) > 0) {
+                // Serialize tool calls with content
+                finalContent = JSON.stringify({
+                    text: message.content,
+                    tool_calls: message.tool_calls
+                });
+            }
+            else if (type === 'tool') {
+                const tm = message;
+                finalContent = JSON.stringify({
+                    content: tm.content,
+                    tool_call_id: tm.tool_call_id,
+                    name: tm.name
+                });
+            }
+            else {
+                finalContent = typeof message.content === "string"
+                    ? message.content
+                    : JSON.stringify(message.content);
+            }
+            const stmt = this.db.prepare("INSERT INTO messages (session_id, type, content, created_at, input_tokens, output_tokens, total_tokens, cache_read_tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
+            stmt.run(this.sessionId, type, finalContent, Date.now(), inputTokens, outputTokens, totalTokens, cacheReadTokens);
         }
         catch (error) {
             // Check for specific SQLite errors
@@ -164,6 +278,22 @@ export class SQLiteChatMessageHistory extends BaseListChatMessageHistory {
             throw new Error(`Failed to add message: ${error}`);
         }
     }
+    /**
+     * Retrieves aggregated usage statistics for all messages in the database.
+     */
+    async getGlobalUsageStats() {
+        try {
+            const stmt = this.db.prepare("SELECT SUM(input_tokens) as totalInput, SUM(output_tokens) as totalOutput FROM messages");
+            const row = stmt.get();
+            return {
+                totalInputTokens: row.totalInput || 0,
+                totalOutputTokens: row.totalOutput || 0
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get usage stats: ${error}`);
+        }
+    }
     /**
      * Clears all messages for the current session from the database.
      */

package/dist/runtime/providers/factory.js

@@ -3,13 +3,30 @@ import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatOllama } from "@langchain/ollama";
 import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
 import { ProviderError } from "../errors.js";
-import { createAgent } from "langchain";
+import { createAgent, createMiddleware } from "langchain";
 // import { MultiServerMCPClient, } from "@langchain/mcp-adapters"; // REMOVED
 import { z } from "zod";
 import { DisplayManager } from "../display.js";
+import { ConfigQueryTool, ConfigUpdateTool, DiagnosticTool, MessageCountTool, TokenUsageTool } from "../tools/index.js";
 export class ProviderFactory {
     static async create(config, tools = []) {
         let display = DisplayManager.getInstance();
+        const toolMonitoringMiddleware = createMiddleware({
+            name: "ToolMonitoringMiddleware",
+            wrapToolCall: (request, handler) => {
+                display.log(`Executing tool: ${request.toolCall.name}`, { level: "warning", source: 'ToolCall' });
+                display.log(`Arguments: ${JSON.stringify(request.toolCall.args)}`, { level: "info", source: 'ToolCall' });
+                try {
+                    const result = handler(request);
+                    display.log("Tool completed successfully", { level: "info", source: 'ToolCall' });
+                    return result;
+                }
+                catch (e) {
+                    display.log(`Tool failed: ${e}`, { level: "error", source: 'ToolCall' });
+                    throw e;
+                }
+            },
+        });
         let model;
         const responseSchema = z.object({
             content: z.string().describe("The main response content from the agent"),
@@ -49,10 +66,18 @@ export class ProviderFactory {
                 default:
                     throw new Error(`Unsupported provider: ${config.provider}`);
             }
-            const toolsForAgent = tools;
+            const toolsForAgent = [
+                ...tools,
+                ConfigQueryTool,
+                ConfigUpdateTool,
+                DiagnosticTool,
+                MessageCountTool,
+                TokenUsageTool
+            ];
             return createAgent({
                 model: model,
                 tools: toolsForAgent,
+                middleware: [toolMonitoringMiddleware]
             });
         }
         catch (error) {

package/dist/runtime/scaffold.js

@@ -1,6 +1,7 @@
 import fs from 'fs-extra';
 import { PATHS } from '../config/paths.js';
 import { ConfigManager } from '../config/manager.js';
+import { DEFAULT_MCP_TEMPLATE } from '../types/mcp.js';
 import chalk from 'chalk';
 import ora from 'ora';
 export async function scaffold() {
@@ -22,6 +23,10 @@ export async function scaffold() {
         else {
             await configManager.load(); // Load if exists (although load handles existence check too)
         }
+        // Create mcps.json if not exists
+        if (!(await fs.pathExists(PATHS.mcps))) {
+            await fs.writeJson(PATHS.mcps, DEFAULT_MCP_TEMPLATE, { spaces: 2 });
+        }
         spinner.succeed('Morpheus environment ready at ' + chalk.cyan(PATHS.root));
     }
     catch (error) {