@mxf-dev/core 1.0.1 → 2.0.0

package/src/protocols/mcp/tools/shell/ShellExecuteHandler.ts

@@ -47,6 +47,12 @@ import {
 } from '../../../../schemas/ShellExecutionEventPayloads.js';
 import { getSecurityGuard, SecurityContext } from '../../security/McpSecurityGuard.js';
 import { getConfirmationManager } from '../../security/McpConfirmationManager.js';
+import {
+    buildShellChildEnv,
+    getShellAllowedCommands,
+    hasShellAllowlist,
+    isShellSandboxEnabled
+} from '../../security/McpToolPolicy.js';
 import { Logger } from '../../../../utils/Logger.js';
 import { createStrictValidator } from '../../../../utils/validation.js';
 import { AgentId, ChannelId } from '../../../../types/ChannelContext.js';
@@ -56,6 +62,7 @@ import { getDestructiveWarnings } from './DestructiveCommandWarnings.js';
 import { interpretExitCode } from './CommandSemantics.js';
 import { processOutput } from './LargeOutputHandler.js';
 import { extractEffectiveCommands } from './ShellCommandParser.js';
+import { executeInSandbox } from './ShellSandbox.js';
 
 const logger = new Logger('info', 'ShellExecuteHandler', 'server');
 const validator = createStrictValidator('ShellExecuteHandler');
@@ -75,14 +82,16 @@ export interface ShellExecuteInput {
     args?: string[];
     /** Working directory for command execution */
     workingDirectory?: string;
-    /** Additional environment variables merged with process.env */
+    /**
+     * Additional environment variables for the child process. Layered on top of
+     * the minimal base environment from the tool policy — the server's own
+     * environment (secrets included) is never forwarded.
+     */
     environment?: Record<string, string>;
     /** Command timeout in milliseconds (default: 30000) */
     timeout?: number;
     /** Whether to capture and return stdout/stderr (default: true) */
     captureOutput?: boolean;
-    /** Whitelist of allowed base commands (checked before execution) */
-    allowedCommands?: string[];
     /** Human-readable description of the command's purpose */
     description?: string;
 }
@@ -307,14 +316,24 @@ export async function execute(
             );
         }
 
-        // ── 6. Check allowed commands whitelist ────────────────────────────
-        if (input.allowedCommands) {
+        // ── 6. Check the configured command allowlist ──────────────────────
+        // The allowlist comes from MXF_SHELL_ALLOWED_COMMANDS, not from the tool
+        // input. An allowlist passed as an argument is chosen by the model it is
+        // meant to restrict, which makes it decorative.
+        if (hasShellAllowlist()) {
+            const allowedCommands = getShellAllowedCommands();
             // Use the parser to extract the effective command, handling env
             // prefixes (FOO=bar cmd) and wrappers (sudo cmd, timeout 5 cmd)
             const effectiveCommands = extractEffectiveCommands(input.command);
             for (const effectiveCmd of effectiveCommands) {
-                if (!input.allowedCommands.includes(effectiveCmd)) {
-                    throw new Error(`Command '${effectiveCmd}' not in allowed commands list`);
+                // extractEffectiveCommands yields the full command line; compare
+                // on the base binary so `git status` matches an allowlisted `git`.
+                const baseCommand = effectiveCmd.trim().split(/\s+/)[0];
+                if (!allowedCommands.includes(baseCommand)) {
+                    throw new Error(
+                        `Command '${baseCommand}' is not in the configured allowlist ` +
+                        `(MXF_SHELL_ALLOWED_COMMANDS). Allowed: ${allowedCommands.join(', ')}`
+                    );
                 }
             }
         }
@@ -346,19 +365,39 @@ export async function execute(
             )
         );
 
-        // ── 9. Execute the command using spawn() ───────────────────────────
+        // ── 9. Execute the command ─────────────────────────────────────────
+        // The child gets a minimal environment built by the tool policy — PATH,
+        // HOME, locale, plus anything the caller declared. The server's own
+        // environment holds JWT_SECRET, MONGODB_URI and OPENROUTER_API_KEY and is
+        // never handed to a command an agent chose.
+        const childEnv = buildShellChildEnv(input.environment) as NodeJS.ProcessEnv;
+
         const startTime = Date.now();
         let stdout = '';
         let stderr = '';
         let exitCode = 0;
 
         try {
-            const result = await spawnCommand(
-                fullCommand,
-                input.workingDirectory || process.cwd(),
-                { ...process.env, ...input.environment } as NodeJS.ProcessEnv,
-                timeout
-            );
+            // When the sandbox is enabled the command runs inside a Docker
+            // container with no network, a read-only root, dropped capabilities
+            // and resource limits. If Docker is unavailable, executeInSandbox
+            // throws — it never falls back to running on the host.
+            const result = isShellSandboxEnabled()
+                ? await executeInSandbox(
+                    fullCommand,
+                    { enabled: true },
+                    {
+                        timeout,
+                        workingDirectory: input.workingDirectory,
+                        environment: childEnv as Record<string, string>
+                    }
+                )
+                : await spawnCommand(
+                    fullCommand,
+                    input.workingDirectory || process.cwd(),
+                    childEnv,
+                    timeout
+                );
             stdout = result.stdout;
             stderr = result.stderr;
             exitCode = result.exitCode;

package/src/schemas/EventPayloadSchema.ts

@@ -41,6 +41,11 @@ import {
     PhaseChangedEventData,
     CycleCompletedEventData
 } from '../events/event-definitions/OrparMemoryEvents.js';
+import {
+    LlmBudgetEventData,
+    LLM_BUDGET_ACTOR_ID,
+    LLM_BUDGET_CHANNEL_ID
+} from '../events/event-definitions/LlmBudgetEvents.js';
 import {
     DagEvents,
     TaskDependenciesResolvedEventData,
@@ -2417,6 +2422,15 @@ export type ExternalMcpServerHealthStatusEventPayload = BaseEventPayload<Externa
  * Data for external MCP server tools discovered events
  */
 export interface ExternalMcpServerToolsDiscoveredEventData {
+    /**
+     * Which server these tools came from.
+     *
+     * Registration flows correlate on this. Without it, a caller awaiting its own server's
+     * tools had to assume any TOOLS_DISCOVERED event was its own — so two concurrent
+     * registrations could each resolve with the other's tools, and a channel-scoped
+     * registration could be completed by an unrelated external server's discovery.
+     */
+    serverId: string;
     name: string;
     version: string;
     tools: Array<{
@@ -4446,3 +4460,26 @@ export function createCompactionSummaryGeneratedPayload(
         options
     );
 }
+
+/**
+ * Build an LLM budget event payload.
+ *
+ * Budget events describe the server's own SystemLLM spend rather than an agent's activity,
+ * so they carry a fixed actor and the global channel.
+ *
+ * @param eventType - One of Events.LlmBudget
+ * @param data - Spend figures for the current budget day
+ * @returns A BaseEventPayload carrying the spend figures
+ */
+export function createLlmBudgetEventPayload(
+    eventType: string,
+    data: LlmBudgetEventData
+): BaseEventPayload<LlmBudgetEventData> {
+    return createBaseEventPayload<LlmBudgetEventData>(
+        eventType,
+        LLM_BUDGET_ACTOR_ID,
+        LLM_BUDGET_CHANNEL_ID,
+        data,
+        { source: 'SystemLlmBudgetService' }
+    );
+}

package/src/services/BackgroundTaskManager.ts

@@ -39,6 +39,7 @@ import { AgentId, ChannelId } from '../types/ChannelContext.js';
 import { Logger } from '../utils/Logger.js';
 import { Events } from '../events/EventNames.js';
 import { EventBus } from '../events/EventBus.js';
+import { buildShellChildEnv } from '../protocols/mcp/security/McpToolPolicy.js';
 import {
     createShellExecutionProgressPayload,
     createShellBackgroundStartedPayload,
@@ -142,6 +143,10 @@ export class BackgroundTaskManager {
     private constructor() {
         // Start periodic cleanup of completed tasks older than cleanupIntervalMs
         this.cleanupTimer = setInterval(() => this.cleanupCompletedTasks(), this.cleanupIntervalMs);
+        // A periodic cleanup sweep must never be the reason a process stays alive. Without
+        // unref() this timer holds Node's event loop open, so any process that merely
+        // touched this singleton — a test worker, a one-shot CLI run — could not exit.
+        this.cleanupTimer.unref();
     }
 
     /** Get the singleton instance of BackgroundTaskManager */
@@ -183,13 +188,16 @@ export class BackgroundTaskManager {
 
         const taskId = crypto.randomUUID();
 
-        // Build spawn options
+        // Build spawn options.
+        //
+        // Background tasks are spawned directly rather than through the shell tool, so this
+        // path bypassed the security guard entirely and handed every child the full server
+        // environment — JWT_SECRET, MONGODB_URI, OPENROUTER_API_KEY and the rest. Children
+        // get the same stripped environment the guarded shell path builds.
         const spawnOptions: SpawnOptions = {
             shell: true,
             cwd: options.workingDirectory || process.cwd(),
-            env: options.environment
-                ? { ...process.env, ...options.environment }
-                : process.env,
+            env: buildShellChildEnv(options.environment),
         };
 
         const childProcess = spawn(command, [], spawnOptions);

package/src/services/BrowserManager.ts

@@ -23,7 +23,41 @@
  * Adapted from MCP-WWW server for internal MXF use
  */
 
-import puppeteer, { Browser, Page } from 'puppeteer';
+// puppeteer is an OPTIONAL peer dependency: it pulls down a ~170MB Chromium,
+// which no consumer who never opens a browser should pay for. The type import is
+// erased at compile time; the runtime module is loaded on first launch and
+// throws a clear install message if it is not present.
+import type { Browser, Page, LaunchOptions } from 'puppeteer';
+
+type PuppeteerModule = {
+    launch: (options?: LaunchOptions) => Promise<Browser>;
+};
+
+let puppeteerModule: PuppeteerModule | null = null;
+
+/**
+ * Load puppeteer on demand.
+ *
+ * @throws If puppeteer is not installed. There is no headless-less fallback —
+ *         a browser tool that quietly does nothing is worse than one that fails.
+ */
+const loadPuppeteer = async (): Promise<PuppeteerModule> => {
+    if (puppeteerModule) {
+        return puppeteerModule;
+    }
+
+    try {
+        const imported = await import('puppeteer');
+        puppeteerModule = (imported.default ?? imported) as unknown as PuppeteerModule;
+        return puppeteerModule;
+    } catch (error) {
+        throw new Error(
+            'Browser automation requires the optional peer dependency "puppeteer", which is not installed. ' +
+            'Install it with `bun add puppeteer` (or `npm install puppeteer`) to use BrowserManager. ' +
+            `Underlying error: ${error instanceof Error ? error.message : String(error)}`
+        );
+    }
+};
 
 export interface BrowserInstance {
     browser: Browser;
@@ -61,6 +95,11 @@ export class BrowserManager {
         this.cleanupInterval = setInterval(() => {
             this.cleanupIdleBrowsers();
         }, 60000); // Check every minute
+        // An idle-browser sweep must not be the reason a process stays alive. Without
+        // unref() this timer holds Node's event loop open, so anything that merely
+        // constructed this manager — including WebSearchService, which builds one — could
+        // never exit. A long-running server is kept alive by its listener, not by this.
+        this.cleanupInterval.unref();
     }
 
     async getBrowser(poolName: string = 'default'): Promise<BrowserInstance> {
@@ -134,6 +173,7 @@ export class BrowserManager {
     }
 
     private async createBrowser(): Promise<BrowserInstance> {
+        const puppeteer = await loadPuppeteer();
         const browser = await puppeteer.launch({
             headless: this.config.headless ?? true,
             args: [

package/src/services/MemoryCompressor.ts

@@ -56,9 +56,20 @@ export interface CompressionResult {
   originalLength: number;
   compressedLength: number;
   compressionRatio: number;
-  method: 'semantic' | 'extractive' | 'abstractive';
+  method: CompressionMethod;
 }
 
+/**
+ * How a memory's content was compressed.
+ *
+ * Only 'extractive' exists: leading sentences are kept and the remainder is discarded.
+ * The union previously also offered 'semantic' and 'abstractive', neither of which was
+ * ever implemented — 'semantic' was reported for what was, in fact, the same extractive
+ * slicing. Listing methods the service cannot perform invites callers to trust compressed
+ * memories more than they should.
+ */
+export type CompressionMethod = 'extractive';
+
 /**
  * Consolidation options
  */
@@ -257,18 +268,14 @@ export class MemoryCompressor {
     const originalLength = content.length;
     const targetRatio = this.compressionRatios[level];
 
-    let compressedContent: string;
-    let method: 'semantic' | 'extractive' | 'abstractive';
-
-    if (this.contextCompressor && level !== CompressionLevel.Light) {
-      // Use semantic compression for moderate/heavy compression
-      compressedContent = await this.semanticCompress(content, targetRatio);
-      method = 'semantic';
-    } else {
-      // Use extractive compression for light compression
-      compressedContent = this.extractiveCompress(content, targetRatio);
-      method = 'extractive';
-    }
+    // Every compression this service performs is extractive: it keeps leading sentences
+    // and discards the rest. The previous 'semantic' branch called a semanticCompress()
+    // that never touched the contextCompressor it checked for, and did exactly the same
+    // sentence-slicing as the extractive path — it differed only in the label it reported.
+    // Callers deciding how much to trust a compressed memory were reading that label, so
+    // it now says what actually happened.
+    const compressedContent = this.extractiveCompress(content, targetRatio);
+    const method: CompressionMethod = 'extractive';
 
     const compressedLength = compressedContent.length;
     const actualRatio = compressedLength / originalLength;
@@ -283,25 +290,6 @@ export class MemoryCompressor {
     };
   }
 
-  private async semanticCompress(content: string, targetRatio: number): Promise<string> {
-    // Use ContextCompressionEngine for semantic compression
-    const targetLength = Math.ceil(content.length * targetRatio);
-
-    // Simple summarization approach
-    const words = content.split(/\s+/);
-    const targetWords = Math.ceil(words.length * targetRatio);
-
-    if (words.length <= targetWords) {
-      return content;
-    }
-
-    // Extract key sentences (simple extractive approach)
-    const sentences = content.split(/[.!?]+/).filter(s => s.trim());
-    const targetSentences = Math.max(1, Math.ceil(sentences.length * targetRatio));
-
-    return sentences.slice(0, targetSentences).join('. ') + '.';
-  }
-
   private extractiveCompress(content: string, targetRatio: number): string {
     // Simple extractive compression: keep first N% of content
     const targetLength = Math.ceil(content.length * targetRatio);

package/src/services/MemoryService.ts

@@ -30,6 +30,8 @@
  */
 
 import { Observable, of } from 'rxjs';
+import { IMemoryPersistence } from '../interfaces/IMemoryPersistence.js';
+import { MxfMeilisearchService } from './MxfMeilisearchService.js';
 import { v4 as uuidv4 } from 'uuid';
 
 import { 
@@ -160,7 +162,7 @@ export interface IEnhancedChannelMemory extends IChannelMemory {
 interface MemoryServiceConfig {
     enablePersistence?: boolean;
     cacheSize?: number;
-    persistenceService?: any; // Optional persistence service (server-only)
+    persistenceService?: IMemoryPersistence; // Optional persistence service (server-only, injected at boot)
 }
 
 /**
@@ -191,7 +193,7 @@ export class MemoryService {
     private config: MemoryServiceConfig;
 
     // Optional persistence service (server-only)
-    private persistenceService?: any;
+    private persistenceService?: IMemoryPersistence;
 
     // Validator
     private validator = createMemoryValidator('MemoryService');
@@ -612,9 +614,10 @@ export class MemoryService {
         }
 
         // If persistence service is available, try loading from MongoDB
-        if (this.persistenceService) {
+        const persistence = this.persistenceService;
+        if (persistence) {
             return new Observable<IEnhancedAgentMemory>(observer => {
-                this.persistenceService.getAgentMemory(agentId).subscribe({
+                persistence.getAgentMemory(agentId).subscribe({
                     next: (loadedMemory: any) => {
                         // Enhance the loaded memory with cognitive memory structure
                         const enhancedMemory: IEnhancedAgentMemory = {
@@ -1628,50 +1631,63 @@ export class MemoryService {
 
         const semanticSearchStart = Date.now();
 
-        // Phase A: Semantic search to get candidates
-        // This would typically integrate with MxfMeilisearchService
-        // For now, we search cognitive memory as a fallback
+        // Phase A: semantic search over the real memory index.
+        //
+        // Meilisearch is the only retrieval path agents actually use, so utility scoring
+        // has to sit on top of it to influence anything. (This previously searched an
+        // in-process cognitiveMemory Map with keyword counting, which nothing populated
+        // in production — so the whole two-phase retrieval could never return a result.)
+        const meilisearch = MxfMeilisearchService.getInstance();
+        const maxCandidates = options.maxCandidates ?? 20;
+        const similarityThreshold = options.similarityThreshold ?? 0.3;
+
+        const filters: string[] = [];
+        if (options.agentId) {
+            filters.push(`agentId = "${options.agentId}"`);
+        }
+        if (options.channelId) {
+            filters.push(`channelId = "${options.channelId}"`);
+        }
+
+        const searchResult = await meilisearch.searchConversations({
+            query: options.query,
+            filter: filters.length > 0 ? filters.join(' AND ') : undefined,
+            limit: maxCandidates,
+            hybridRatio: options.hybridRatio ?? 0.7
+        });
+
+        const semanticSearchTimeMs = Date.now() - semanticSearchStart;
+
         const candidates: MemoryCandidate[] = [];
 
-        // Search cognitive memory entries matching the query
-        const allCognitiveEntries = Array.from(this.cognitiveMemory.values());
-        for (const entry of allCognitiveEntries) {
-            // Filter by agent/channel if specified
-            if (options.agentId && entry.agentId !== options.agentId) continue;
-            if (options.channelId && entry.channelId !== options.channelId) continue;
-
-            // Simple text similarity for cognitive memory (in production, use Meilisearch)
-            const contentStr = JSON.stringify(entry.content).toLowerCase();
-            const queryWords = options.query.toLowerCase().split(/\s+/);
-            const matchCount = queryWords.filter(word => contentStr.includes(word)).length;
-            const similarity = matchCount / Math.max(queryWords.length, 1);
-
-            // Apply similarity threshold
-            const threshold = options.similarityThreshold ?? 0.3;
-            if (similarity >= threshold) {
-                // Get Q-value from manager or use default
-                const qValue = qValueManager.getQValue(entry.id);
-
-                candidates.push({
-                    memoryId: entry.id,
-                    similarity,
-                    qValue,
-                    content: entry.content,
-                    metadata: {
-                        memoryType: entry.memoryType,
-                        labels: entry.labels,
-                        createdAt: entry.createdAt
-                    }
-                });
+        for (const hit of searchResult.hits) {
+            // Meilisearch ranking scores are already normalised to 0..1.
+            const similarity = hit._rankingScore;
+            if (similarity < similarityThreshold) {
+                continue;
             }
+
+            candidates.push({
+                memoryId: hit.id,
+                similarity,
+                qValue: qValueManager.getQValue(hit.id),
+                content: hit.content,
+                metadata: {
+                    memoryType: hit.role,
+                    createdAt: new Date(hit.timestamp)
+                }
+            });
         }
 
-        const semanticSearchTimeMs = Date.now() - semanticSearchStart;
+        // Load Q-values learned in previous runs before scoring, otherwise every memory
+        // this process has not seen yet would score at the default and ranking would
+        // silently ignore everything the system has learned.
+        await this.hydrateQValues(candidates.map(c => c.memoryId));
+        for (const candidate of candidates) {
+            candidate.qValue = qValueManager.getQValue(candidate.memoryId);
+        }
 
-        // Limit candidates for Phase B
-        const maxCandidates = options.maxCandidates ?? 20;
-        candidates.sort((a, b) => b.similarity - a.similarity);
-        const topCandidates = candidates.slice(0, maxCandidates);
+        const topCandidates = candidates;
 
         // Phase B: Utility scoring
         const utilityScoringStart = Date.now();
@@ -1735,28 +1751,50 @@ export class MemoryService {
     }
 
     /**
-     * Update Q-value for a specific memory
-     * Used by QValueManager persistence callback
+     * Persist the utility subdocument for a memory.
+     *
+     * Registered as QValueManager's persistence callback at server boot, so this runs
+     * on every Q-value change and on dirty-cache eviction. It must throw on failure:
+     * swallowing the error here is what previously made the whole learning system look
+     * like it worked while retaining nothing across a restart.
      */
     public async updateMemoryUtility(memoryId: string, utilityUpdate: Partial<MemoryUtilitySubdocument>): Promise<void> {
-        // Find the memory in cognitive memory cache
-        const cognitiveEntry = this.cognitiveMemory.get(memoryId);
-        if (cognitiveEntry) {
-            // Update in-memory cache (cognitive memory doesn't have utility subdoc by default)
-            // For cognitive memory, we just rely on QValueManager's cache
+        if (!this.persistenceService) {
+            throw new Error(
+                `[MemoryService] Cannot persist utility for memory ${memoryId}: no persistence service is configured. ` +
+                `MULS requires the server to inject a persistence service at boot.`
+            );
+        }
+
+        await this.persistenceService.updateAgentMemoryUtility(memoryId, utilityUpdate);
+    }
+
+    /**
+     * Load stored Q-values for a batch of memories into the QValueManager cache.
+     *
+     * Called before utility scoring so that ranking uses values learned in previous
+     * runs rather than the default for every memory the current process has not seen.
+     * Memories with no stored utility keep the configured default.
+     */
+    public async hydrateQValues(memoryIds: string[]): Promise<void> {
+        if (!this.persistenceService || memoryIds.length === 0) {
             return;
         }
 
-        // For persistent memory, delegate to persistence service
-        if (this.persistenceService) {
-            try {
-                // Update agent memory with utility subdocument
-                // This is a partial update to just the utility field
-                await this.persistenceService.updateAgentMemoryUtility?.(memoryId, utilityUpdate);
-            } catch (error) {
-                this.logger.warn(`[MemoryService] Failed to update utility for memory ${memoryId}: ${error}`);
-            }
+        const qValueManager = QValueManager.getInstance();
+        const uncached = memoryIds.filter(id => !qValueManager.isCached(id));
+        if (uncached.length === 0) {
+            return;
         }
+
+        const utilities = await this.persistenceService.getAgentMemoryUtilities(uncached);
+        for (const [memoryId, utility] of utilities) {
+            qValueManager.setQValueInCache(memoryId, utility.qValue);
+        }
+
+        this.logger.debug(
+            `[MemoryService] Hydrated ${utilities.size}/${uncached.length} Q-values from persistence`
+        );
     }
 
     /**