@mxf-dev/core 1.0.1 → 2.0.1

package/src/protocols/mcp/tools/InfrastructureTools.ts

@@ -37,6 +37,7 @@ import { promisify } from 'util';
 import crypto from 'crypto';
 import { getSecurityGuard, SecurityContext } from '../security/McpSecurityGuard.js';
 import { getConfirmationManager } from '../security/McpConfirmationManager.js';
+import { buildShellChildEnv } from '../security/McpToolPolicy.js';
 import { execute as shellExecuteHandler } from './shell/ShellExecuteHandler.js';
 import { processOutput } from './shell/LargeOutputHandler.js';
 import { BackgroundTaskManager } from '../../../services/BackgroundTaskManager.js';
@@ -58,68 +59,173 @@ const logger = new Logger('info', 'InfrastructureTools', 'server');
 const validator = createStrictValidator('InfrastructureTools');
 const execAsync = promisify(exec);
 
-// Initialize security modules
-const securityGuard = getSecurityGuard(process.cwd());
-const confirmationManager = getConfirmationManager();
+// Initialize the security guard singleton. Importing this module is what binds
+// the guard to the project root; ShellExecuteHandler and executeShellCommand
+// below then retrieve it with getSecurityGuard().
+getSecurityGuard(process.cwd());
 
 /**
- * Helper function to execute shell commands - can be used by other tools
+ * Identity of the caller running a shell command.
+ *
+ * Required — the security guard makes its decision in the context of an agent
+ * and a channel, and every shell execution is attributable. There is no default:
+ * a missing identity is a wiring bug, not something to paper over with 'system'.
+ */
+export interface ShellCommandContext {
+    agentId: AgentId;
+    channelId: ChannelId;
+    requestId: string;
+}
+
+/**
+ * Result of running a shell command through {@link executeShellCommand}.
+ */
+export interface ShellCommandResult {
+    command: string;
+    exitCode: number;
+    stdout?: string;
+    stderr?: string;
+    executionTime: number;
+    executedAt: number;
+    /** Destructive-pattern warnings raised by the security guard, if any */
+    warnings?: string[];
+}
+
+/**
+ * Run a shell command on behalf of an agent.
+ *
+ * Every tool that shells out — git, tsc, eslint, prettier, jest, mocha, vitest,
+ * find, rg, and the rollback/backup tools — goes through here, so this is where
+ * the security controls have to live:
+ *
+ *   1. The command is validated by the McpSecurityGuard, which parses compound
+ *      expressions and checks each effective command. A blocked command throws.
+ *   2. A command the guard flags for confirmation goes to the confirmation
+ *      manager, and a refusal throws.
+ *   3. The child process receives a minimal environment built by the tool policy
+ *      — not the server's, which holds JWT_SECRET, MONGODB_URI and
+ *      OPENROUTER_API_KEY.
+ *   4. Arguments are shell-escaped before they reach the shell.
+ *
+ * @param command - The base command (for example 'git')
+ * @param args - Arguments, shell-escaped before use
+ * @param options - Execution options; `context` identifies the calling agent
+ * @throws Error when the guard blocks the command or confirmation is refused
  */
 export async function executeShellCommand(
     command: string,
-    args?: string[],
-    options?: {
+    args: string[] | undefined,
+    options: {
+        /** Who is running this command. Required. */
+        context: ShellCommandContext;
         workingDirectory?: string;
         environment?: Record<string, string>;
         timeout?: number;
         captureOutput?: boolean;
     }
-): Promise<{
-    command: string;
-    exitCode: number;
-    stdout?: string;
-    stderr?: string;
-    executionTime: number;
-    executedAt: number;
-}> {
+): Promise<ShellCommandResult> {
     const startTime = Date.now();
-    
-    try {
-        // Build the full command string with properly escaped arguments
-        // Shell-escape each argument by wrapping in single quotes and escaping any embedded single quotes
-        const quotedArgs = args ? args.map(arg => `'${arg.replace(/'/g, "'\\''")}'`).join(' ') : '';
-        const fullCommand = args ? `${command} ${quotedArgs}` : command;
 
-        // Execute the command
+    validator.assertIsString(command, 'command');
+
+    const context = options?.context;
+    if (!context || typeof context.agentId !== 'string' || context.agentId.length === 0) {
+        throw new Error(
+            'executeShellCommand requires context.agentId — shell execution must be attributable to an agent.'
+        );
+    }
+    if (typeof context.channelId !== 'string' || context.channelId.length === 0) {
+        throw new Error(
+            'executeShellCommand requires context.channelId — shell execution must be attributable to a channel.'
+        );
+    }
+
+    // Build the full command string with properly escaped arguments. Each argument
+    // is wrapped in single quotes with embedded quotes escaped, so an argument can
+    // never break out into a new shell word.
+    const quotedArgs = args ? args.map(arg => `'${arg.replace(/'/g, "'\\''")}'`).join(' ') : '';
+    const fullCommand = args && args.length > 0 ? `${command} ${quotedArgs}` : command;
+
+    // ── Security validation ────────────────────────────────────────────────
+    // The guard parses compound expressions (`a && b`, `a; b`) and validates each
+    // effective command, so `git status; rm -rf /` cannot slip through on the
+    // strength of its first word.
+    const securityGuard = getSecurityGuard();
+    const securityContext: SecurityContext = {
+        agentId: context.agentId,
+        channelId: context.channelId,
+        requestId: context.requestId
+    };
+
+    const validation = securityGuard.validateCommand(fullCommand, securityContext);
+
+    if (!validation.allowed) {
+        throw new Error(
+            `Command blocked by security policy: ${validation.reason ?? 'command is not allowed'} ` +
+            `(command: ${command}, risk: ${validation.riskLevel ?? 'unknown'})`
+        );
+    }
+
+    if (validation.warnings && validation.warnings.length > 0) {
+        logger.warn(
+            `Destructive command warnings for "${fullCommand}": ${validation.warnings.join('; ')}`
+        );
+    }
+
+    if (validation.requiresConfirmation) {
+        const confirmationManager = getConfirmationManager();
+        const confirmed = await confirmationManager.requestConfirmation(
+            'command',
+            'Execute shell command',
+            {
+                command: fullCommand,
+                riskLevel: validation.riskLevel || 'medium',
+                reason: validation.reason || 'Command requires confirmation'
+            },
+            securityContext,
+            options?.timeout || 30000
+        );
+
+        if (!confirmed) {
+            throw new Error(`Command execution denied by confirmation policy: ${command}`);
+        }
+    }
+
+    // ── Execution ──────────────────────────────────────────────────────────
+    // Least privilege: the child gets PATH/HOME/locale plus whatever the caller
+    // declares, never the server's full environment.
+    const childEnv = buildShellChildEnv(options?.environment);
+
+    try {
         const result = await execAsync(fullCommand, {
             cwd: options?.workingDirectory || process.cwd(),
-            env: { ...process.env, ...options?.environment },
+            env: childEnv,
             timeout: options?.timeout || 30000,
             maxBuffer: 1024 * 1024 * 10 // 10MB buffer
         });
-        
-        const executionTime = Date.now() - startTime;
-        
+
         return {
             command: fullCommand,
             exitCode: 0,
             stdout: options?.captureOutput !== false ? result.stdout : undefined,
             stderr: options?.captureOutput !== false ? result.stderr : undefined,
-            executionTime,
-            executedAt: Date.now()
+            executionTime: Date.now() - startTime,
+            executedAt: Date.now(),
+            warnings: validation.warnings
         };
-        
+
     } catch (execError: any) {
-        const executionTime = Date.now() - startTime;
-        
-        // Handle execution errors (non-zero exit codes)
+        // A non-zero exit code is a result, not an exception — `git diff --quiet`
+        // and `grep` use exit codes to mean "no changes" / "no matches". The caller
+        // inspects exitCode. Genuine policy failures throw above, before we get here.
         return {
-            command: command,
-            exitCode: execError.code || 1,
+            command: fullCommand,
+            exitCode: execError.code ?? 1,
             stdout: options?.captureOutput !== false ? execError.stdout || '' : undefined,
             stderr: options?.captureOutput !== false ? execError.stderr || execError.message : undefined,
-            executionTime,
-            executedAt: Date.now()
+            executionTime: Date.now() - startTime,
+            executedAt: Date.now(),
+            warnings: validation.warnings
         };
     }
 }
@@ -351,11 +457,6 @@ export const shellExecTool = {
                 default: true,
                 description: 'Whether to capture command output'
             },
-            allowedCommands: {
-                type: 'array',
-                items: { type: 'string' },
-                description: 'List of allowed commands (for security)'
-            },
             description: {
                 type: 'string',
                 description: 'Human-readable summary of what this command does (for logging, audit trails, and display)'
@@ -379,6 +480,7 @@ export const shellExecTool = {
      * to BackgroundTaskManager for background execution.
      *
      * Foreground mode provides:
+     * - Security guard validation and confirmation
      * - Command semantics (exit code interpretation)
      * - Destructive command warnings
      * - Command classification (read-only, silent, etc.)
@@ -391,6 +493,10 @@ export const shellExecTool = {
      * - Ring-buffer output accumulation
      * - Throttled progress events via EventBus
      * - Query via shell_task_status tool
+     *
+     * The command allowlist is NOT an input. It comes from MXF_SHELL_ALLOWED_COMMANDS
+     * (see McpToolPolicy) — an allowlist passed as a tool argument would be supplied
+     * by the same model it is meant to restrict.
      */
     async handler(input: {
         command: string;
@@ -399,7 +505,6 @@ export const shellExecTool = {
         environment?: Record<string, string>;
         timeout?: number;
         captureOutput?: boolean;
-        allowedCommands?: string[];
         description?: string;
         runInBackground?: boolean;
         backgroundTimeout?: number;
@@ -408,14 +513,55 @@ export const shellExecTool = {
         channelId: ChannelId;
         requestId: string;
     }) {
-        // Background execution: delegate to BackgroundTaskManager
+        // Background execution: delegate to BackgroundTaskManager.
+        //
+        // The guard runs here, at the tool boundary, because BackgroundTaskManager
+        // spawns directly. Without this the background path would be an unguarded
+        // way to run exactly the commands the foreground path blocks.
         if (input.runInBackground) {
+            const securityGuard = getSecurityGuard();
+            const securityContext: SecurityContext = {
+                agentId: context.agentId,
+                channelId: context.channelId,
+                requestId: context.requestId
+            };
+
+            const validation = securityGuard.validateCommand(input.command, securityContext);
+
+            if (!validation.allowed) {
+                throw new Error(
+                    `Command blocked by security policy: ${validation.reason ?? 'command is not allowed'} ` +
+                    `(command: ${input.command}, risk: ${validation.riskLevel ?? 'unknown'})`
+                );
+            }
+
+            if (validation.requiresConfirmation) {
+                const confirmed = await getConfirmationManager().requestConfirmation(
+                    'command',
+                    'Execute background shell command',
+                    {
+                        command: input.command,
+                        riskLevel: validation.riskLevel || 'medium',
+                        reason: validation.reason || 'Command requires confirmation'
+                    },
+                    securityContext,
+                    input.timeout || 30000
+                );
+
+                if (!confirmed) {
+                    throw new Error(`Background command execution denied by confirmation policy: ${input.command}`);
+                }
+            }
+
             const btm = BackgroundTaskManager.getInstance();
             const { taskId } = await btm.startBackground(
                 input.command,
                 {
                     workingDirectory: input.workingDirectory,
-                    environment: input.environment,
+                    // Hand the child a minimal environment. BackgroundTaskManager
+                    // still merges this over process.env internally — see the
+                    // reported fix for BackgroundTaskManager.ts:190.
+                    environment: buildShellChildEnv(input.environment),
                     timeout: input.backgroundTimeout,
                     description: input.description
                 },

package/src/protocols/mcp/tools/MemorySearchTools.ts

@@ -25,6 +25,12 @@
 import { McpToolDefinition, McpToolHandlerContext, McpToolHandlerResult, McpToolResultContent } from '../McpServerTypes.js';
 import { Logger } from '../../../utils/Logger.js';
 import { MxfMeilisearchService, SearchParams } from '../../../services/MxfMeilisearchService.js';
+import { MemoryService } from '../../../services/MemoryService.js';
+import { QValueManager } from '../../../services/QValueManager.js';
+import { UtilityScorerService } from '../../../services/UtilityScorerService.js';
+import { RewardSignalProcessor } from '../../../services/RewardSignalProcessor.js';
+import { MemoryCandidate } from '../../../types/MemoryUtilityTypes.js';
+import { getCurrentMemoryPhase } from './OrparTools.js';
 import { paginationInputSchema, checkResultSize, PaginationMetadata } from '../../../utils/ToolPaginationUtils.js';
 
 const logger = new Logger('info', 'MemorySearchTools', 'server');
@@ -32,6 +38,73 @@ const logger = new Logger('info', 'MemorySearchTools', 'server');
 /**
  * Search conversation history semantically
  */
+/**
+ * Re-rank search hits by learned utility, and record what was retrieved.
+ *
+ * This is the join between semantic search and the Memory Utility Learning System.
+ * Before this existed, Q-values were computed and stored but no reachable retrieval
+ * path consulted them, so learning could not change anything an agent saw — and no
+ * production code recorded which memories a task used, so no reward was ever
+ * attributed. Both halves of the loop close here.
+ *
+ * When MULS is disabled this returns the hits untouched, so search behaves exactly as
+ * it did before.
+ */
+async function applyUtilityRanking<T extends { id: string; _rankingScore: number }>(
+    query: string,
+    hits: T[],
+    context: McpToolHandlerContext
+): Promise<T[]> {
+    const qValueManager = QValueManager.getInstance();
+    if (!qValueManager.isEnabled() || hits.length === 0) {
+        return hits;
+    }
+
+    const { agentId, channelId } = context;
+    const memoryIds = hits.map(hit => hit.id);
+
+    // Load Q-values learned in previous runs; without this every memory this process
+    // has not seen yet would score at the default and ranking would ignore all learning.
+    const memoryService = MemoryService.getInstance();
+    await memoryService.hydrateQValues(memoryIds);
+
+    const candidates: MemoryCandidate[] = hits.map(hit => ({
+        memoryId: hit.id,
+        similarity: hit._rankingScore,
+        qValue: qValueManager.getQValue(hit.id)
+    }));
+
+    // Phase-specific lambda when the agent is inside an ORPAR cycle; otherwise the
+    // configured default. We do not guess a phase.
+    const phase = agentId && channelId ? getCurrentMemoryPhase(agentId, channelId) : null;
+    const scorer = UtilityScorerService.getInstance();
+    const scoringResult = phase
+        ? scorer.scoreForPhase(query, candidates, phase)
+        : scorer.scoreMemories(query, candidates);
+
+    // Record the retrieval so RewardSignalProcessor can attribute the task's outcome to
+    // these memories when it completes.
+    if (agentId && channelId) {
+        RewardSignalProcessor.getInstance().trackAgentMemoriesUsage(
+            agentId,
+            channelId,
+            scoringResult.memories.map(m => m.memoryId),
+            phase ?? 'observation',
+            'context'
+        );
+    }
+
+    const hitsById = new Map(hits.map(hit => [hit.id, hit]));
+    const ranked: T[] = [];
+    for (const scored of scoringResult.memories) {
+        const hit = hitsById.get(scored.memoryId);
+        if (hit) {
+            ranked.push(hit);
+        }
+    }
+    return ranked;
+}
+
 export const memory_search_conversations = {
     name: 'memory_search_conversations',
     description: 'Search your entire conversation history using semantic search. Find relevant past discussions even if they happened hundreds of messages ago. Use this when you need to recall "that time we talked about X".',
@@ -159,7 +232,10 @@ export const memory_search_conversations = {
 
             const result = await meilisearch.searchConversations(searchParams);
 
-            const formattedResults = result.hits.map(hit => ({
+            // Re-rank by learned utility and record the retrieval for reward attribution.
+            const rankedHits = await applyUtilityRanking(input.query, result.hits, context);
+
+            const formattedResults = rankedHits.map(hit => ({
                 content: hit.content,
                 role: hit.role,
                 agentId: hit.agentId,

package/src/protocols/mcp/tools/OrparTools.ts

@@ -48,6 +48,9 @@ import { createStrictValidator } from '../../../utils/validation.js';
 import { AgentId } from '../../../types/Agent.js';
 import { ChannelId } from '../../../types/ChannelContext.js';
 import { Observation, Reflection } from '../../../types/ControlLoopTypes.js';
+import { OrparPhase as MemoryOrparPhase } from '../../../types/MemoryUtilityTypes.js';
+import { OrparGraphIntegration } from '../../../services/kg/OrparGraphIntegration.js';
+import { EntityExtractionService } from '../../../services/kg/EntityExtractionService.js';
 import {
     normalizeOrparParameters,
     stripUnknownParameters,
@@ -68,6 +71,75 @@ let agentDisconnectListenerRegistered = false;
  */
 export type OrparPhase = 'observe' | 'reason' | 'plan' | 'act' | 'reflect';
 
+/**
+ * The ORPAR tools name phases with verbs ('observe'), while the memory-utility system
+ * names them with nouns ('observation' — see MemoryUtilityTypes.OrparPhase). They are
+ * the same five phases in two vocabularies, which is part of why memory retrieval and
+ * phase-weighted learning were never connected. This map is the bridge.
+ */
+const TOOL_PHASE_TO_MEMORY_PHASE: Record<OrparPhase, MemoryOrparPhase> = {
+    observe: 'observation',
+    reason: 'reasoning',
+    plan: 'planning',
+    act: 'action',
+    reflect: 'reflection'
+};
+
+/**
+ * The agent's current ORPAR phase, in the vocabulary the memory-utility system uses.
+ *
+ * Returns null when the agent has no active ORPAR cycle, in which case callers should
+ * fall back to phase-independent scoring rather than guessing a phase.
+ */
+export function getCurrentMemoryPhase(agentId: string, channelId: string): MemoryOrparPhase | null {
+    const state = agentOrparStates.get(`${agentId}:${channelId}`);
+    if (!state?.currentPhase) {
+        return null;
+    }
+    return TOOL_PHASE_TO_MEMORY_PHASE[state.currentPhase];
+}
+
+/**
+ * Turn a reflection into knowledge-graph learning.
+ *
+ * Extracts the entities the agent named in its reflection, records them, and propagates
+ * the cycle's outcome as a reward across them — so entities that keep appearing in
+ * successful cycles gain utility and entities tied to failures lose it.
+ *
+ * Returns a summary of what changed. When the knowledge graph is disabled this reports
+ * zeroes rather than pretending work happened.
+ */
+async function applyReflectionToGraph(
+    agentId: AgentId,
+    channelId: ChannelId,
+    reflection: string,
+    taskSuccess: boolean
+): Promise<{ entitiesCreated: number; relationshipsCreated: number; qValuesUpdated: number }> {
+    const graph = OrparGraphIntegration.getInstance();
+    if (!graph.isEnabled()) {
+        return { entitiesCreated: 0, relationshipsCreated: 0, qValuesUpdated: 0 };
+    }
+
+    const extraction = await EntityExtractionService.getInstance().extractFromText(
+        channelId,
+        reflection
+    );
+
+    const involvedEntityIds = extraction.entities
+        .map(entity => entity.id)
+        .filter((id): id is string => Boolean(id));
+
+    return graph.applyReflectionUpdate(channelId, agentId, {
+        // Extraction already persisted the entities and relationships it found, so this
+        // update only has to attribute the outcome to them.
+        newEntities: [],
+        newRelationships: [],
+        qValueUpdates: [],
+        involvedEntityIds,
+        taskSuccess
+    });
+}
+
 /**
  * Valid phase transitions
  */
@@ -898,6 +970,20 @@ IMPORTANT - Parameter format:
             { learnings: input.learnings, phaseData: { reflection: input.reflection, learnings: input.learnings, expectationsMet: input.expectationsMet, adjustments: input.adjustments } }
         );
 
+        // Feed the reflection back into the knowledge graph.
+        //
+        // OrparGraphIntegration.applyReflectionUpdate — which creates entities from what the
+        // agent learned and propagates the task outcome as a reward across the entities
+        // involved — had no caller anywhere, so kg_get_high_utility_entities ranked entities
+        // on Q-values that were never once updated. Reflection is the only phase that knows
+        // whether expectations were met, so entity learning belongs here.
+        const knowledgeGraph = await applyReflectionToGraph(
+            agentId,
+            channelId,
+            input.reflection,
+            input.expectationsMet !== false
+        );
+
         return {
             success: true,
             phase: 'reflect',
@@ -909,6 +995,7 @@ IMPORTANT - Parameter format:
                 expectationsMet: input.expectationsMet,
                 adjustments: input.adjustments
             },
+            knowledgeGraph,
             cycleComplete: true,
             nextPhase: 'observe',
             guidance: getPhaseGuidance('reflect', context?.allowedTools || context?._allowedTools),