@mxf-dev/core 1.0.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/adapters/FrameworkAdapter.js +1 -1
- package/dist/adapters/FrameworkAdapter.js.map +1 -1
- package/dist/config/ConfigManager.d.ts +9 -8
- package/dist/config/ConfigManager.d.ts.map +1 -1
- package/dist/config/ConfigManager.js +47 -17
- package/dist/config/ConfigManager.js.map +1 -1
- package/dist/config/memory-strata.config.d.ts +25 -0
- package/dist/config/memory-strata.config.d.ts.map +1 -0
- package/dist/config/memory-strata.config.js +107 -0
- package/dist/config/memory-strata.config.js.map +1 -0
- package/dist/constants/ToolNames.d.ts +1 -29
- package/dist/constants/ToolNames.d.ts.map +1 -1
- package/dist/constants/ToolNames.js +4 -17
- package/dist/constants/ToolNames.js.map +1 -1
- package/dist/database/DatabaseAdapterFactory.d.ts +14 -2
- package/dist/database/DatabaseAdapterFactory.d.ts.map +1 -1
- package/dist/database/DatabaseAdapterFactory.js +21 -6
- package/dist/database/DatabaseAdapterFactory.js.map +1 -1
- package/dist/database/adapters/mongodb/MongoBaseRepository.d.ts +7 -1
- package/dist/database/adapters/mongodb/MongoBaseRepository.d.ts.map +1 -1
- package/dist/database/adapters/mongodb/MongoBaseRepository.js +12 -2
- package/dist/database/adapters/mongodb/MongoBaseRepository.js.map +1 -1
- package/dist/events/ClientEventBus.d.ts +16 -77
- package/dist/events/ClientEventBus.d.ts.map +1 -1
- package/dist/events/ClientEventBus.js +38 -311
- package/dist/events/ClientEventBus.js.map +1 -1
- package/dist/events/EventBus.d.ts +1 -1
- package/dist/events/EventBus.d.ts.map +1 -1
- package/dist/events/EventBus.js +5 -4
- package/dist/events/EventBus.js.map +1 -1
- package/dist/events/EventBusBase.d.ts +139 -34
- package/dist/events/EventBusBase.d.ts.map +1 -1
- package/dist/events/EventBusBase.js +366 -88
- package/dist/events/EventBusBase.js.map +1 -1
- package/dist/events/EventNames.d.ts +10 -1
- package/dist/events/EventNames.d.ts.map +1 -1
- package/dist/events/EventNames.js +3 -0
- package/dist/events/EventNames.js.map +1 -1
- package/dist/events/PublicEvents.d.ts +6 -2
- package/dist/events/PublicEvents.d.ts.map +1 -1
- package/dist/events/PublicEvents.js +7 -4
- package/dist/events/PublicEvents.js.map +1 -1
- package/dist/events/ServerEventBus.d.ts +23 -112
- package/dist/events/ServerEventBus.d.ts.map +1 -1
- package/dist/events/ServerEventBus.js +58 -323
- package/dist/events/ServerEventBus.js.map +1 -1
- package/dist/events/event-definitions/LlmBudgetEvents.d.ts +68 -0
- package/dist/events/event-definitions/LlmBudgetEvents.d.ts.map +1 -0
- package/dist/events/event-definitions/LlmBudgetEvents.js +35 -0
- package/dist/events/event-definitions/LlmBudgetEvents.js.map +1 -0
- package/dist/interfaces/IMemoryPersistence.d.ts +41 -0
- package/dist/interfaces/IMemoryPersistence.d.ts.map +1 -0
- package/dist/interfaces/IMemoryPersistence.js +15 -0
- package/dist/interfaces/IMemoryPersistence.js.map +1 -0
- package/dist/middleware/MxpMiddleware.d.ts +8 -1
- package/dist/middleware/MxpMiddleware.d.ts.map +1 -1
- package/dist/middleware/MxpMiddleware.js +42 -20
- package/dist/middleware/MxpMiddleware.js.map +1 -1
- package/dist/models/channelKey.d.ts +1 -0
- package/dist/models/channelKey.d.ts.map +1 -1
- package/dist/models/channelKey.js +5 -0
- package/dist/models/channelKey.js.map +1 -1
- package/dist/models/memoryUtility.d.ts +39 -0
- package/dist/models/memoryUtility.d.ts.map +1 -0
- package/dist/models/memoryUtility.js +39 -0
- package/dist/models/memoryUtility.js.map +1 -0
- package/dist/models/user.d.ts.map +1 -1
- package/dist/models/user.js +23 -1
- package/dist/models/user.js.map +1 -1
- package/dist/mxp/BinaryProtocolLayer.d.ts +24 -8
- package/dist/mxp/BinaryProtocolLayer.d.ts.map +1 -1
- package/dist/mxp/BinaryProtocolLayer.js +90 -103
- package/dist/mxp/BinaryProtocolLayer.js.map +1 -1
- package/dist/mxp/ContextCompressionEngine.d.ts +57 -10
- package/dist/mxp/ContextCompressionEngine.d.ts.map +1 -1
- package/dist/mxp/ContextCompressionEngine.js +170 -128
- package/dist/mxp/ContextCompressionEngine.js.map +1 -1
- package/dist/mxp/MxpTokenOptimizer.d.ts +7 -3
- package/dist/mxp/MxpTokenOptimizer.d.ts.map +1 -1
- package/dist/mxp/MxpTokenOptimizer.js +8 -5
- package/dist/mxp/MxpTokenOptimizer.js.map +1 -1
- package/dist/prompts/MxfAgentSystemPrompt.d.ts +1 -1
- package/dist/prompts/MxfAgentSystemPrompt.d.ts.map +1 -1
- package/dist/prompts/MxfAgentSystemPrompt.js +47 -57
- package/dist/prompts/MxfAgentSystemPrompt.js.map +1 -1
- package/dist/protocols/mcp/ToolError.d.ts +123 -0
- package/dist/protocols/mcp/ToolError.d.ts.map +1 -0
- package/dist/protocols/mcp/ToolError.js +108 -0
- package/dist/protocols/mcp/ToolError.js.map +1 -0
- package/dist/protocols/mcp/defineTool.d.ts +133 -0
- package/dist/protocols/mcp/defineTool.d.ts.map +1 -0
- package/dist/protocols/mcp/defineTool.js +236 -0
- package/dist/protocols/mcp/defineTool.js.map +1 -0
- package/dist/protocols/mcp/security/HttpTargetGuard.d.ts +38 -0
- package/dist/protocols/mcp/security/HttpTargetGuard.d.ts.map +1 -0
- package/dist/protocols/mcp/security/HttpTargetGuard.js +227 -0
- package/dist/protocols/mcp/security/HttpTargetGuard.js.map +1 -0
- package/dist/protocols/mcp/security/McpSecurityGuard.d.ts +9 -1
- package/dist/protocols/mcp/security/McpSecurityGuard.d.ts.map +1 -1
- package/dist/protocols/mcp/security/McpSecurityGuard.js +10 -2
- package/dist/protocols/mcp/security/McpSecurityGuard.js.map +1 -1
- package/dist/protocols/mcp/security/McpToolPolicy.d.ts +81 -0
- package/dist/protocols/mcp/security/McpToolPolicy.d.ts.map +1 -0
- package/dist/protocols/mcp/security/McpToolPolicy.js +202 -0
- package/dist/protocols/mcp/security/McpToolPolicy.js.map +1 -0
- package/dist/protocols/mcp/services/ExternalMcpServerManager.d.ts +51 -18
- package/dist/protocols/mcp/services/ExternalMcpServerManager.d.ts.map +1 -1
- package/dist/protocols/mcp/services/ExternalMcpServerManager.js +256 -283
- package/dist/protocols/mcp/services/ExternalMcpServerManager.js.map +1 -1
- package/dist/protocols/mcp/services/ExternalServerConfigs.d.ts +39 -6
- package/dist/protocols/mcp/services/ExternalServerConfigs.d.ts.map +1 -1
- package/dist/protocols/mcp/services/ExternalServerConfigs.js +76 -18
- package/dist/protocols/mcp/services/ExternalServerConfigs.js.map +1 -1
- package/dist/protocols/mcp/tools/AnalyticsTools.js +1 -1
- package/dist/protocols/mcp/tools/AnalyticsTools.js.map +1 -1
- package/dist/protocols/mcp/tools/CodeAnalysisTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/CodeAnalysisTools.js +15 -0
- package/dist/protocols/mcp/tools/CodeAnalysisTools.js.map +1 -1
- package/dist/protocols/mcp/tools/EffectivenessTools.js +3 -3
- package/dist/protocols/mcp/tools/EffectivenessTools.js.map +1 -1
- package/dist/protocols/mcp/tools/GitTools.d.ts +9 -369
- package/dist/protocols/mcp/tools/GitTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/GitTools.js +222 -517
- package/dist/protocols/mcp/tools/GitTools.js.map +1 -1
- package/dist/protocols/mcp/tools/InfrastructureTools.d.ts +56 -24
- package/dist/protocols/mcp/tools/InfrastructureTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/InfrastructureTools.js +117 -26
- package/dist/protocols/mcp/tools/InfrastructureTools.js.map +1 -1
- package/dist/protocols/mcp/tools/MemorySearchTools.d.ts +0 -3
- package/dist/protocols/mcp/tools/MemorySearchTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/MemorySearchTools.js +58 -1
- package/dist/protocols/mcp/tools/MemorySearchTools.js.map +1 -1
- package/dist/protocols/mcp/tools/OrparTools.d.ts +8 -0
- package/dist/protocols/mcp/tools/OrparTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/OrparTools.js +66 -0
- package/dist/protocols/mcp/tools/OrparTools.js.map +1 -1
- package/dist/protocols/mcp/tools/SafetyTools.d.ts +7 -525
- package/dist/protocols/mcp/tools/SafetyTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/SafetyTools.js +349 -514
- package/dist/protocols/mcp/tools/SafetyTools.js.map +1 -1
- package/dist/protocols/mcp/tools/TaskPlanningTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/TaskPlanningTools.js +56 -26
- package/dist/protocols/mcp/tools/TaskPlanningTools.js.map +1 -1
- package/dist/protocols/mcp/tools/TestTools.d.ts +15 -376
- package/dist/protocols/mcp/tools/TestTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/TestTools.js +241 -363
- package/dist/protocols/mcp/tools/TestTools.js.map +1 -1
- package/dist/protocols/mcp/tools/TypeScriptTools.d.ts +6 -276
- package/dist/protocols/mcp/tools/TypeScriptTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/TypeScriptTools.js +223 -337
- package/dist/protocols/mcp/tools/TypeScriptTools.js.map +1 -1
- package/dist/protocols/mcp/tools/WebTools.d.ts +15 -728
- package/dist/protocols/mcp/tools/WebTools.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/WebTools.js +151 -257
- package/dist/protocols/mcp/tools/WebTools.js.map +1 -1
- package/dist/protocols/mcp/tools/shell/ShellExecuteHandler.d.ts +5 -3
- package/dist/protocols/mcp/tools/shell/ShellExecuteHandler.d.ts.map +1 -1
- package/dist/protocols/mcp/tools/shell/ShellExecuteHandler.js +31 -6
- package/dist/protocols/mcp/tools/shell/ShellExecuteHandler.js.map +1 -1
- package/dist/schemas/EventPayloadSchema.d.ts +21 -0
- package/dist/schemas/EventPayloadSchema.d.ts.map +1 -1
- package/dist/schemas/EventPayloadSchema.js +14 -0
- package/dist/schemas/EventPayloadSchema.js.map +1 -1
- package/dist/services/BackgroundTaskManager.d.ts.map +1 -1
- package/dist/services/BackgroundTaskManager.js +12 -4
- package/dist/services/BackgroundTaskManager.js.map +1 -1
- package/dist/services/BrowserManager.d.ts +1 -1
- package/dist/services/BrowserManager.d.ts.map +1 -1
- package/dist/services/BrowserManager.js +26 -3
- package/dist/services/BrowserManager.js.map +1 -1
- package/dist/services/MemoryCompressor.d.ts +11 -2
- package/dist/services/MemoryCompressor.d.ts.map +1 -1
- package/dist/services/MemoryCompressor.js +8 -26
- package/dist/services/MemoryCompressor.js.map +1 -1
- package/dist/services/MemoryService.d.ts +16 -3
- package/dist/services/MemoryService.d.ts.map +1 -1
- package/dist/services/MemoryService.js +79 -55
- package/dist/services/MemoryService.js.map +1 -1
- package/dist/services/MxfMLService.d.ts +5 -0
- package/dist/services/MxfMLService.d.ts.map +1 -1
- package/dist/services/MxfMLService.js +32 -6
- package/dist/services/MxfMLService.js.map +1 -1
- package/dist/services/PredictiveAnalyticsService.d.ts +45 -9
- package/dist/services/PredictiveAnalyticsService.d.ts.map +1 -1
- package/dist/services/PredictiveAnalyticsService.js +82 -44
- package/dist/services/PredictiveAnalyticsService.js.map +1 -1
- package/dist/services/QValueManager.d.ts +7 -0
- package/dist/services/QValueManager.d.ts.map +1 -1
- package/dist/services/QValueManager.js +9 -0
- package/dist/services/QValueManager.js.map +1 -1
- package/dist/services/RewardSignalProcessor.d.ts +38 -1
- package/dist/services/RewardSignalProcessor.d.ts.map +1 -1
- package/dist/services/RewardSignalProcessor.js +109 -7
- package/dist/services/RewardSignalProcessor.js.map +1 -1
- package/dist/services/SharedStateService.d.ts +16 -2
- package/dist/services/SharedStateService.d.ts.map +1 -1
- package/dist/services/SharedStateService.js +44 -3
- package/dist/services/SharedStateService.js.map +1 -1
- package/dist/services/TaskEffectivenessService.d.ts +34 -18
- package/dist/services/TaskEffectivenessService.d.ts.map +1 -1
- package/dist/services/TaskEffectivenessService.js +91 -155
- package/dist/services/TaskEffectivenessService.js.map +1 -1
- package/dist/services/WebSearchService.d.ts.map +1 -1
- package/dist/services/WebSearchService.js.map +1 -1
- package/dist/services/dag/TaskDagService.d.ts.map +1 -1
- package/dist/services/dag/TaskDagService.js +2 -0
- package/dist/services/dag/TaskDagService.js.map +1 -1
- package/dist/services/orpar-memory/OrparMemoryCoordinator.d.ts +7 -0
- package/dist/services/orpar-memory/OrparMemoryCoordinator.d.ts.map +1 -1
- package/dist/services/orpar-memory/OrparMemoryCoordinator.js +71 -10
- package/dist/services/orpar-memory/OrparMemoryCoordinator.js.map +1 -1
- package/dist/types/MemoryUtilityTypes.d.ts +2 -0
- package/dist/types/MemoryUtilityTypes.d.ts.map +1 -1
- package/dist/types/MemoryUtilityTypes.js.map +1 -1
- package/dist/utils/Logger.d.ts +43 -3
- package/dist/utils/Logger.d.ts.map +1 -1
- package/dist/utils/Logger.js +78 -36
- package/dist/utils/Logger.js.map +1 -1
- package/dist/utils/MxpEncryption.d.ts +19 -6
- package/dist/utils/MxpEncryption.d.ts.map +1 -1
- package/dist/utils/MxpEncryption.js +48 -16
- package/dist/utils/MxpEncryption.js.map +1 -1
- package/package.json +19 -4
- package/src/adapters/FrameworkAdapter.ts +1 -1
- package/src/config/ConfigManager.ts +52 -19
- package/src/config/memory-strata.config.ts +136 -0
- package/src/constants/ToolNames.ts +4 -18
- package/src/database/DatabaseAdapterFactory.ts +23 -6
- package/src/database/adapters/mongodb/MongoBaseRepository.ts +16 -2
- package/src/events/ClientEventBus.ts +68 -403
- package/src/events/EventBus.ts +7 -8
- package/src/events/EventBusBase.ts +479 -116
- package/src/events/EventNames.ts +6 -0
- package/src/events/PublicEvents.ts +7 -4
- package/src/events/ServerEventBus.ts +78 -473
- package/src/events/event-definitions/LlmBudgetEvents.ts +74 -0
- package/src/interfaces/IMemoryPersistence.ts +51 -0
- package/src/middleware/MxpMiddleware.ts +51 -24
- package/src/models/channelKey.ts +18 -4
- package/src/models/memoryUtility.ts +64 -0
- package/src/models/user.ts +36 -4
- package/src/mxp/BinaryProtocolLayer.ts +108 -124
- package/src/mxp/ContextCompressionEngine.ts +219 -161
- package/src/mxp/MxpTokenOptimizer.ts +10 -10
- package/src/prompts/MxfAgentSystemPrompt.ts +45 -55
- package/src/protocols/mcp/ToolError.ts +201 -0
- package/src/protocols/mcp/defineTool.ts +379 -0
- package/src/protocols/mcp/security/HttpTargetGuard.ts +267 -0
- package/src/protocols/mcp/security/McpSecurityGuard.ts +10 -3
- package/src/protocols/mcp/security/McpToolPolicy.ts +221 -0
- package/src/protocols/mcp/services/ExternalMcpServerManager.ts +340 -322
- package/src/protocols/mcp/services/ExternalServerConfigs.ts +82 -19
- package/src/protocols/mcp/tools/AnalyticsTools.ts +1 -1
- package/src/protocols/mcp/tools/CodeAnalysisTools.ts +15 -0
- package/src/protocols/mcp/tools/EffectivenessTools.ts +3 -3
- package/src/protocols/mcp/tools/GitTools.ts +290 -569
- package/src/protocols/mcp/tools/InfrastructureTools.ts +189 -43
- package/src/protocols/mcp/tools/MemorySearchTools.ts +77 -1
- package/src/protocols/mcp/tools/OrparTools.ts +87 -0
- package/src/protocols/mcp/tools/SafetyTools.ts +501 -579
- package/src/protocols/mcp/tools/TaskPlanningTools.ts +81 -39
- package/src/protocols/mcp/tools/TestTools.ts +339 -419
- package/src/protocols/mcp/tools/TypeScriptTools.ts +316 -376
- package/src/protocols/mcp/tools/WebTools.ts +209 -307
- package/src/protocols/mcp/tools/shell/ShellExecuteHandler.ts +53 -14
- package/src/schemas/EventPayloadSchema.ts +37 -0
- package/src/services/BackgroundTaskManager.ts +12 -4
- package/src/services/BrowserManager.ts +41 -1
- package/src/services/MemoryCompressor.ts +20 -32
- package/src/services/MemoryService.ts +96 -58
- package/src/services/MxfMLService.ts +41 -7
- package/src/services/PredictiveAnalyticsService.ts +101 -51
- package/src/services/QValueManager.ts +10 -0
- package/src/services/RewardSignalProcessor.ts +138 -9
- package/src/services/SharedStateService.ts +56 -8
- package/src/services/TaskEffectivenessService.ts +114 -187
- package/src/services/WebSearchService.ts +2 -1
- package/src/services/dag/TaskDagService.ts +2 -0
- package/src/services/orpar-memory/OrparMemoryCoordinator.ts +86 -10
- package/src/types/MemoryUtilityTypes.ts +2 -0
- package/src/utils/Logger.ts +116 -40
- package/src/utils/MxpEncryption.ts +69 -34
- package/dist/mxp/MxpEventForwardingEnhancer.d.ts +0 -134
- package/dist/mxp/MxpEventForwardingEnhancer.d.ts.map +0 -1
- package/dist/mxp/MxpEventForwardingEnhancer.js +0 -361
- package/dist/mxp/MxpEventForwardingEnhancer.js.map +0 -1
- package/dist/protocols/mcp/services/ToolSchemaEnhancements.d.ts +0 -56
- package/dist/protocols/mcp/services/ToolSchemaEnhancements.d.ts.map +0 -1
- package/dist/protocols/mcp/services/ToolSchemaEnhancements.js +0 -57
- package/dist/protocols/mcp/services/ToolSchemaEnhancements.js.map +0 -1
- package/dist/protocols/mcp/tools/ControlLoopLifecycle.d.ts +0 -270
- package/dist/protocols/mcp/tools/ControlLoopLifecycle.d.ts.map +0 -1
- package/dist/protocols/mcp/tools/ControlLoopLifecycle.js +0 -256
- package/dist/protocols/mcp/tools/ControlLoopLifecycle.js.map +0 -1
- package/dist/protocols/mcp/tools/LspTools.d.ts +0 -495
- package/dist/protocols/mcp/tools/LspTools.d.ts.map +0 -1
- package/dist/protocols/mcp/tools/LspTools.js +0 -469
- package/dist/protocols/mcp/tools/LspTools.js.map +0 -1
- package/src/mxp/MxpEventForwardingEnhancer.ts +0 -471
- package/src/protocols/mcp/services/ToolSchemaEnhancements.ts +0 -92
- package/src/protocols/mcp/tools/ControlLoopLifecycle.ts +0 -302
- package/src/protocols/mcp/tools/LspTools.ts +0 -476
|
@@ -47,6 +47,12 @@ import {
|
|
|
47
47
|
} from '../../../../schemas/ShellExecutionEventPayloads.js';
|
|
48
48
|
import { getSecurityGuard, SecurityContext } from '../../security/McpSecurityGuard.js';
|
|
49
49
|
import { getConfirmationManager } from '../../security/McpConfirmationManager.js';
|
|
50
|
+
import {
|
|
51
|
+
buildShellChildEnv,
|
|
52
|
+
getShellAllowedCommands,
|
|
53
|
+
hasShellAllowlist,
|
|
54
|
+
isShellSandboxEnabled
|
|
55
|
+
} from '../../security/McpToolPolicy.js';
|
|
50
56
|
import { Logger } from '../../../../utils/Logger.js';
|
|
51
57
|
import { createStrictValidator } from '../../../../utils/validation.js';
|
|
52
58
|
import { AgentId, ChannelId } from '../../../../types/ChannelContext.js';
|
|
@@ -56,6 +62,7 @@ import { getDestructiveWarnings } from './DestructiveCommandWarnings.js';
|
|
|
56
62
|
import { interpretExitCode } from './CommandSemantics.js';
|
|
57
63
|
import { processOutput } from './LargeOutputHandler.js';
|
|
58
64
|
import { extractEffectiveCommands } from './ShellCommandParser.js';
|
|
65
|
+
import { executeInSandbox } from './ShellSandbox.js';
|
|
59
66
|
|
|
60
67
|
const logger = new Logger('info', 'ShellExecuteHandler', 'server');
|
|
61
68
|
const validator = createStrictValidator('ShellExecuteHandler');
|
|
@@ -75,14 +82,16 @@ export interface ShellExecuteInput {
|
|
|
75
82
|
args?: string[];
|
|
76
83
|
/** Working directory for command execution */
|
|
77
84
|
workingDirectory?: string;
|
|
78
|
-
/**
|
|
85
|
+
/**
|
|
86
|
+
* Additional environment variables for the child process. Layered on top of
|
|
87
|
+
* the minimal base environment from the tool policy — the server's own
|
|
88
|
+
* environment (secrets included) is never forwarded.
|
|
89
|
+
*/
|
|
79
90
|
environment?: Record<string, string>;
|
|
80
91
|
/** Command timeout in milliseconds (default: 30000) */
|
|
81
92
|
timeout?: number;
|
|
82
93
|
/** Whether to capture and return stdout/stderr (default: true) */
|
|
83
94
|
captureOutput?: boolean;
|
|
84
|
-
/** Whitelist of allowed base commands (checked before execution) */
|
|
85
|
-
allowedCommands?: string[];
|
|
86
95
|
/** Human-readable description of the command's purpose */
|
|
87
96
|
description?: string;
|
|
88
97
|
}
|
|
@@ -307,14 +316,24 @@ export async function execute(
|
|
|
307
316
|
);
|
|
308
317
|
}
|
|
309
318
|
|
|
310
|
-
// ── 6. Check
|
|
311
|
-
|
|
319
|
+
// ── 6. Check the configured command allowlist ──────────────────────
|
|
320
|
+
// The allowlist comes from MXF_SHELL_ALLOWED_COMMANDS, not from the tool
|
|
321
|
+
// input. An allowlist passed as an argument is chosen by the model it is
|
|
322
|
+
// meant to restrict, which makes it decorative.
|
|
323
|
+
if (hasShellAllowlist()) {
|
|
324
|
+
const allowedCommands = getShellAllowedCommands();
|
|
312
325
|
// Use the parser to extract the effective command, handling env
|
|
313
326
|
// prefixes (FOO=bar cmd) and wrappers (sudo cmd, timeout 5 cmd)
|
|
314
327
|
const effectiveCommands = extractEffectiveCommands(input.command);
|
|
315
328
|
for (const effectiveCmd of effectiveCommands) {
|
|
316
|
-
|
|
317
|
-
|
|
329
|
+
// extractEffectiveCommands yields the full command line; compare
|
|
330
|
+
// on the base binary so `git status` matches an allowlisted `git`.
|
|
331
|
+
const baseCommand = effectiveCmd.trim().split(/\s+/)[0];
|
|
332
|
+
if (!allowedCommands.includes(baseCommand)) {
|
|
333
|
+
throw new Error(
|
|
334
|
+
`Command '${baseCommand}' is not in the configured allowlist ` +
|
|
335
|
+
`(MXF_SHELL_ALLOWED_COMMANDS). Allowed: ${allowedCommands.join(', ')}`
|
|
336
|
+
);
|
|
318
337
|
}
|
|
319
338
|
}
|
|
320
339
|
}
|
|
@@ -346,19 +365,39 @@ export async function execute(
|
|
|
346
365
|
)
|
|
347
366
|
);
|
|
348
367
|
|
|
349
|
-
// ── 9. Execute the command
|
|
368
|
+
// ── 9. Execute the command ─────────────────────────────────────────
|
|
369
|
+
// The child gets a minimal environment built by the tool policy — PATH,
|
|
370
|
+
// HOME, locale, plus anything the caller declared. The server's own
|
|
371
|
+
// environment holds JWT_SECRET, MONGODB_URI and OPENROUTER_API_KEY and is
|
|
372
|
+
// never handed to a command an agent chose.
|
|
373
|
+
const childEnv = buildShellChildEnv(input.environment) as NodeJS.ProcessEnv;
|
|
374
|
+
|
|
350
375
|
const startTime = Date.now();
|
|
351
376
|
let stdout = '';
|
|
352
377
|
let stderr = '';
|
|
353
378
|
let exitCode = 0;
|
|
354
379
|
|
|
355
380
|
try {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
381
|
+
// When the sandbox is enabled the command runs inside a Docker
|
|
382
|
+
// container with no network, a read-only root, dropped capabilities
|
|
383
|
+
// and resource limits. If Docker is unavailable, executeInSandbox
|
|
384
|
+
// throws — it never falls back to running on the host.
|
|
385
|
+
const result = isShellSandboxEnabled()
|
|
386
|
+
? await executeInSandbox(
|
|
387
|
+
fullCommand,
|
|
388
|
+
{ enabled: true },
|
|
389
|
+
{
|
|
390
|
+
timeout,
|
|
391
|
+
workingDirectory: input.workingDirectory,
|
|
392
|
+
environment: childEnv as Record<string, string>
|
|
393
|
+
}
|
|
394
|
+
)
|
|
395
|
+
: await spawnCommand(
|
|
396
|
+
fullCommand,
|
|
397
|
+
input.workingDirectory || process.cwd(),
|
|
398
|
+
childEnv,
|
|
399
|
+
timeout
|
|
400
|
+
);
|
|
362
401
|
stdout = result.stdout;
|
|
363
402
|
stderr = result.stderr;
|
|
364
403
|
exitCode = result.exitCode;
|
|
@@ -41,6 +41,11 @@ import {
|
|
|
41
41
|
PhaseChangedEventData,
|
|
42
42
|
CycleCompletedEventData
|
|
43
43
|
} from '../events/event-definitions/OrparMemoryEvents.js';
|
|
44
|
+
import {
|
|
45
|
+
LlmBudgetEventData,
|
|
46
|
+
LLM_BUDGET_ACTOR_ID,
|
|
47
|
+
LLM_BUDGET_CHANNEL_ID
|
|
48
|
+
} from '../events/event-definitions/LlmBudgetEvents.js';
|
|
44
49
|
import {
|
|
45
50
|
DagEvents,
|
|
46
51
|
TaskDependenciesResolvedEventData,
|
|
@@ -2417,6 +2422,15 @@ export type ExternalMcpServerHealthStatusEventPayload = BaseEventPayload<Externa
|
|
|
2417
2422
|
* Data for external MCP server tools discovered events
|
|
2418
2423
|
*/
|
|
2419
2424
|
export interface ExternalMcpServerToolsDiscoveredEventData {
|
|
2425
|
+
/**
|
|
2426
|
+
* Which server these tools came from.
|
|
2427
|
+
*
|
|
2428
|
+
* Registration flows correlate on this. Without it, a caller awaiting its own server's
|
|
2429
|
+
* tools had to assume any TOOLS_DISCOVERED event was its own — so two concurrent
|
|
2430
|
+
* registrations could each resolve with the other's tools, and a channel-scoped
|
|
2431
|
+
* registration could be completed by an unrelated external server's discovery.
|
|
2432
|
+
*/
|
|
2433
|
+
serverId: string;
|
|
2420
2434
|
name: string;
|
|
2421
2435
|
version: string;
|
|
2422
2436
|
tools: Array<{
|
|
@@ -4446,3 +4460,26 @@ export function createCompactionSummaryGeneratedPayload(
|
|
|
4446
4460
|
options
|
|
4447
4461
|
);
|
|
4448
4462
|
}
|
|
4463
|
+
|
|
4464
|
+
/**
|
|
4465
|
+
* Build an LLM budget event payload.
|
|
4466
|
+
*
|
|
4467
|
+
* Budget events describe the server's own SystemLLM spend rather than an agent's activity,
|
|
4468
|
+
* so they carry a fixed actor and the global channel.
|
|
4469
|
+
*
|
|
4470
|
+
* @param eventType - One of Events.LlmBudget
|
|
4471
|
+
* @param data - Spend figures for the current budget day
|
|
4472
|
+
* @returns A BaseEventPayload carrying the spend figures
|
|
4473
|
+
*/
|
|
4474
|
+
export function createLlmBudgetEventPayload(
|
|
4475
|
+
eventType: string,
|
|
4476
|
+
data: LlmBudgetEventData
|
|
4477
|
+
): BaseEventPayload<LlmBudgetEventData> {
|
|
4478
|
+
return createBaseEventPayload<LlmBudgetEventData>(
|
|
4479
|
+
eventType,
|
|
4480
|
+
LLM_BUDGET_ACTOR_ID,
|
|
4481
|
+
LLM_BUDGET_CHANNEL_ID,
|
|
4482
|
+
data,
|
|
4483
|
+
{ source: 'SystemLlmBudgetService' }
|
|
4484
|
+
);
|
|
4485
|
+
}
|
|
@@ -39,6 +39,7 @@ import { AgentId, ChannelId } from '../types/ChannelContext.js';
|
|
|
39
39
|
import { Logger } from '../utils/Logger.js';
|
|
40
40
|
import { Events } from '../events/EventNames.js';
|
|
41
41
|
import { EventBus } from '../events/EventBus.js';
|
|
42
|
+
import { buildShellChildEnv } from '../protocols/mcp/security/McpToolPolicy.js';
|
|
42
43
|
import {
|
|
43
44
|
createShellExecutionProgressPayload,
|
|
44
45
|
createShellBackgroundStartedPayload,
|
|
@@ -142,6 +143,10 @@ export class BackgroundTaskManager {
|
|
|
142
143
|
private constructor() {
|
|
143
144
|
// Start periodic cleanup of completed tasks older than cleanupIntervalMs
|
|
144
145
|
this.cleanupTimer = setInterval(() => this.cleanupCompletedTasks(), this.cleanupIntervalMs);
|
|
146
|
+
// A periodic cleanup sweep must never be the reason a process stays alive. Without
|
|
147
|
+
// unref() this timer holds Node's event loop open, so any process that merely
|
|
148
|
+
// touched this singleton — a test worker, a one-shot CLI run — could not exit.
|
|
149
|
+
this.cleanupTimer.unref();
|
|
145
150
|
}
|
|
146
151
|
|
|
147
152
|
/** Get the singleton instance of BackgroundTaskManager */
|
|
@@ -183,13 +188,16 @@ export class BackgroundTaskManager {
|
|
|
183
188
|
|
|
184
189
|
const taskId = crypto.randomUUID();
|
|
185
190
|
|
|
186
|
-
// Build spawn options
|
|
191
|
+
// Build spawn options.
|
|
192
|
+
//
|
|
193
|
+
// Background tasks are spawned directly rather than through the shell tool, so this
|
|
194
|
+
// path bypassed the security guard entirely and handed every child the full server
|
|
195
|
+
// environment — JWT_SECRET, MONGODB_URI, OPENROUTER_API_KEY and the rest. Children
|
|
196
|
+
// get the same stripped environment the guarded shell path builds.
|
|
187
197
|
const spawnOptions: SpawnOptions = {
|
|
188
198
|
shell: true,
|
|
189
199
|
cwd: options.workingDirectory || process.cwd(),
|
|
190
|
-
env: options.environment
|
|
191
|
-
? { ...process.env, ...options.environment }
|
|
192
|
-
: process.env,
|
|
200
|
+
env: buildShellChildEnv(options.environment),
|
|
193
201
|
};
|
|
194
202
|
|
|
195
203
|
const childProcess = spawn(command, [], spawnOptions);
|
|
@@ -23,7 +23,41 @@
|
|
|
23
23
|
* Adapted from MCP-WWW server for internal MXF use
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
// puppeteer is an OPTIONAL peer dependency: it pulls down a ~170MB Chromium,
|
|
27
|
+
// which no consumer who never opens a browser should pay for. The type import is
|
|
28
|
+
// erased at compile time; the runtime module is loaded on first launch and
|
|
29
|
+
// throws a clear install message if it is not present.
|
|
30
|
+
import type { Browser, Page, LaunchOptions } from 'puppeteer';
|
|
31
|
+
|
|
32
|
+
type PuppeteerModule = {
|
|
33
|
+
launch: (options?: LaunchOptions) => Promise<Browser>;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
let puppeteerModule: PuppeteerModule | null = null;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Load puppeteer on demand.
|
|
40
|
+
*
|
|
41
|
+
* @throws If puppeteer is not installed. There is no headless-less fallback —
|
|
42
|
+
* a browser tool that quietly does nothing is worse than one that fails.
|
|
43
|
+
*/
|
|
44
|
+
const loadPuppeteer = async (): Promise<PuppeteerModule> => {
|
|
45
|
+
if (puppeteerModule) {
|
|
46
|
+
return puppeteerModule;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
const imported = await import('puppeteer');
|
|
51
|
+
puppeteerModule = (imported.default ?? imported) as unknown as PuppeteerModule;
|
|
52
|
+
return puppeteerModule;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
throw new Error(
|
|
55
|
+
'Browser automation requires the optional peer dependency "puppeteer", which is not installed. ' +
|
|
56
|
+
'Install it with `bun add puppeteer` (or `npm install puppeteer`) to use BrowserManager. ' +
|
|
57
|
+
`Underlying error: ${error instanceof Error ? error.message : String(error)}`
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
27
61
|
|
|
28
62
|
export interface BrowserInstance {
|
|
29
63
|
browser: Browser;
|
|
@@ -61,6 +95,11 @@ export class BrowserManager {
|
|
|
61
95
|
this.cleanupInterval = setInterval(() => {
|
|
62
96
|
this.cleanupIdleBrowsers();
|
|
63
97
|
}, 60000); // Check every minute
|
|
98
|
+
// An idle-browser sweep must not be the reason a process stays alive. Without
|
|
99
|
+
// unref() this timer holds Node's event loop open, so anything that merely
|
|
100
|
+
// constructed this manager — including WebSearchService, which builds one — could
|
|
101
|
+
// never exit. A long-running server is kept alive by its listener, not by this.
|
|
102
|
+
this.cleanupInterval.unref();
|
|
64
103
|
}
|
|
65
104
|
|
|
66
105
|
async getBrowser(poolName: string = 'default'): Promise<BrowserInstance> {
|
|
@@ -134,6 +173,7 @@ export class BrowserManager {
|
|
|
134
173
|
}
|
|
135
174
|
|
|
136
175
|
private async createBrowser(): Promise<BrowserInstance> {
|
|
176
|
+
const puppeteer = await loadPuppeteer();
|
|
137
177
|
const browser = await puppeteer.launch({
|
|
138
178
|
headless: this.config.headless ?? true,
|
|
139
179
|
args: [
|
|
@@ -56,9 +56,20 @@ export interface CompressionResult {
|
|
|
56
56
|
originalLength: number;
|
|
57
57
|
compressedLength: number;
|
|
58
58
|
compressionRatio: number;
|
|
59
|
-
method:
|
|
59
|
+
method: CompressionMethod;
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* How a memory's content was compressed.
|
|
64
|
+
*
|
|
65
|
+
* Only 'extractive' exists: leading sentences are kept and the remainder is discarded.
|
|
66
|
+
* The union previously also offered 'semantic' and 'abstractive', neither of which was
|
|
67
|
+
* ever implemented — 'semantic' was reported for what was, in fact, the same extractive
|
|
68
|
+
* slicing. Listing methods the service cannot perform invites callers to trust compressed
|
|
69
|
+
* memories more than they should.
|
|
70
|
+
*/
|
|
71
|
+
export type CompressionMethod = 'extractive';
|
|
72
|
+
|
|
62
73
|
/**
|
|
63
74
|
* Consolidation options
|
|
64
75
|
*/
|
|
@@ -257,18 +268,14 @@ export class MemoryCompressor {
|
|
|
257
268
|
const originalLength = content.length;
|
|
258
269
|
const targetRatio = this.compressionRatios[level];
|
|
259
270
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
// Use extractive compression for light compression
|
|
269
|
-
compressedContent = this.extractiveCompress(content, targetRatio);
|
|
270
|
-
method = 'extractive';
|
|
271
|
-
}
|
|
271
|
+
// Every compression this service performs is extractive: it keeps leading sentences
|
|
272
|
+
// and discards the rest. The previous 'semantic' branch called a semanticCompress()
|
|
273
|
+
// that never touched the contextCompressor it checked for, and did exactly the same
|
|
274
|
+
// sentence-slicing as the extractive path — it differed only in the label it reported.
|
|
275
|
+
// Callers deciding how much to trust a compressed memory were reading that label, so
|
|
276
|
+
// it now says what actually happened.
|
|
277
|
+
const compressedContent = this.extractiveCompress(content, targetRatio);
|
|
278
|
+
const method: CompressionMethod = 'extractive';
|
|
272
279
|
|
|
273
280
|
const compressedLength = compressedContent.length;
|
|
274
281
|
const actualRatio = compressedLength / originalLength;
|
|
@@ -283,25 +290,6 @@ export class MemoryCompressor {
|
|
|
283
290
|
};
|
|
284
291
|
}
|
|
285
292
|
|
|
286
|
-
private async semanticCompress(content: string, targetRatio: number): Promise<string> {
|
|
287
|
-
// Use ContextCompressionEngine for semantic compression
|
|
288
|
-
const targetLength = Math.ceil(content.length * targetRatio);
|
|
289
|
-
|
|
290
|
-
// Simple summarization approach
|
|
291
|
-
const words = content.split(/\s+/);
|
|
292
|
-
const targetWords = Math.ceil(words.length * targetRatio);
|
|
293
|
-
|
|
294
|
-
if (words.length <= targetWords) {
|
|
295
|
-
return content;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// Extract key sentences (simple extractive approach)
|
|
299
|
-
const sentences = content.split(/[.!?]+/).filter(s => s.trim());
|
|
300
|
-
const targetSentences = Math.max(1, Math.ceil(sentences.length * targetRatio));
|
|
301
|
-
|
|
302
|
-
return sentences.slice(0, targetSentences).join('. ') + '.';
|
|
303
|
-
}
|
|
304
|
-
|
|
305
293
|
private extractiveCompress(content: string, targetRatio: number): string {
|
|
306
294
|
// Simple extractive compression: keep first N% of content
|
|
307
295
|
const targetLength = Math.ceil(content.length * targetRatio);
|
|
@@ -30,6 +30,8 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import { Observable, of } from 'rxjs';
|
|
33
|
+
import { IMemoryPersistence } from '../interfaces/IMemoryPersistence.js';
|
|
34
|
+
import { MxfMeilisearchService } from './MxfMeilisearchService.js';
|
|
33
35
|
import { v4 as uuidv4 } from 'uuid';
|
|
34
36
|
|
|
35
37
|
import {
|
|
@@ -160,7 +162,7 @@ export interface IEnhancedChannelMemory extends IChannelMemory {
|
|
|
160
162
|
interface MemoryServiceConfig {
|
|
161
163
|
enablePersistence?: boolean;
|
|
162
164
|
cacheSize?: number;
|
|
163
|
-
persistenceService?:
|
|
165
|
+
persistenceService?: IMemoryPersistence; // Optional persistence service (server-only, injected at boot)
|
|
164
166
|
}
|
|
165
167
|
|
|
166
168
|
/**
|
|
@@ -191,7 +193,7 @@ export class MemoryService {
|
|
|
191
193
|
private config: MemoryServiceConfig;
|
|
192
194
|
|
|
193
195
|
// Optional persistence service (server-only)
|
|
194
|
-
private persistenceService?:
|
|
196
|
+
private persistenceService?: IMemoryPersistence;
|
|
195
197
|
|
|
196
198
|
// Validator
|
|
197
199
|
private validator = createMemoryValidator('MemoryService');
|
|
@@ -612,9 +614,10 @@ export class MemoryService {
|
|
|
612
614
|
}
|
|
613
615
|
|
|
614
616
|
// If persistence service is available, try loading from MongoDB
|
|
615
|
-
|
|
617
|
+
const persistence = this.persistenceService;
|
|
618
|
+
if (persistence) {
|
|
616
619
|
return new Observable<IEnhancedAgentMemory>(observer => {
|
|
617
|
-
|
|
620
|
+
persistence.getAgentMemory(agentId).subscribe({
|
|
618
621
|
next: (loadedMemory: any) => {
|
|
619
622
|
// Enhance the loaded memory with cognitive memory structure
|
|
620
623
|
const enhancedMemory: IEnhancedAgentMemory = {
|
|
@@ -1628,50 +1631,63 @@ export class MemoryService {
|
|
|
1628
1631
|
|
|
1629
1632
|
const semanticSearchStart = Date.now();
|
|
1630
1633
|
|
|
1631
|
-
// Phase A:
|
|
1632
|
-
//
|
|
1633
|
-
//
|
|
1634
|
+
// Phase A: semantic search over the real memory index.
|
|
1635
|
+
//
|
|
1636
|
+
// Meilisearch is the only retrieval path agents actually use, so utility scoring
|
|
1637
|
+
// has to sit on top of it to influence anything. (This previously searched an
|
|
1638
|
+
// in-process cognitiveMemory Map with keyword counting, which nothing populated
|
|
1639
|
+
// in production — so the whole two-phase retrieval could never return a result.)
|
|
1640
|
+
const meilisearch = MxfMeilisearchService.getInstance();
|
|
1641
|
+
const maxCandidates = options.maxCandidates ?? 20;
|
|
1642
|
+
const similarityThreshold = options.similarityThreshold ?? 0.3;
|
|
1643
|
+
|
|
1644
|
+
const filters: string[] = [];
|
|
1645
|
+
if (options.agentId) {
|
|
1646
|
+
filters.push(`agentId = "${options.agentId}"`);
|
|
1647
|
+
}
|
|
1648
|
+
if (options.channelId) {
|
|
1649
|
+
filters.push(`channelId = "${options.channelId}"`);
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
const searchResult = await meilisearch.searchConversations({
|
|
1653
|
+
query: options.query,
|
|
1654
|
+
filter: filters.length > 0 ? filters.join(' AND ') : undefined,
|
|
1655
|
+
limit: maxCandidates,
|
|
1656
|
+
hybridRatio: options.hybridRatio ?? 0.7
|
|
1657
|
+
});
|
|
1658
|
+
|
|
1659
|
+
const semanticSearchTimeMs = Date.now() - semanticSearchStart;
|
|
1660
|
+
|
|
1634
1661
|
const candidates: MemoryCandidate[] = [];
|
|
1635
1662
|
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
if (options.channelId && entry.channelId !== options.channelId) continue;
|
|
1642
|
-
|
|
1643
|
-
// Simple text similarity for cognitive memory (in production, use Meilisearch)
|
|
1644
|
-
const contentStr = JSON.stringify(entry.content).toLowerCase();
|
|
1645
|
-
const queryWords = options.query.toLowerCase().split(/\s+/);
|
|
1646
|
-
const matchCount = queryWords.filter(word => contentStr.includes(word)).length;
|
|
1647
|
-
const similarity = matchCount / Math.max(queryWords.length, 1);
|
|
1648
|
-
|
|
1649
|
-
// Apply similarity threshold
|
|
1650
|
-
const threshold = options.similarityThreshold ?? 0.3;
|
|
1651
|
-
if (similarity >= threshold) {
|
|
1652
|
-
// Get Q-value from manager or use default
|
|
1653
|
-
const qValue = qValueManager.getQValue(entry.id);
|
|
1654
|
-
|
|
1655
|
-
candidates.push({
|
|
1656
|
-
memoryId: entry.id,
|
|
1657
|
-
similarity,
|
|
1658
|
-
qValue,
|
|
1659
|
-
content: entry.content,
|
|
1660
|
-
metadata: {
|
|
1661
|
-
memoryType: entry.memoryType,
|
|
1662
|
-
labels: entry.labels,
|
|
1663
|
-
createdAt: entry.createdAt
|
|
1664
|
-
}
|
|
1665
|
-
});
|
|
1663
|
+
for (const hit of searchResult.hits) {
|
|
1664
|
+
// Meilisearch ranking scores are already normalised to 0..1.
|
|
1665
|
+
const similarity = hit._rankingScore;
|
|
1666
|
+
if (similarity < similarityThreshold) {
|
|
1667
|
+
continue;
|
|
1666
1668
|
}
|
|
1669
|
+
|
|
1670
|
+
candidates.push({
|
|
1671
|
+
memoryId: hit.id,
|
|
1672
|
+
similarity,
|
|
1673
|
+
qValue: qValueManager.getQValue(hit.id),
|
|
1674
|
+
content: hit.content,
|
|
1675
|
+
metadata: {
|
|
1676
|
+
memoryType: hit.role,
|
|
1677
|
+
createdAt: new Date(hit.timestamp)
|
|
1678
|
+
}
|
|
1679
|
+
});
|
|
1667
1680
|
}
|
|
1668
1681
|
|
|
1669
|
-
|
|
1682
|
+
// Load Q-values learned in previous runs before scoring, otherwise every memory
|
|
1683
|
+
// this process has not seen yet would score at the default and ranking would
|
|
1684
|
+
// silently ignore everything the system has learned.
|
|
1685
|
+
await this.hydrateQValues(candidates.map(c => c.memoryId));
|
|
1686
|
+
for (const candidate of candidates) {
|
|
1687
|
+
candidate.qValue = qValueManager.getQValue(candidate.memoryId);
|
|
1688
|
+
}
|
|
1670
1689
|
|
|
1671
|
-
|
|
1672
|
-
const maxCandidates = options.maxCandidates ?? 20;
|
|
1673
|
-
candidates.sort((a, b) => b.similarity - a.similarity);
|
|
1674
|
-
const topCandidates = candidates.slice(0, maxCandidates);
|
|
1690
|
+
const topCandidates = candidates;
|
|
1675
1691
|
|
|
1676
1692
|
// Phase B: Utility scoring
|
|
1677
1693
|
const utilityScoringStart = Date.now();
|
|
@@ -1735,28 +1751,50 @@ export class MemoryService {
|
|
|
1735
1751
|
}
|
|
1736
1752
|
|
|
1737
1753
|
/**
|
|
1738
|
-
*
|
|
1739
|
-
*
|
|
1754
|
+
* Persist the utility subdocument for a memory.
|
|
1755
|
+
*
|
|
1756
|
+
* Registered as QValueManager's persistence callback at server boot, so this runs
|
|
1757
|
+
* on every Q-value change and on dirty-cache eviction. It must throw on failure:
|
|
1758
|
+
* swallowing the error here is what previously made the whole learning system look
|
|
1759
|
+
* like it worked while retaining nothing across a restart.
|
|
1740
1760
|
*/
|
|
1741
1761
|
public async updateMemoryUtility(memoryId: string, utilityUpdate: Partial<MemoryUtilitySubdocument>): Promise<void> {
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1762
|
+
if (!this.persistenceService) {
|
|
1763
|
+
throw new Error(
|
|
1764
|
+
`[MemoryService] Cannot persist utility for memory ${memoryId}: no persistence service is configured. ` +
|
|
1765
|
+
`MULS requires the server to inject a persistence service at boot.`
|
|
1766
|
+
);
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
await this.persistenceService.updateAgentMemoryUtility(memoryId, utilityUpdate);
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
/**
|
|
1773
|
+
* Load stored Q-values for a batch of memories into the QValueManager cache.
|
|
1774
|
+
*
|
|
1775
|
+
* Called before utility scoring so that ranking uses values learned in previous
|
|
1776
|
+
* runs rather than the default for every memory the current process has not seen.
|
|
1777
|
+
* Memories with no stored utility keep the configured default.
|
|
1778
|
+
*/
|
|
1779
|
+
public async hydrateQValues(memoryIds: string[]): Promise<void> {
|
|
1780
|
+
if (!this.persistenceService || memoryIds.length === 0) {
|
|
1747
1781
|
return;
|
|
1748
1782
|
}
|
|
1749
1783
|
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
// This is a partial update to just the utility field
|
|
1755
|
-
await this.persistenceService.updateAgentMemoryUtility?.(memoryId, utilityUpdate);
|
|
1756
|
-
} catch (error) {
|
|
1757
|
-
this.logger.warn(`[MemoryService] Failed to update utility for memory ${memoryId}: ${error}`);
|
|
1758
|
-
}
|
|
1784
|
+
const qValueManager = QValueManager.getInstance();
|
|
1785
|
+
const uncached = memoryIds.filter(id => !qValueManager.isCached(id));
|
|
1786
|
+
if (uncached.length === 0) {
|
|
1787
|
+
return;
|
|
1759
1788
|
}
|
|
1789
|
+
|
|
1790
|
+
const utilities = await this.persistenceService.getAgentMemoryUtilities(uncached);
|
|
1791
|
+
for (const [memoryId, utility] of utilities) {
|
|
1792
|
+
qValueManager.setQValueInCache(memoryId, utility.qValue);
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
this.logger.debug(
|
|
1796
|
+
`[MemoryService] Hydrated ${utilities.size}/${uncached.length} Q-values from persistence`
|
|
1797
|
+
);
|
|
1760
1798
|
}
|
|
1761
1799
|
|
|
1762
1800
|
/**
|
|
@@ -480,6 +480,16 @@ export class MxfMLService {
|
|
|
480
480
|
throw new Error(`[MxfMLService] Model "${modelId}" has not been built yet`);
|
|
481
481
|
}
|
|
482
482
|
|
|
483
|
+
// Fail fast on concurrent training: TF.js rejects overlapping fit() calls
|
|
484
|
+
// on a single model, and two callers would race on this entry's status
|
|
485
|
+
// and metrics. Callers must serialize training per model.
|
|
486
|
+
if (entry.status === ModelStatus.TRAINING) {
|
|
487
|
+
throw new Error(
|
|
488
|
+
`[MxfMLService] Model "${modelId}" is already training — ` +
|
|
489
|
+
`concurrent training on the same model is not allowed`
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
|
|
483
493
|
if (xData.length < entry.config.minTrainingSamples) {
|
|
484
494
|
throw new Error(
|
|
485
495
|
`[MxfMLService] Insufficient training data for "${modelId}": ` +
|
|
@@ -497,10 +507,14 @@ export class MxfMLService {
|
|
|
497
507
|
entry.status = ModelStatus.TRAINING;
|
|
498
508
|
const startTime = Date.now();
|
|
499
509
|
|
|
510
|
+
// fit() is async, so tf.tidy() cannot wrap it — create the tensors
|
|
511
|
+
// eagerly and dispose them in finally so a failed fit() does not leak.
|
|
512
|
+
let xs: import('@tensorflow/tfjs').Tensor2D | undefined;
|
|
513
|
+
let ys: import('@tensorflow/tfjs').Tensor2D | undefined;
|
|
514
|
+
|
|
500
515
|
try {
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
const ys = tfRef.tensor2d(yData);
|
|
516
|
+
xs = tfRef.tensor2d(xData);
|
|
517
|
+
ys = tfRef.tensor2d(yData);
|
|
504
518
|
|
|
505
519
|
const layersModel = entry.model as any;
|
|
506
520
|
const result = await layersModel.fit(xs, ys, {
|
|
@@ -524,10 +538,6 @@ export class MxfMLService {
|
|
|
524
538
|
samplesUsed: xData.length,
|
|
525
539
|
};
|
|
526
540
|
|
|
527
|
-
// Clean up tensors
|
|
528
|
-
xs.dispose();
|
|
529
|
-
ys.dispose();
|
|
530
|
-
|
|
531
541
|
// Update model state
|
|
532
542
|
entry.status = ModelStatus.TRAINED;
|
|
533
543
|
entry.lastTrainingMetrics = metrics;
|
|
@@ -554,6 +564,9 @@ export class MxfMLService {
|
|
|
554
564
|
|
|
555
565
|
this.logger.error(`[MxfMLService] Training failed for ${modelId}: ${errorMsg}`);
|
|
556
566
|
throw error;
|
|
567
|
+
} finally {
|
|
568
|
+
xs?.dispose();
|
|
569
|
+
ys?.dispose();
|
|
557
570
|
}
|
|
558
571
|
}
|
|
559
572
|
|
|
@@ -586,6 +599,14 @@ export class MxfMLService {
|
|
|
586
599
|
throw new Error(`[MxfMLService] Model "${modelId}" has not been built yet`);
|
|
587
600
|
}
|
|
588
601
|
|
|
602
|
+
// Fail fast on concurrent training (same contract as train())
|
|
603
|
+
if (entry.status === ModelStatus.TRAINING) {
|
|
604
|
+
throw new Error(
|
|
605
|
+
`[MxfMLService] Model "${modelId}" is already training — ` +
|
|
606
|
+
`concurrent training on the same model is not allowed`
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
|
|
589
610
|
// Emit training started (we don't know sample count upfront for custom training)
|
|
590
611
|
this.emitTrainingStarted(modelId, 0, entry.config.epochs);
|
|
591
612
|
|
|
@@ -1067,6 +1088,11 @@ export class MxfMLService {
|
|
|
1067
1088
|
*
|
|
1068
1089
|
* Calls the provided trainCallback at the model's retrainIntervalMs.
|
|
1069
1090
|
* The callback is responsible for collecting training data and calling train().
|
|
1091
|
+
* No-ops when auto-training is disabled (autoTrainEnabled=false).
|
|
1092
|
+
*
|
|
1093
|
+
* A model must have exactly one retraining owner: either a timer registered
|
|
1094
|
+
* here or a consumer-managed schedule — never both. Overlapping schedules
|
|
1095
|
+
* train on the same data twice and collide on TF.js fit() calls.
|
|
1070
1096
|
*
|
|
1071
1097
|
* @param modelId - ID of a registered model
|
|
1072
1098
|
* @param trainCallback - Function to call when retraining is due
|
|
@@ -1077,6 +1103,14 @@ export class MxfMLService {
|
|
|
1077
1103
|
): void {
|
|
1078
1104
|
const entry = this.getModelEntry(modelId);
|
|
1079
1105
|
|
|
1106
|
+
if (!getTensorFlowConfig().autoTrainEnabled) {
|
|
1107
|
+
this.logger.info(
|
|
1108
|
+
`[MxfMLService] Auto-training disabled (autoTrainEnabled=false), ` +
|
|
1109
|
+
`not scheduling retrain for ${modelId}`
|
|
1110
|
+
);
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1080
1114
|
// Clear any existing timer for this model
|
|
1081
1115
|
const existingTimer = this.retrainTimers.get(modelId);
|
|
1082
1116
|
if (existingTimer) {
|