@mxf-dev/core 1.0.0

package/src/mxp/MxpTokenOptimizer.ts

@@ -0,0 +1,266 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/mxf-dev/mxf
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+/**
+ * MXP Token Optimizer
+ * 
+ * Provides intelligent token reduction for MXF messages using multiple optimization strategies.
+ * This is a foundational implementation ready for integration with existing MXF services.
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import { Logger } from '../utils/Logger.js';
+import { EventBus } from '../events/EventBus.js';
+import { Events } from '../events/EventNames.js';
+import { createStrictValidator } from '../utils/validation.js';
+import { ChannelMessage, AgentMessage } from '../schemas/MessageSchemas.js';
+import { 
+    MxpConfig, 
+    TokenOptimizationStrategy, 
+    MxpOptimizationResult 
+} from '../types/MxpTypes.js';
+import { MxpConfigManager } from './MxpConfigManager.js';
+import { ContextCompressionEngine, CompressedContext } from './ContextCompressionEngine.js';
+
+/**
+ * MXP Token Optimizer - Foundational Implementation
+ * Ready for integration with existing MXF services
+ */
+export class MxpTokenOptimizer {
+    private readonly logger: Logger;
+    private readonly validator = createStrictValidator('MxpTokenOptimizer');
+    private readonly eventBus = EventBus.server;
+    
+    // Performance tracking
+    private optimizationStats = {
+        totalOptimizations: 0,
+        totalTokensSaved: 0,
+        averageCompressionRatio: 0
+    };
+
+    constructor(private config: MxpConfig) {
+        this.logger = new Logger('info', 'MxpTokenOptimizer', 'server');
+    }
+
+    /**
+     * Optimize a message for token reduction with configuration-aware processing
+     * Respects channel and agent-level MXP settings
+     */
+    public async optimizeMessage(
+        message: ChannelMessage | AgentMessage,
+        options: {
+            strategy?: TokenOptimizationStrategy;
+            compressionLevel?: 'light' | 'standard' | 'aggressive';
+            channelId?: string;
+            agentId?: string;
+        } = {}
+    ): Promise<MxpOptimizationResult> {
+        this.validator.assertIsObject(message, 'Message must be a valid object');
+        
+        const originalSize = JSON.stringify(message).length;
+        const channelId = options.channelId || message.context?.channelId || 'default';
+        const agentId = options.agentId || message.senderId || 'system';
+        
+        
+        // Check if MXP token optimization is enabled for this channel/agent combination
+        const isTokenOptimizationEnabled = MxpConfigManager.getInstance().isFeatureEnabled(
+            channelId, 
+            'tokenOptimization', 
+            agentId
+        );
+        
+        if (!isTokenOptimizationEnabled) {
+            return {
+                operationId: `mxp_disabled_${Date.now()}`,
+                timestamp: Date.now(),
+                agentId: agentId as any,
+                channelId: channelId as any,
+                tokenOptimization: {
+                    originalTokens: originalSize,
+                    optimizedTokens: originalSize,
+                    reductionPercentage: 0,
+                    strategy: 'disabled_by_config'
+                },
+                performance: {
+                    processingTimeMs: 0.1,
+                    memoryUsageMb: 0.01,
+                    cpuUtilization: 0.001
+                }
+            };
+        }
+        
+        // Check if the specific strategy is enabled
+        const strategy = options.strategy || 'context_compression';
+        const strategyKey = this.mapStrategyToConfigKey(strategy);
+        const isStrategyEnabled = MxpConfigManager.getInstance().isTokenStrategyEnabled(
+            channelId,
+            strategyKey,
+            agentId
+        );
+        
+        if (!isStrategyEnabled) {
+            return {
+                operationId: `mxp_strategy_disabled_${Date.now()}`,
+                timestamp: Date.now(),
+                agentId: agentId as any,
+                channelId: channelId as any,
+                tokenOptimization: {
+                    originalTokens: originalSize,
+                    optimizedTokens: originalSize,
+                    reductionPercentage: 0,
+                    strategy: `${strategy}_disabled`
+                },
+                performance: {
+                    processingTimeMs: 0.1,
+                    memoryUsageMb: 0.01,
+                    cpuUtilization: 0.001
+                }
+            };
+        }
+        
+        // Update statistics
+        this.optimizationStats.totalOptimizations++;
+        
+        // Get effective configuration for this channel/agent
+        const effectiveConfig = MxpConfigManager.getInstance().getEffectiveConfig(channelId, agentId);
+        
+        // Apply optimization based on configuration
+        const result: MxpOptimizationResult = {
+            operationId: `mxp_${uuidv4()}`,
+            timestamp: Date.now(),
+            agentId: agentId as any,
+            channelId: channelId as any,
+            
+            tokenOptimization: {
+                originalTokens: originalSize,
+                optimizedTokens: originalSize, // Will be enhanced with actual optimization
+                reductionPercentage: 0,
+                strategy: strategy
+            },
+            
+            performance: {
+                processingTimeMs: 1,
+                memoryUsageMb: 0.1,
+                cpuUtilization: 0.01
+            }
+        };
+        
+        
+        return result;
+    }
+
+    /**
+     * Apply context compression to a conversation history
+     * Integrates with ContextCompressionEngine for intelligent compression
+     */
+    public async compressConversationContext(
+        messages: (ChannelMessage | AgentMessage)[],
+        options: {
+            channelId: string;
+            agentId?: string;
+            windowSize?: number;
+            compressionRatio?: number;
+            preserveKeywords?: string[];
+        }
+    ): Promise<CompressedContext | null> {
+        this.validator.assertIsArray(messages, 'Messages must be an array');
+        this.validator.assertIsNonEmptyString(options.channelId, 'Channel ID is required');
+
+        // Check if context compression is enabled for this channel/agent
+        const isCompressionEnabled = MxpConfigManager.getInstance().isTokenStrategyEnabled(
+            options.channelId,
+            'contextCompression',
+            options.agentId
+        );
+
+        if (!isCompressionEnabled) {
+            return null; // Return null when disabled to avoid processing
+        }
+
+
+        try {
+            // Use ContextCompressionEngine for intelligent compression
+            const compressed = await ContextCompressionEngine.getInstance().compressConversation(messages, {
+                channelId: options.channelId,
+                agentId: options.agentId,
+                windowSize: options.windowSize || 5,
+                compressionRatio: options.compressionRatio || 0.3,
+                preserveKeywords: options.preserveKeywords || [],
+                useContextReferences: true
+            });
+
+            if (compressed) {
+                // Update our internal statistics
+                this.optimizationStats.totalOptimizations++;
+                this.optimizationStats.totalTokensSaved += (compressed.originalTokens - compressed.compressedTokens);
+                
+                // Recalculate average compression ratio
+                const totalProcessed = this.optimizationStats.totalOptimizations;
+                this.optimizationStats.averageCompressionRatio = 
+                    ((this.optimizationStats.averageCompressionRatio * (totalProcessed - 1)) + compressed.ratio) / totalProcessed;
+
+            }
+
+            return compressed;
+
+        } catch (error) {
+            this.logger.error('Context compression failed', {
+                error: error instanceof Error ? error.message : String(error),
+                channelId: options.channelId,
+                messageCount: messages.length
+            });
+            return null; // Fail gracefully
+        }
+    }
+    
+    /**
+     * Get optimization statistics
+     */
+    public getOptimizationStats(): {
+        totalOptimizations: number;
+        totalTokensSaved: number;
+        averageCompressionRatio: number;
+    } {
+        return { ...this.optimizationStats };
+    }
+
+    /**
+     * Map TokenOptimizationStrategy to configuration key names
+     */
+    private mapStrategyToConfigKey(strategy: TokenOptimizationStrategy): 
+        'contextCompression' | 'promptOptimization' | 'templateMatching' | 'entityDeduplication' | 'toolSchemaReduction' | 'conversationSummarization' {
+        switch (strategy) {
+            case 'context_compression':
+                return 'contextCompression';
+            case 'prompt_optimization':
+                return 'promptOptimization';
+            case 'template_matching':
+                return 'templateMatching';
+            case 'entity_deduplication':
+                return 'entityDeduplication';
+            case 'tool_schema_reduction':
+                return 'toolSchemaReduction';
+            case 'conversation_summarization':
+                return 'conversationSummarization';
+            default:
+                return 'contextCompression'; // Default fallback
+        }
+    }
+}

package/src/prompts/ChannelContextProvider.ts

@@ -0,0 +1,212 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/mxf-dev/mxf
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+/**
+ * Channel Context Provider
+ * 
+ * Provides channel-specific context and configuration for agent prompts.
+ * This includes channel metadata, participants, memory summaries, and
+ * channel-specific rules or behaviors.
+ */
+
+import { ChannelConfig } from '../interfaces/ChannelConfig.js';
+import { Logger } from '../utils/Logger.js';
+import { createStrictValidator } from '../utils/validation.js';
+
+const logger = new Logger('info', 'ChannelContextProvider', 'client');
+const validator = createStrictValidator('ChannelContextProvider');
+
+export interface ChannelContext {
+    channelId: string;
+    channelName: string;
+    channelDescription?: string;
+    participants?: string[];
+    sharedMemory?: any;
+    channelMetadata?: Record<string, any>;
+    channelRules?: string[];
+}
+
+export class ChannelContextProvider {
+    /**
+     * Build channel context prompt section
+     */
+    public static buildChannelContext(
+        channelConfig: ChannelConfig | null,
+        channelContext?: ChannelContext
+    ): string {
+        if (!channelConfig && !channelContext) {
+            return '';
+        }
+
+        const sections = [];
+
+        // Channel Identity
+        if (channelContext?.channelName || channelConfig?.name) {
+            sections.push(`## Channel Context
+
+**Channel Name**: ${channelContext?.channelName || channelConfig?.name}
+**Channel ID**: ${channelContext?.channelId || 'Unknown'}`);
+        }
+
+        // Channel Description
+        if (channelContext?.channelDescription || channelConfig?.description) {
+            sections.push(`**Channel Purpose**: ${channelContext?.channelDescription || channelConfig?.description}`);
+        }
+
+        // Channel Configuration
+        if (channelConfig) {
+            const configDetails = [];
+            if (channelConfig.isPrivate !== undefined) {
+                configDetails.push(`- **Access**: ${channelConfig.isPrivate ? 'Private' : 'Public'}`);
+            }
+            if (channelConfig.requireApproval !== undefined) {
+                configDetails.push(`- **Approval Required**: ${channelConfig.requireApproval ? 'Yes' : 'No'}`);
+            }
+            if (channelConfig.maxAgents) {
+                configDetails.push(`- **Max Agents**: ${channelConfig.maxAgents}`);
+            }
+            if (channelConfig.allowAnonymous !== undefined) {
+                configDetails.push(`- **Anonymous Access**: ${channelConfig.allowAnonymous ? 'Allowed' : 'Not Allowed'}`);
+            }
+
+            if (configDetails.length > 0) {
+                sections.push(`**Channel Settings**:
+${configDetails.join('\n')}`);
+            }
+        }
+
+        // Channel Participants
+        if (channelContext?.participants && channelContext.participants.length > 0) {
+            sections.push(`**Active Participants**:
+${channelContext.participants.map(p => `- ${p}`).join('\n')}`);
+        }
+
+        // Channel Rules
+        if (channelContext?.channelRules && channelContext.channelRules.length > 0) {
+            sections.push(`**Channel Rules**:
+${channelContext.channelRules.map(rule => `- ${rule}`).join('\n')}`);
+        }
+
+        // Channel Metadata
+        if (channelContext?.channelMetadata && Object.keys(channelContext.channelMetadata).length > 0) {
+            const metadataEntries = Object.entries(channelContext.channelMetadata)
+                .filter(([key, value]) => value !== null && value !== undefined)
+                .map(([key, value]) => `- **${key}**: ${String(value)}`);
+            
+            if (metadataEntries.length > 0) {
+                sections.push(`**Channel Metadata**:
+${metadataEntries.join('\n')}`);
+            }
+        }
+
+        // Channel-specific memory summary
+        if (channelContext?.sharedMemory) {
+            const memorySummary = this.summarizeSharedMemory(channelContext.sharedMemory);
+            if (memorySummary) {
+                sections.push(`**Channel Memory Summary**:
+${memorySummary}`);
+            }
+        }
+
+        return sections.filter(Boolean).join('\n\n');
+    }
+
+    /**
+     * Summarize shared memory for prompt inclusion
+     */
+    private static summarizeSharedMemory(sharedMemory: any): string {
+        const summaryParts = [];
+
+        // Count memory types
+        if (sharedMemory.notes && sharedMemory.notes.length > 0) {
+            summaryParts.push(`- ${sharedMemory.notes.length} notes`);
+        }
+        if (sharedMemory.sharedState && Object.keys(sharedMemory.sharedState).length > 0) {
+            summaryParts.push(`- ${Object.keys(sharedMemory.sharedState).length} shared state items`);
+        }
+        if (sharedMemory.conversationHistory && sharedMemory.conversationHistory.length > 0) {
+            summaryParts.push(`- ${sharedMemory.conversationHistory.length} conversation entries`);
+        }
+        if (sharedMemory.customData && Object.keys(sharedMemory.customData).length > 0) {
+            summaryParts.push(`- ${Object.keys(sharedMemory.customData).length} custom data items`);
+        }
+
+        return summaryParts.length > 0 ? summaryParts.join('\n') : '';
+    }
+
+    /**
+     * Extract important notes from shared memory
+     */
+    public static extractImportantNotes(sharedMemory: any, limit: number = 3): string[] {
+        if (!sharedMemory?.notes || sharedMemory.notes.length === 0) {
+            return [];
+        }
+
+        // Sort by timestamp (most recent first) and take top N
+        const sortedNotes = [...sharedMemory.notes]
+            .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
+            .slice(0, limit);
+
+        return sortedNotes.map(note => note.content);
+    }
+
+    /**
+     * Build collaborative context for multi-agent scenarios
+     */
+    public static buildCollaborativeContext(
+        agentId: string,
+        otherAgents: string[],
+        channelContext?: ChannelContext
+    ): string {
+        if (!otherAgents || otherAgents.length === 0) {
+            return '';
+        }
+
+        const sections = [];
+
+        sections.push(`## Collaborative Context
+
+You are working alongside ${otherAgents.length} other agent(s) in this channel:`);
+
+        sections.push(otherAgents.map(agent => `- ${agent}`).join('\n'));
+
+        // Add collaboration guidelines
+        sections.push(`
+**Collaboration Guidelines**:
+- Share relevant information through the channel's shared memory
+- Coordinate actions to avoid conflicts
+- Update shared state when making significant changes
+- Respect other agents' ongoing tasks`);
+
+        // Add any channel-specific collaboration rules
+        if (channelContext?.channelRules) {
+            const collabRules = channelContext.channelRules
+                .filter(rule => rule.toLowerCase().includes('collab') || rule.toLowerCase().includes('coordinate'))
+                .map(rule => `- ${rule}`);
+            
+            if (collabRules.length > 0) {
+                sections.push(`**Channel-Specific Collaboration Rules**:
+${collabRules.join('\n')}`);
+            }
+        }
+
+        return sections.join('\n\n');
+    }
+}

package/src/prompts/DeferredToolSchemaRegistry.ts

@@ -0,0 +1,212 @@
+/**
+ * Copyright 2024 Brad Anderson
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Brad Anderson <BradA1878@pm.me>
+ * @repository https://github.com/mxf-dev/mxf
+ * @documentation https://mxf-dev.github.io/mxf/
+ */
+
+/**
+ * Deferred Tool Schema Registry
+ *
+ * Classifies tools into tier-1 (always full schema) and tier-2
+ * (name-only listing, schema loaded on demand). When total tools
+ * exceed the tier-2 threshold, only tier-1 tools get full schemas
+ * in the system prompt; tier-2 tools are listed as available and
+ * discoverable via the existing tools_recommend meta-tool.
+ *
+ * Inspired by Claude Code's progressive tool schema disclosure.
+ */
+
+import { Logger } from '../utils/Logger.js';
+
+const logger = new Logger('info', 'DeferredToolSchemaRegistry', 'server');
+
+/** Classification tier for a tool */
+export type ToolTier = 'tier1' | 'tier2';
+
+/**
+ * Deferred Tool Schema Registry — singleton
+ */
+export class DeferredToolSchemaRegistry {
+    private static instance: DeferredToolSchemaRegistry | null = null;
+
+    /** Tools explicitly assigned to tier-1 (always get full schema) */
+    private tier1Tools = new Set<string>();
+
+    /** Tools explicitly assigned to tier-2 (deferred schema) */
+    private tier2Tools = new Set<string>();
+
+    /**
+     * When total tool count exceeds this threshold, unclassified tools
+     * default to tier-2 instead of tier-1.
+     */
+    private tier2Threshold = 15;
+
+    private constructor() {
+        this.registerDefaults();
+    }
+
+    /** Get singleton instance */
+    public static getInstance(): DeferredToolSchemaRegistry {
+        if (!DeferredToolSchemaRegistry.instance) {
+            DeferredToolSchemaRegistry.instance = new DeferredToolSchemaRegistry();
+        }
+        return DeferredToolSchemaRegistry.instance;
+    }
+
+    /** Set the tier-2 threshold (tool count above which unclassified tools become tier-2) */
+    public setTier2Threshold(threshold: number): void {
+        this.tier2Threshold = threshold;
+        logger.info('Tier-2 threshold updated', { threshold });
+    }
+
+    /** Get the current tier-2 threshold */
+    public getTier2Threshold(): number {
+        return this.tier2Threshold;
+    }
+
+    /** Explicitly assign a tool to tier-1 (always full schema) */
+    public setTier1(toolName: string): void {
+        this.tier1Tools.add(toolName);
+        this.tier2Tools.delete(toolName);
+    }
+
+    /** Explicitly assign a tool to tier-2 (deferred schema) */
+    public setTier2(toolName: string): void {
+        this.tier2Tools.add(toolName);
+        this.tier1Tools.delete(toolName);
+    }
+
+    /**
+     * Classify a tool given the total number of tools available.
+     * - Explicitly assigned tools return their assigned tier.
+     * - When total tools <= threshold, everything is tier-1.
+     * - When total tools > threshold, unclassified tools are tier-2.
+     */
+    public classify(toolName: string, totalToolCount: number): ToolTier {
+        if (this.tier1Tools.has(toolName)) return 'tier1';
+        if (this.tier2Tools.has(toolName)) return 'tier2';
+
+        // Below threshold, everything gets full schema
+        if (totalToolCount <= this.tier2Threshold) return 'tier1';
+
+        // Above threshold, unclassified tools are deferred
+        return 'tier2';
+    }
+
+    /**
+     * Partition a list of tool names into tier-1 and tier-2 groups.
+     * Convenience method for MxfAgentSystemPrompt.buildToolSchemas().
+     */
+    public partition(toolNames: string[]): { tier1: string[]; tier2: string[] } {
+        const total = toolNames.length;
+        const tier1: string[] = [];
+        const tier2: string[] = [];
+
+        for (const name of toolNames) {
+            if (this.classify(name, total) === 'tier1') {
+                tier1.push(name);
+            } else {
+                tier2.push(name);
+            }
+        }
+
+        logger.debug('Tool schema partition', {
+            total,
+            tier1Count: tier1.length,
+            tier2Count: tier2.length,
+            threshold: this.tier2Threshold,
+        });
+
+        return { tier1, tier2 };
+    }
+
+    /**
+     * Build a summary string for deferred (tier-2) tools.
+     * This replaces their full schemas in the system prompt.
+     *
+     * @param tier2Names - Names of deferred tools
+     * @param discoveryToolName - Name of the tool used to discover deferred tools (default: 'tools_recommend')
+     */
+    public buildDeferredSummary(tier2Names: string[], discoveryToolName: string = 'tools_recommend'): string {
+        if (tier2Names.length === 0) return '';
+
+        return `## Additional Tools Available
+
+The following ${tier2Names.length} tools are available but not shown in detail. Use \`${discoveryToolName}\` to discover the right tool for your task, or \`tools_validate\` to check parameters.
+
+Available: ${tier2Names.join(', ')}`;
+    }
+
+    /** List all explicitly tier-1 tools */
+    public listTier1(): string[] {
+        return Array.from(this.tier1Tools);
+    }
+
+    /** List all explicitly tier-2 tools */
+    public listTier2(): string[] {
+        return Array.from(this.tier2Tools);
+    }
+
+    /** Clear all classifications (useful for testing) */
+    public clear(): void {
+        this.tier1Tools.clear();
+        this.tier2Tools.clear();
+    }
+
+    /** Reset to default classifications */
+    public reset(): void {
+        this.clear();
+        this.registerDefaults();
+    }
+
+    /**
+     * Register default tier-1 tools — these always get full schemas
+     * because agents use them frequently and must know their parameters.
+     */
+    private registerDefaults(): void {
+        // ORPAR control loop — core to agent operation
+        const alwaysFull = [
+            'controlLoop_observe',
+            'controlLoop_reason',
+            'controlLoop_plan',
+            'controlLoop_act',
+            'controlLoop_reflect',
+            // Communication — used every task
+            'messaging_send',
+            'agent_broadcast',
+            // Task lifecycle
+            'task_complete',
+            'task_create',
+            'task_create_with_plan',
+            // Meta tools — needed to discover tier-2 tools
+            'tools_recommend',
+            'tools_validate',
+            'tools_recommend_on_error',
+            // Context and memory
+            'context_inject',
+            'context_get',
+            // User interaction
+            'user_input',
+        ];
+
+        for (const tool of alwaysFull) {
+            this.tier1Tools.add(tool);
+        }
+
+        logger.info('Default tier-1 tools registered', { count: this.tier1Tools.size });
+    }
+}