0xray 2.0.0

package/dist/delegation/analytics/outcome-tracker.js

@@ -0,0 +1,279 @@
+/**
+ * Routing Outcome Tracker
+ *
+ * Tracks routing outcomes and provides statistics for performance analysis.
+ * Extracted from task-skill-router.ts as part of Phase 2 refactoring.
+ *
+ * @version 1.0.0
+ * @since 2026-03-12
+ */
+import fs from 'fs';
+import path from 'path';
+/**
+ * RoutingOutcomeTracker class
+ *
+ * Tracks routing outcomes with circular buffer pattern for memory efficiency.
+ * Provides methods for recording outcomes, calculating statistics, and
+ * retrieving data for analytics.
+ *
+ * Persists outcomes to logs/framework/routing-outcomes.json for analytics to read.
+ */
+export class RoutingOutcomeTracker {
+    outcomes = [];
+    maxOutcomes;
+    persistencePath;
+    saveTimeout = null;
+    SAVE_DEBOUNCE_MS = 5000; // Save max once per 5 seconds
+    /**
+     * Create a new outcome tracker
+     * @param maxOutcomes Maximum number of outcomes to retain (default: 1000)
+     */
+    constructor(maxOutcomes = 1000) {
+        this.maxOutcomes = maxOutcomes;
+        // Persist to logs/framework/routing-outcomes.json
+        const cwd = process.cwd() || '.';
+        this.persistencePath = `${cwd}/logs/framework/routing-outcomes.json`;
+        this.initializeFile();
+        // Note: Don't auto-load here - use reloadFromDisk() to load existing data
+    }
+    /**
+     * Ensure the log directory and file exist on initialization
+     */
+    async initializeFile() {
+        try {
+            const fs = await import('fs');
+            const path = await import('path');
+            // Ensure directory exists
+            const dir = path.dirname(this.persistencePath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            // Create empty file if it doesn't exist
+            if (!fs.existsSync(this.persistencePath)) {
+                fs.writeFileSync(this.persistencePath, '[]');
+            }
+        }
+        catch (error) {
+            // Silent fail - don't break tracking
+        }
+    }
+    /**
+     * Load outcomes from disk on initialization
+     */
+    loadFromDisk() {
+        try {
+            if (fs.existsSync(this.persistencePath)) {
+                const data = fs.readFileSync(this.persistencePath, 'utf-8');
+                const parsed = JSON.parse(data);
+                if (Array.isArray(parsed)) {
+                    // Load outcomes, keeping within max limit
+                    this.outcomes = parsed.slice(-this.maxOutcomes);
+                }
+            }
+        }
+        catch (error) {
+            // Silent fail - don't break tracking if persistence fails
+        }
+    }
+    /**
+     * Save outcomes to disk (debounced)
+     */
+    scheduleSave() {
+        if (this.saveTimeout) {
+            clearTimeout(this.saveTimeout);
+        }
+        this.saveTimeout = setTimeout(() => {
+            this.saveToDisk();
+        }, this.SAVE_DEBOUNCE_MS);
+    }
+    /**
+     * Immediately save to disk (synchronous for reliability)
+     */
+    saveToDisk() {
+        try {
+            // Ensure directory exists
+            const dir = path.dirname(this.persistencePath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            // Save outcomes as JSON
+            fs.writeFileSync(this.persistencePath, JSON.stringify(this.outcomes, null, 2));
+        }
+        catch (error) {
+            // Silent fail - don't break tracking
+        }
+    }
+    /**
+     * Record a new routing outcome
+     * @param outcome The outcome to record (timestamp will be added)
+     */
+    recordOutcome(outcome) {
+        if (!this.maxOutcomes)
+            return;
+        this.outcomes.push({ ...outcome, timestamp: new Date() });
+        // Keep only recent outcomes (circular buffer)
+        if (this.outcomes.length > this.maxOutcomes) {
+            this.outcomes = this.outcomes.slice(-this.maxOutcomes);
+        }
+        // Persist to disk immediately (not debounced for better tracking during tests/CI)
+        this.saveToDisk();
+    }
+    /**
+     * Update an existing outcome by taskId
+     * @param taskId The task ID to update
+     * @param updates The fields to update
+     * @returns true if updated, false if not found
+     */
+    updateOutcome(taskId, updates) {
+        const index = this.outcomes.findIndex(o => o.taskId === taskId);
+        if (index === -1)
+            return false;
+        this.outcomes[index] = { ...this.outcomes[index], ...updates };
+        this.saveToDisk();
+        return true;
+    }
+    /**
+     * Force reload from disk - call this before analytics to get latest data
+     */
+    async reloadFromDisk() {
+        await this.loadFromDisk();
+    }
+    /**
+     * Get all outcomes, optionally filtered by agent
+     * @param agent Optional agent name to filter by
+     * @returns Array of routing outcomes
+     */
+    getOutcomes(agent) {
+        if (agent) {
+            return this.outcomes.filter((o) => o.routedAgent === agent);
+        }
+        return [...this.outcomes];
+    }
+    /**
+     * Calculate success rate for a specific agent
+     * @param agent The agent name
+     * @returns Success rate between 0 and 1
+     */
+    getSuccessRate(agent) {
+        const agentOutcomes = this.outcomes.filter((o) => o.routedAgent === agent && o.success !== undefined);
+        if (agentOutcomes.length === 0)
+            return 0;
+        const successes = agentOutcomes.filter((o) => o.success).length;
+        return successes / agentOutcomes.length;
+    }
+    /**
+     * Get statistics for all agents
+     * @returns Array of agent statistics
+     */
+    getStats() {
+        const stats = new Map();
+        for (const outcome of this.outcomes) {
+            if (outcome.success === undefined)
+                continue;
+            const current = stats.get(outcome.routedAgent) || { total: 0, successes: 0 };
+            current.total++;
+            if (outcome.success)
+                current.successes++;
+            stats.set(outcome.routedAgent, current);
+        }
+        return Array.from(stats.entries()).map(([agent, data]) => ({
+            agent,
+            total: data.total,
+            attempts: data.total,
+            successes: data.successes,
+            successRate: data.total > 0 ? data.successes / data.total : 0,
+        }));
+    }
+    /**
+     * Clear all recorded outcomes
+     */
+    clear() {
+        this.outcomes = [];
+        // Also clear the file to ensure clean state
+        try {
+            fs.writeFileSync(this.persistencePath, '[]');
+        }
+        catch (error) {
+            // Silent fail
+        }
+    }
+    /**
+     * Get the count of recorded outcomes
+     * @returns Number of outcomes
+     */
+    getCount() {
+        return this.outcomes.length;
+    }
+    /**
+     * Export outcomes to JSON string
+     * @returns JSON string of all outcomes
+     */
+    toJSON() {
+        return JSON.stringify(this.outcomes);
+    }
+    /**
+     * Convert outcomes to prompt data points for pattern analysis
+     * @returns Array of prompt data points
+     */
+    getPromptData() {
+        return this.outcomes.map((outcome) => ({
+            taskId: outcome.taskId,
+            prompt: outcome.taskDescription,
+            timestamp: outcome.timestamp,
+            complexity: Math.min(100, Math.floor(outcome.taskDescription.length / 5)),
+            keywords: [...new Set(outcome.taskDescription.toLowerCase().split(/\s+/)
+                    .filter(w => w.length > 3)
+                    .slice(0, 10))],
+            context: {},
+            routingDecision: {
+                taskId: outcome.taskId,
+                agent: outcome.routedAgent,
+                skill: outcome.routedSkill,
+                confidence: outcome.confidence,
+                reason: outcome.feedback || 'Historical routing',
+                timestamp: outcome.timestamp,
+            },
+            outcome: {
+                success: outcome.success ?? false,
+                agent: outcome.routedAgent,
+                skill: outcome.routedSkill,
+                feedback: outcome.feedback || '',
+            },
+        }));
+    }
+    /**
+     * Convert outcomes to routing decisions
+     * @returns Array of routing decisions
+     */
+    getRoutingDecisions() {
+        return this.outcomes.map((outcome) => ({
+            taskId: outcome.taskId,
+            agent: outcome.routedAgent,
+            skill: outcome.routedSkill,
+            confidence: outcome.confidence,
+            reason: outcome.feedback || 'Historical routing',
+            timestamp: outcome.timestamp,
+        }));
+    }
+    /**
+     * Apply routing refinements to existing outcomes
+     * @param changes Array of changes to apply
+     */
+    applyRoutingRefinements(changes) {
+        for (const change of changes) {
+            const outcome = this.outcomes.find((o) => o.taskId === change.taskId);
+            if (outcome) {
+                if (change.agent)
+                    outcome.routedAgent = change.agent;
+                if (change.skill)
+                    outcome.routedSkill = change.skill;
+                if (change.confidence)
+                    outcome.confidence = change.confidence;
+            }
+        }
+    }
+}
+/**
+ * Global routing outcome tracker instance
+ */
+export const routingOutcomeTracker = new RoutingOutcomeTracker();

package/dist/delegation/analytics/routing-analytics.js

@@ -0,0 +1,193 @@
+/**
+ * Routing Analytics
+ *
+ * Provides analytics aggregation and insights generation for routing data.
+ * Extracted from task-skill-router.ts as part of Phase 2 refactoring.
+ *
+ * @version 1.0.0
+ * @since 2026-03-12
+ */
+import { frameworkLogger } from "../../core/framework-logger.js";
+/**
+ * RoutingAnalytics class
+ *
+ * Aggregates routing data and provides analytics insights.
+ * Works with RoutingOutcomeTracker to generate reports and recommendations.
+ */
+export class RoutingAnalytics {
+    tracker;
+    constructor(tracker) {
+        this.tracker = tracker;
+    }
+    /**
+     * Get daily analytics summary for reporting
+     * @returns Daily summary with metrics and insights
+     */
+    getDailySummary() {
+        const stats = this.tracker.getStats();
+        const agentStats = stats.map((data) => ({
+            agent: data.agent,
+            count: data.total,
+            successRate: data.successRate,
+        }));
+        const topAgents = agentStats
+            .sort((a, b) => b.count - a.count)
+            .slice(0, 10);
+        const totalRoutings = agentStats.reduce((sum, a) => sum + a.count, 0);
+        const totalSuccesses = stats.reduce((sum, s) => sum + s.successes, 0);
+        const averageSuccessRate = totalRoutings > 0 ? totalSuccesses / totalRoutings : 0;
+        const insights = [];
+        if (totalRoutings > 0 && averageSuccessRate < 0.5) {
+            insights.push(`Low overall success rate: ${(averageSuccessRate * 100).toFixed(1)}%`);
+        }
+        return {
+            totalRoutings,
+            averageConfidence: 0.85, // Placeholder for actual confidence calculation
+            templateMatchRate: 0.9, // Placeholder for template matching
+            successRate: averageSuccessRate,
+            topAgents,
+            topKeywords: [], // Placeholder for keyword analytics
+            insights,
+        };
+    }
+    /**
+     * Get full routing analytics data
+     * @returns Comprehensive analytics data
+     */
+    getFullAnalytics() {
+        const stats = this.tracker.getStats();
+        const totalRoutings = stats.reduce((sum, s) => sum + s.total, 0);
+        const overallSuccessRate = stats.length > 0
+            ? stats.reduce((sum, s) => sum + s.successRate, 0) / stats.length
+            : 0;
+        return {
+            promptPatterns: {
+                totalPrompts: totalRoutings,
+                templateMatches: Math.floor(totalRoutings * 0.9),
+                templateMatchRate: 0.9,
+                gaps: [],
+                emergingPatterns: [],
+            },
+            routingPerformance: {
+                totalRoutings,
+                overallSuccessRate,
+                avgConfidence: 0.85,
+                timeRange: {
+                    start: new Date(Date.now() - 24 * 60 * 60 * 1000),
+                    end: new Date(),
+                },
+                recommendations: [],
+                agentMetrics: stats.map((data) => ({
+                    agent: data.agent,
+                    successRate: data.successRate,
+                    count: data.total,
+                })),
+                keywordEffectiveness: [],
+                confidenceMetrics: [],
+            },
+        };
+    }
+    /**
+     * Apply routing refinements based on analytics
+     * @param apply Whether to actually apply the changes
+     * @returns Refinement result with changes list
+     */
+    applyRoutingRefinements(apply) {
+        const changes = [];
+        // Analyze low-performing agents
+        const stats = this.tracker.getStats();
+        for (const data of stats) {
+            if (data.successRate < 0.5 && data.total > 10) {
+                changes.push({
+                    type: 'removed',
+                    reason: `Low success rate (${(data.successRate * 100).toFixed(1)}%) with ${data.total} attempts`,
+                    data: { agent: data.agent },
+                });
+            }
+        }
+        if (apply) {
+            // Apply the changes here
+            // This would modify the mappings in production
+            frameworkLogger.log("routing-analytics", "routing-refinements-applied", "info", { message: `Applied ${changes.length} routing refinements` });
+        }
+        return {
+            appliedMappings: apply
+                ? changes.filter((c) => c.type === 'added').length
+                : 0,
+            optimizedMappings: apply
+                ? changes.filter((c) => c.type === 'optimized').length
+                : 0,
+            removedMappings: apply
+                ? changes.filter((c) => c.type === 'removed').length
+                : 0,
+            changes,
+        };
+    }
+    /**
+     * Generate insights from routing data
+     * @returns Array of insight strings
+     */
+    generateInsights() {
+        const insights = [];
+        const stats = this.tracker.getStats();
+        if (stats.length === 0) {
+            insights.push('No routing data available yet');
+            return insights;
+        }
+        // Find most used agent
+        const topAgent = stats.reduce((prev, current) => prev.total > current.total ? prev : current);
+        insights.push(`Most used agent: ${topAgent.agent} (${topAgent.total} routes)`);
+        // Find most successful agent
+        const bestAgent = stats.reduce((prev, current) => prev.successRate > current.successRate ? prev : current);
+        insights.push(`Best performing agent: ${bestAgent.agent} (${(bestAgent.successRate * 100).toFixed(1)}% success rate)`);
+        // Find worst performing agent (with at least 5 attempts)
+        const agentsWithData = stats.filter((s) => s.total >= 5);
+        if (agentsWithData.length > 0) {
+            const worstAgent = agentsWithData.reduce((prev, current) => prev.successRate < current.successRate ? prev : current);
+            if (worstAgent.successRate < 0.7) {
+                insights.push(`Agent needing attention: ${worstAgent.agent} (${(worstAgent.successRate * 100).toFixed(1)}% success rate)`);
+            }
+        }
+        // Overall statistics
+        const totalRoutes = stats.reduce((sum, s) => sum + s.total, 0);
+        const avgSuccess = stats.reduce((sum, s) => sum + s.successRate, 0) / stats.length;
+        insights.push(`Total routes: ${totalRoutes}`);
+        insights.push(`Average success rate: ${(avgSuccess * 100).toFixed(1)}%`);
+        return insights;
+    }
+    /**
+     * Get raw statistics from the tracker
+     * @returns Record of agent statistics
+     */
+    getRawStats() {
+        const stats = this.tracker.getStats();
+        const result = {};
+        for (const stat of stats) {
+            result[stat.agent] = {
+                attempts: stat.total,
+                successes: stat.successes,
+                successRate: stat.successRate,
+            };
+        }
+        return result;
+    }
+    /**
+     * Compare current performance to historical baseline
+     * @returns Comparison metrics
+     */
+    compareToBaseline() {
+        const stats = this.tracker.getStats();
+        const currentRate = stats.length > 0
+            ? stats.reduce((sum, s) => sum + s.successRate, 0) / stats.length
+            : 0;
+        // Baseline is hardcoded at 70% for now (could be configurable)
+        const baselineRate = 0.7;
+        const changePercent = ((currentRate - baselineRate) / baselineRate) * 100;
+        return {
+            improved: currentRate > baselineRate,
+            changePercent: Math.abs(changePercent),
+            currentRate,
+            baselineRate,
+        };
+    }
+}