0xray 4.0.13 → 4.0.15

package/dist/analytics/predictive-analytics.js

@@ -0,0 +1,247 @@
+/**
+ * Predictive Analytics for Optimal Agent Routing
+ *
+ * Predicts the best agent to route a task to based on historical
+ * success rates using simple keyword overlap scoring.
+ *
+ */
+import { routingOutcomeTracker } from '../delegation/analytics/outcome-tracker.js';
+/**
+ * Calculate keyword overlap score between a task description and a set of
+ * historical task descriptions routed to a specific agent.
+ */
+function keywordOverlapScore(description, historicalDescriptions) {
+    const descWords = new Set(description.toLowerCase().split(/\W+/).filter(w => w.length > 2));
+    if (descWords.size === 0 || historicalDescriptions.length === 0)
+        return 0;
+    let totalScore = 0;
+    let matchCount = 0;
+    for (const hist of historicalDescriptions) {
+        const histWords = new Set(hist.toLowerCase().split(/\W+/).filter(w => w.length > 2));
+        let overlap = 0;
+        for (const w of descWords) {
+            if (histWords.has(w))
+                overlap++;
+        }
+        if (overlap > 0) {
+            totalScore += overlap / Math.max(descWords.size, histWords.size);
+            matchCount++;
+        }
+    }
+    return matchCount > 0 ? totalScore / matchCount : 0;
+}
+/**
+ * Calculate risk level based on confidence and success probability.
+ * Ported from advanced-features for enhanced decision-making.
+ */
+function calculateRiskLevel(confidence, successProbability) {
+    const riskScore = 1 - confidence + (1 - successProbability);
+    if (riskScore < 0.3)
+        return "low";
+    if (riskScore < 0.7)
+        return "medium";
+    return "high";
+}
+/**
+ * Estimate task duration based on agent's historical performance.
+ */
+function estimateTaskDuration(agentOutcomes) {
+    if (agentOutcomes.length === 0)
+        return 0;
+    const durations = agentOutcomes
+        .filter(o => o.executionTimeMs !== undefined)
+        .map(o => o.executionTimeMs);
+    if (durations.length === 0)
+        return 0;
+    const avg = durations.reduce((a, b) => a + b, 0) / durations.length;
+    const variance = durations.reduce((acc, d) => acc + Math.pow(d - avg, 2), 0) / durations.length;
+    const stdDev = Math.sqrt(variance);
+    const coefficientOfVariation = avg > 0 ? stdDev / avg : 0;
+    return Math.max(avg * (1 + coefficientOfVariation * 0.1), 10);
+}
+/**
+ * Build agent performance summary from historical outcomes.
+ */
+function buildAgentMetrics(agentOutcomes) {
+    const totalTasks = agentOutcomes.length;
+    const successfulTasks = agentOutcomes.filter(o => o.success).length;
+    const failedTasks = totalTasks - successfulTasks;
+    // Calculate average execution time
+    const durations = agentOutcomes
+        .filter(o => o.executionTimeMs !== undefined)
+        .map(o => o.executionTimeMs);
+    const averageExecutionTime = durations.length > 0
+        ? durations.reduce((a, b) => a + b, 0) / durations.length
+        : 0;
+    // Recent performance (last 20 tasks)
+    const recentPerformance = agentOutcomes
+        .slice(-20)
+        .map(o => o.success ? 1 : 0);
+    // Task type breakdown
+    const taskTypeBreakdown = {};
+    for (const outcome of agentOutcomes) {
+        const taskType = outcome.taskType || "unknown";
+        if (!taskTypeBreakdown[taskType]) {
+            taskTypeBreakdown[taskType] = { count: 0, successRate: 0 };
+        }
+        taskTypeBreakdown[taskType].count++;
+    }
+    // Calculate per-task-type success rates
+    for (const taskType of Object.keys(taskTypeBreakdown)) {
+        const taskOutcomes = agentOutcomes.filter(o => (o.taskType || "unknown") === taskType);
+        const successes = taskOutcomes.filter(o => o.success).length;
+        const entry = taskTypeBreakdown[taskType];
+        if (entry) {
+            entry.successRate = successes / taskOutcomes.length;
+        }
+    }
+    return {
+        totalTasks,
+        successfulTasks,
+        failedTasks,
+        averageExecutionTime,
+        successRate: totalTasks > 0 ? successfulTasks / totalTasks : 0,
+        recentPerformance,
+        taskTypeBreakdown,
+    };
+}
+export const predictiveAnalytics = {
+    /**
+     * Predict the optimal agent for a given task description.
+     *
+     * Loads outcomes from routingOutcomeTracker, groups by agent,
+     * and picks the agent with the best historical success rate
+     * among those that have keyword overlap with the task.
+     */
+    async predict(taskDescription) {
+        await routingOutcomeTracker.reloadFromDisk();
+        const outcomes = routingOutcomeTracker.getOutcomes();
+        // Filter to outcomes that have a resolved success field
+        const resolved = outcomes.filter((o) => o.success !== undefined && o.taskDescription);
+        if (resolved.length === 0)
+            return null;
+        // Group outcomes by agent
+        const agentMap = new Map();
+        for (const o of resolved) {
+            const list = agentMap.get(o.routedAgent) || [];
+            list.push(o);
+            agentMap.set(o.routedAgent, list);
+        }
+        let bestPrediction = null;
+        let bestScore = -1;
+        for (const [agent, agentOutcomes] of agentMap.entries()) {
+            const descriptions = agentOutcomes.map(o => o.taskDescription);
+            const overlap = keywordOverlapScore(taskDescription, descriptions);
+            if (overlap <= 0)
+                continue; // skip agents with no keyword overlap
+            const successes = agentOutcomes.filter(o => o.success).length;
+            const successRate = successes / agentOutcomes.length;
+            // Weighted score: 70% keyword overlap, 30% success rate
+            const score = (overlap * 0.7) + (successRate * 0.3);
+            if (score > bestScore) {
+                bestScore = score;
+                const riskLevel = calculateRiskLevel(score, successRate);
+                const estimatedDuration = estimateTaskDuration(agentOutcomes);
+                const agentMetrics = buildAgentMetrics(agentOutcomes);
+                bestPrediction = {
+                    agent,
+                    confidence: Math.min(score, 1),
+                    historicalSuccessRate: successRate,
+                    sampleSize: agentOutcomes.length,
+                    riskLevel,
+                    estimatedDuration,
+                    agentMetrics,
+                };
+            }
+        }
+        return bestPrediction;
+    },
+    /**
+     * Predict the globally optimal agent regardless of task description.
+     *
+     * Returns the agent with the highest success rate that has at least
+     * 3 historical samples.
+     */
+    async predictOptimalAgent() {
+        await routingOutcomeTracker.reloadFromDisk();
+        const stats = routingOutcomeTracker.getStats();
+        // Filter to agents with >= 3 samples
+        const qualified = stats.filter(s => s.total >= 3);
+        if (qualified.length === 0)
+            return null;
+        // Sort by success rate descending, then by total samples descending
+        qualified.sort((a, b) => {
+            if (b.successRate !== a.successRate)
+                return b.successRate - a.successRate;
+            return b.total - a.total;
+        });
+        const top = qualified[0];
+        const outcomes = routingOutcomeTracker.getOutcomes();
+        const agentOutcomes = outcomes.filter(o => o.routedAgent === top.agent);
+        const riskLevel = calculateRiskLevel(top.successRate, top.successRate);
+        const estimatedDuration = estimateTaskDuration(agentOutcomes);
+        const agentMetrics = buildAgentMetrics(agentOutcomes);
+        return {
+            agent: top.agent,
+            confidence: top.successRate,
+            historicalSuccessRate: top.successRate,
+            sampleSize: top.total,
+            riskLevel,
+            estimatedDuration,
+            agentMetrics,
+        };
+    },
+    /**
+     * Synchronous prediction — uses in-memory outcome data without disk reload.
+     * Suitable for hot-path usage in agent-delegator.
+     */
+    predictSync(taskDescription) {
+        const stats = routingOutcomeTracker.getStats();
+        const outcomes = routingOutcomeTracker.getOutcomes();
+        if (stats.length === 0 || outcomes.length === 0)
+            return null;
+        // Group outcomes by agent
+        const byAgent = new Map();
+        for (const o of outcomes) {
+            const arr = byAgent.get(o.routedAgent) || [];
+            arr.push(o);
+            byAgent.set(o.routedAgent, arr);
+        }
+        let bestAgent = null;
+        let bestScore = -1;
+        let bestRate = 0;
+        let bestSamples = 0;
+        let bestAgentOutcomes = [];
+        for (const [agent, agentOutcomes] of byAgent) {
+            const descriptions = agentOutcomes.map(o => o.taskDescription);
+            const overlap = keywordOverlapScore(taskDescription, descriptions);
+            const agentStat = stats.find(s => s.agent === agent);
+            const successRate = agentStat?.successRate ?? 0;
+            const total = agentStat?.total ?? 0;
+            // Weighted score: 70% keyword overlap + 30% historical success rate
+            const score = (0.7 * overlap) + (0.3 * successRate);
+            if (score > bestScore && total >= 3) {
+                bestScore = score;
+                bestAgent = agent;
+                bestRate = successRate;
+                bestSamples = total;
+                // Store for enhanced return
+                bestAgentOutcomes = agentOutcomes;
+            }
+        }
+        if (!bestAgent || bestScore < 0.3)
+            return null;
+        const riskLevel = calculateRiskLevel(bestScore, bestRate);
+        const estimatedDuration = estimateTaskDuration(bestAgentOutcomes);
+        const agentMetrics = buildAgentMetrics(bestAgentOutcomes);
+        return {
+            agent: bestAgent,
+            confidence: Math.min(bestScore, 1.0),
+            historicalSuccessRate: bestRate,
+            sampleSize: bestSamples,
+            riskLevel,
+            estimatedDuration,
+            agentMetrics,
+        };
+    },
+};

package/dist/analytics/prompt-pattern-analyzer.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Prompt Pattern Analyzer for 0xRay
+ *
+ * Analyzes actual vs. template prompts to detect gaps and emerging patterns.
+ * Integrates with RoutingOutcomeTracker to provide insights for template optimization.
+ *
+ * @since 2026-03-05
+ */
+export interface TemplateGap {
+    gapType: "missing_template" | "pattern_mismatch" | "emerging_pattern";
+    userRequest: string;
+    generatedPrompt: string;
+    suggestedAgent: string;
+    suggestedSkill: string;
+    frequency: number;
+    lastSeen: Date;
+    confidence: number;
+}
+export interface EmergingPattern {
+    patternId: string;
+    keywords: string[];
+    sampleRequests: string[];
+    suggestedAgent: string;
+    suggestedSkill: string;
+    confidence: number;
+    frequency: number;
+    avgConfidence: number;
+    successRate: number;
+}
+export interface PromptComparisonResult {
+    totalPrompts: number;
+    templateMatches: number;
+    nonTemplatePrompts: number;
+    templateMatchRate: number;
+    gaps: TemplateGap[];
+    emergingPatterns: EmergingPattern[];
+    topMissedKeywords: Array<{
+        keyword: string;
+        count: number;
+        suggestedMappings: string[];
+    }>;
+    agentCoverage: Map<string, {
+        total: number;
+        withTemplate: number;
+        withoutTemplate: number;
+    }>;
+}
+export interface TemplateOptimizationSuggestion {
+    suggestionType: "add_template" | "update_keywords" | "merge_patterns" | "remove_template";
+    templateName?: string;
+    keywords: string[];
+    targetAgent: string;
+    targetSkill: string;
+    confidence: number;
+    expectedImpact: string;
+    reasoning: string;
+}
+declare class PromptPatternAnalyzer {
+    private readonly minFrequencyThreshold;
+    private readonly confidenceThreshold;
+    private readonly emergingPatternMinSamples;
+    /**
+     * Analyze prompt patterns and compare against templates
+     */
+    analyzePromptPatterns(): PromptComparisonResult;
+    /**
+     * Detect gaps where templates don't match user requests
+     */
+    private detectTemplateGaps;
+    /**
+     * Identify emerging patterns from real usage
+     */
+    private identifyEmergingPatterns;
+    /**
+     * Analyze frequently missed keywords
+     */
+    private analyzeMissedKeywords;
+    /**
+     * Calculate agent coverage metrics
+     */
+    private calculateAgentCoverage;
+    /**
+     * Generate template optimization suggestions
+     */
+    generateOptimizationSuggestions(comparisonResult: PromptComparisonResult): TemplateOptimizationSuggestion[];
+    /**
+     * Get analytics summary as formatted report
+     */
+    generateReport(comparisonResult: PromptComparisonResult): string;
+    private classifyGapType;
+    private normalizeRequest;
+    private extractKeywords;
+    private generatePatternId;
+    private selectBestAgent;
+    private emptyComparisonResult;
+}
+export declare const promptPatternAnalyzer: PromptPatternAnalyzer;
+export {};