@skillsmith/core 0.5.5 → 0.5.6

package/dist/testing/MultiLLMProvider.helpers.js

@@ -0,0 +1,126 @@
+/**
+ * Multi-LLM Provider Helper Functions and Classes
+ * @module @skillsmith/core/testing/MultiLLMProvider.helpers
+ */
+/**
+ * Simple circuit breaker for provider failover
+ */
+export class CircuitBreaker {
+    config;
+    state = 'closed';
+    failures = 0;
+    successes = 0;
+    lastFailureTime = null;
+    totalRequests = 0;
+    constructor(config) {
+        this.config = config;
+    }
+    getState() {
+        if (this.state === 'open' && this.lastFailureTime) {
+            const elapsed = Date.now() - this.lastFailureTime.getTime();
+            if (elapsed >= this.config.resetTimeoutMs) {
+                this.state = 'half-open';
+            }
+        }
+        return this.state;
+    }
+    recordSuccess() {
+        this.successes++;
+        this.totalRequests++;
+        if (this.state === 'half-open') {
+            this.state = 'closed';
+            this.failures = 0;
+        }
+    }
+    recordFailure() {
+        this.failures++;
+        this.totalRequests++;
+        this.lastFailureTime = new Date();
+        if (this.totalRequests >= this.config.volumeThreshold) {
+            const errorRate = this.failures / this.totalRequests;
+            if (errorRate * 100 >= this.config.errorThresholdPercentage) {
+                this.state = 'open';
+            }
+        }
+    }
+    canExecute() {
+        const state = this.getState();
+        return state === 'closed' || state === 'half-open';
+    }
+    reset() {
+        this.state = 'closed';
+        this.failures = 0;
+        this.successes = 0;
+        this.totalRequests = 0;
+        this.lastFailureTime = null;
+    }
+    getMetrics() {
+        const errorRate = this.totalRequests > 0 ? this.failures / this.totalRequests : 0;
+        return {
+            failures: this.failures,
+            successes: this.successes,
+            errorRate,
+        };
+    }
+}
+// ============================================================================
+// Utility Functions
+// ============================================================================
+/**
+ * Determine error condition from error object
+ */
+export function getErrorCondition(error) {
+    if (error instanceof Error) {
+        const message = error.message.toLowerCase();
+        if (message.includes('rate limit') || message.includes('429')) {
+            return 'rate_limit';
+        }
+        if (message.includes('timeout') || message.includes('timed out')) {
+            return 'timeout';
+        }
+        if (message.includes('unavailable') ||
+            message.includes('503') ||
+            message.includes('connection')) {
+            return 'unavailable';
+        }
+    }
+    return 'error';
+}
+/**
+ * Estimate token count from text
+ * Rough estimate: ~4 characters per token
+ */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Calculate cost from token counts
+ */
+export function calculateCost(config, inputTokens, outputTokens) {
+    const inputCost = ((config.costPerInputToken ?? 0) * inputTokens) / 1000;
+    const outputCost = ((config.costPerOutputToken ?? 0) * outputTokens) / 1000;
+    return inputCost + outputCost;
+}
+/**
+ * Calculate compatibility score from response
+ * Simple scoring based on latency and success
+ */
+export function calculateCompatibilityScore(response) {
+    // Lower latency = higher score, max score 1.0
+    const latencyScore = Math.max(0, 1 - response.latencyMs / 5000);
+    return Math.min(1, latencyScore + 0.3); // Base 0.3 for successful response
+}
+/**
+ * Simulate network request (for testing)
+ */
+export async function simulateRequest(_config) {
+    // Simulate network latency (50-200ms)
+    // In production, this would use _config to make actual API calls
+    const latency = 50 + Math.random() * 150;
+    await new Promise((resolve) => setTimeout(resolve, latency));
+    // Simulate occasional failures (5% rate)
+    if (Math.random() < 0.05) {
+        throw new Error('Simulated provider error');
+    }
+}
+//# sourceMappingURL=MultiLLMProvider.helpers.js.map

package/dist/testing/MultiLLMProvider.helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MultiLLMProvider.helpers.js","sourceRoot":"","sources":["../../src/testing/MultiLLMProvider.helpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH;;GAEG;AACH,MAAM,OAAO,cAAc;IAOL;IANZ,KAAK,GAAiB,QAAQ,CAAA;IAC9B,QAAQ,GAAG,CAAC,CAAA;IACZ,SAAS,GAAG,CAAC,CAAA;IACb,eAAe,GAAgB,IAAI,CAAA;IACnC,aAAa,GAAG,CAAC,CAAA;IAEzB,YAAoB,MAA4B;QAA5B,WAAM,GAAN,MAAM,CAAsB;IAAG,CAAC;IAEpD,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAA;YAC3D,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC1C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAA;YAC1B,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;IAED,aAAa;QACX,IAAI,CAAC,SAAS,EAAE,CAAA;QAChB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAA;YACrB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;QACnB,CAAC;IACH,CAAC;IAED,aAAa;QACX,IAAI,CAAC,QAAQ,EAAE,CAAA;QACf,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,EAAE,CAAA;QAEjC,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YACtD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAA;YACpD,IAAI,SAAS,GAAG,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;gBAC5D,IAAI,CAAC,KAAK,GAAG,MAAM,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,UAAU;QACR,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC7B,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,CAAA;IACpD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAA;QACrB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;QACjB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAA;QAClB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAA;QACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAA;IAC7B,CAAC;IAED,UAAU;QACR,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;QACjF,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS;SACV,CAAA;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO,YAAY,CAAA;QACrB,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACjE,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,IACE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAC9B,CAAC;YACD,OAAO,aAAa,CAAA;QACtB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,WAAmB,EACnB,YAAoB;IAEpB,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,IAAI,CAAA;IACxE,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,IAAI,CAAA;IAC3E,OAAO,SAAS,GAAG,UAAU,CAAA;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CAAC,QAAqB;IAC/D,8CAA8C;IAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,CAAA;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,GAAG,CAAC,CAAA,CAAC,mCAAmC;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAuB;IAC3D,sCAAsC;IACtC,iEAAiE;IACjE,MAAM,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAA;IACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;IAE5D,yCAAyC;IACzC,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAC7C,CAAC;AACH,CAAC"}

package/dist/testing/MultiLLMProvider.js

@@ -0,0 +1,321 @@
+/**
+ * @fileoverview Multi-LLM Provider Chain for Skill Compatibility Testing
+ * @module @skillsmith/core/testing/MultiLLMProvider
+ * @see SMI-1523: Configure multi-LLM provider chain
+ * @see SMI-2741: Split to meet 500-line standard
+ *
+ * Provides a unified interface for testing skills across multiple LLM providers:
+ * - Claude (Anthropic) - Primary
+ * - GPT (OpenAI) - Fallback 1
+ * - Gemini (Google) - Fallback 2
+ * - Cohere (Command) - Fallback 3
+ * - Ollama (Local) - Fallback 4
+ *
+ * Features:
+ * - Automatic failover with configurable strategy
+ * - Health monitoring and circuit breaker pattern
+ * - Cost-aware provider selection
+ * - Skill compatibility testing across providers
+ */
+import { EventEmitter } from 'events';
+export { DEFAULT_MULTI_LLM_CONFIG } from './MultiLLMProvider.types.js';
+// Re-export helpers for consumers
+export { CircuitBreaker, getErrorCondition, estimateTokens, calculateCost, calculateCompatibilityScore, } from './MultiLLMProvider.helpers.js';
+// Re-export selection utilities for consumers
+export { selectProvider, selectRoundRobin, selectLeastLoaded, selectByLatency, selectByCost, getFallbackProvider, } from './MultiLLMProvider.selection.js';
+// Re-export metrics utilities for consumers
+export { aggregateMetrics, runCompatibilityTest } from './MultiLLMProvider.metrics.js';
+import { DEFAULT_MULTI_LLM_CONFIG } from './MultiLLMProvider.types.js';
+import { CircuitBreaker, estimateTokens, calculateCost, simulateRequest, } from './MultiLLMProvider.helpers.js';
+import { selectProvider, getFallbackProvider } from './MultiLLMProvider.selection.js';
+import { aggregateMetrics, runCompatibilityTest } from './MultiLLMProvider.metrics.js';
+// ============================================================================
+// Multi-LLM Provider Implementation
+// ============================================================================
+/**
+ * Multi-LLM Provider for skill compatibility testing
+ *
+ * Provides unified access to multiple LLM providers with automatic failover,
+ * load balancing, and cost optimization.
+ *
+ * @example
+ * ```typescript
+ * const provider = new MultiLLMProvider()
+ * await provider.initialize()
+ *
+ * // Complete a request with automatic failover
+ * const response = await provider.complete({
+ *   messages: [{ role: 'user', content: 'Explain this skill' }]
+ * })
+ *
+ * // Test skill compatibility across providers
+ * const compatibility = await provider.testSkillCompatibility('commit')
+ * ```
+ */
+export class MultiLLMProvider extends EventEmitter {
+    config;
+    initialized = false;
+    circuitBreakers = new Map();
+    providerMetrics = new Map();
+    activeRequests = new Map();
+    roundRobinIndex = 0;
+    // V3 integration disabled (SMI-3600); field retained for close() cleanup path
+    v3ProviderManager = null;
+    constructor(config = {}) {
+        super();
+        this.config = {
+            ...DEFAULT_MULTI_LLM_CONFIG,
+            ...config,
+            providers: { ...DEFAULT_MULTI_LLM_CONFIG.providers, ...config.providers },
+            fallbackStrategy: {
+                ...DEFAULT_MULTI_LLM_CONFIG.fallbackStrategy,
+                ...config.fallbackStrategy,
+            },
+        };
+    }
+    /** Initialize the multi-LLM provider */
+    async initialize() {
+        if (this.initialized)
+            return;
+        // Initialize circuit breakers for each enabled provider
+        for (const [providerType, providerConfig] of Object.entries(this.config.providers)) {
+            if (providerConfig?.enabled) {
+                this.circuitBreakers.set(providerType, new CircuitBreaker(this.config.circuitBreaker));
+                this.activeRequests.set(providerType, 0);
+                this.initializeMetrics(providerType);
+            }
+        }
+        // IMPORTANT: Keep dynamic import here for V3 lazy loading / graceful degradation
+        if (this.config.useV3Integration) {
+            await this.initializeV3Integration();
+        }
+        this.initialized = true;
+        this.emit('initialized', {
+            providers: this.getEnabledProviders(),
+            defaultProvider: this.config.defaultProvider,
+        });
+    }
+    /** Check if provider is initialized */
+    isInitialized() {
+        return this.initialized;
+    }
+    /** Get enabled providers */
+    getEnabledProviders() {
+        return Object.entries(this.config.providers)
+            .filter(([, config]) => config?.enabled)
+            .map(([type]) => type);
+    }
+    /** Get available providers (enabled + circuit closed) */
+    getAvailableProviders() {
+        return this.getEnabledProviders().filter((provider) => {
+            const breaker = this.circuitBreakers.get(provider);
+            return breaker?.canExecute() ?? false;
+        });
+    }
+    /** Check provider health */
+    async healthCheck(provider) {
+        const startTime = Date.now();
+        try {
+            const config = this.config.providers[provider];
+            if (!config?.enabled) {
+                return { healthy: false, error: 'Provider not enabled', timestamp: new Date() };
+            }
+            const breaker = this.circuitBreakers.get(provider);
+            const healthy = breaker?.canExecute() ?? false;
+            return {
+                healthy,
+                latencyMs: Date.now() - startTime,
+                timestamp: new Date(),
+                details: { circuitState: breaker?.getState() ?? 'unknown', model: config.model },
+            };
+        }
+        catch (error) {
+            return {
+                healthy: false,
+                latencyMs: Date.now() - startTime,
+                error: error instanceof Error ? error.message : 'Unknown error',
+                timestamp: new Date(),
+            };
+        }
+    }
+    /** Get provider status */
+    getProviderStatus(provider) {
+        const breaker = this.circuitBreakers.get(provider);
+        const activeReqs = this.activeRequests.get(provider) ?? 0;
+        const config = this.config.providers[provider];
+        const maxConcurrency = config?.maxConcurrency ?? 50;
+        return {
+            available: breaker?.canExecute() ?? false,
+            currentLoad: activeReqs / maxConcurrency,
+            queueLength: 0,
+            activeRequests: activeReqs,
+            circuitState: breaker?.getState() ?? 'closed',
+        };
+    }
+    /** Complete a request with automatic failover */
+    async complete(request) {
+        this.ensureInitialized();
+        const startTime = Date.now();
+        const provider = request.provider ?? this.doSelectProvider(request);
+        const fallbackChain = [];
+        let currentProvider = provider;
+        let attempts = 0;
+        const maxAttempts = this.config.fallbackStrategy.maxAttempts;
+        while (attempts < maxAttempts) {
+            attempts++;
+            try {
+                const response = await this.executeRequest(currentProvider, request);
+                this.circuitBreakers.get(currentProvider)?.recordSuccess();
+                this.updateMetrics(currentProvider, {
+                    success: true,
+                    latencyMs: Date.now() - startTime,
+                    cost: response.cost,
+                });
+                return {
+                    ...response,
+                    usedFallback: fallbackChain.length > 0,
+                    fallbackChain: fallbackChain.length > 0 ? fallbackChain : undefined,
+                };
+            }
+            catch (error) {
+                this.circuitBreakers.get(currentProvider)?.recordFailure();
+                this.updateMetrics(currentProvider, { success: false, latencyMs: Date.now() - startTime });
+                this.emit('provider_error', { provider: currentProvider, error });
+                const fallbackProvider = this.doGetFallbackProvider(currentProvider, error);
+                if (fallbackProvider && attempts < maxAttempts) {
+                    fallbackChain.push(currentProvider);
+                    currentProvider = fallbackProvider;
+                    continue;
+                }
+                throw error;
+            }
+        }
+        throw new Error(`All providers failed after ${maxAttempts} attempts`);
+    }
+    /** Test skill compatibility across providers */
+    async testSkillCompatibility(skillId) {
+        this.ensureInitialized();
+        return runCompatibilityTest(skillId, this.getEnabledProviders(), (req) => this.complete(req));
+    }
+    /** Get provider metrics */
+    getMetrics() {
+        return new Map(this.providerMetrics);
+    }
+    /** Get aggregated metrics */
+    getAggregatedMetrics() {
+        return aggregateMetrics(this.providerMetrics);
+    }
+    /** Close and cleanup */
+    close() {
+        this.circuitBreakers.clear();
+        this.providerMetrics.clear();
+        this.activeRequests.clear();
+        this.removeAllListeners();
+        this.initialized = false;
+        this.v3ProviderManager = null;
+        this.roundRobinIndex = 0;
+    }
+    // ============================================================================
+    // Private Methods
+    // ============================================================================
+    ensureInitialized() {
+        if (!this.initialized) {
+            throw new Error('MultiLLMProvider not initialized. Call initialize() first.');
+        }
+    }
+    // V3 provider modules unavailable after claude-flow → ruflo rename (SMI-3600)
+    // @claude-flow/cli restricts subpath imports; always use local provider chain
+    async initializeV3Integration() {
+        this.v3ProviderManager = null;
+        this.emit('v3_integration', { enabled: false, reason: 'V3 not available' });
+    }
+    doSelectProvider(request) {
+        const available = this.getAvailableProviders();
+        const result = selectProvider(request, available, this.config, this.roundRobinIndex, (p) => this.getProviderStatus(p), (p) => this.providerMetrics.get(p));
+        this.roundRobinIndex = result.nextRoundRobinIndex;
+        return result.provider;
+    }
+    doGetFallbackProvider(currentProvider, error) {
+        if (!this.config.fallbackStrategy.enabled)
+            return null;
+        const available = this.getAvailableProviders();
+        return getFallbackProvider(currentProvider, error, this.config.fallbackStrategy.rules, available);
+    }
+    async executeRequest(provider, request) {
+        const config = this.config.providers[provider];
+        if (!config) {
+            throw new Error(`Provider ${provider} not configured`);
+        }
+        const startTime = Date.now();
+        const current = this.activeRequests.get(provider) ?? 0;
+        this.activeRequests.set(provider, current + 1);
+        try {
+            await simulateRequest(config);
+            const latencyMs = Date.now() - startTime;
+            const inputTokens = estimateTokens(request.messages.map((m) => m.content).join(' '));
+            const outputTokens = 100; // Simulated
+            return {
+                content: `[Simulated response from ${provider}]`,
+                provider,
+                model: request.model ?? config.model,
+                usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
+                cost: calculateCost(config, inputTokens, outputTokens),
+                latencyMs,
+                usedFallback: false,
+            };
+        }
+        finally {
+            const currentAfter = this.activeRequests.get(provider) ?? 1;
+            this.activeRequests.set(provider, Math.max(0, currentAfter - 1));
+        }
+    }
+    initializeMetrics(provider) {
+        this.providerMetrics.set(provider, {
+            provider,
+            timestamp: new Date(),
+            avgLatencyMs: 0,
+            errorRate: 0,
+            successRate: 1,
+            load: 0,
+            totalCost: 0,
+            totalRequests: 0,
+            availability: 1,
+        });
+    }
+    updateMetrics(provider, update) {
+        const metrics = this.providerMetrics.get(provider);
+        if (!metrics)
+            return;
+        metrics.totalRequests++;
+        metrics.timestamp = new Date();
+        // Rolling average for latency
+        metrics.avgLatencyMs =
+            (metrics.avgLatencyMs * (metrics.totalRequests - 1) + update.latencyMs) /
+                metrics.totalRequests;
+        // Update success/error rates
+        const breaker = this.circuitBreakers.get(provider);
+        if (breaker) {
+            const breakerMetrics = breaker.getMetrics();
+            metrics.errorRate = breakerMetrics.errorRate;
+            metrics.successRate = 1 - breakerMetrics.errorRate;
+        }
+        if (update.cost) {
+            metrics.totalCost += update.cost;
+        }
+        const status = this.getProviderStatus(provider);
+        metrics.load = status.currentLoad;
+        this.emit('metrics', metrics);
+    }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+/**
+ * Create and initialize a MultiLLMProvider instance
+ */
+export async function createMultiLLMProvider(config = {}) {
+    const provider = new MultiLLMProvider(config);
+    await provider.initialize();
+    return provider;
+}
+//# sourceMappingURL=MultiLLMProvider.js.map

package/dist/testing/MultiLLMProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MultiLLMProvider.js","sourceRoot":"","sources":["../../src/testing/MultiLLMProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAqBrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAA;AAEtE,kCAAkC;AAClC,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,2BAA2B,GAC5B,MAAM,+BAA+B,CAAA;AAGtC,8CAA8C;AAC9C,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,iCAAiC,CAAA;AAExC,4CAA4C;AAC5C,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAA;AAetF,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAA;AACtE,OAAO,EACL,cAAc,EACd,cAAc,EACd,aAAa,EACb,eAAe,GAChB,MAAM,+BAA+B,CAAA;AACtC,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAA;AACrF,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAA;AAEtF,+EAA+E;AAC/E,oCAAoC;AACpC,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,OAAO,gBAAiB,SAAQ,YAAY;IACxC,MAAM,CAAwB;IAC9B,WAAW,GAAG,KAAK,CAAA;IACnB,eAAe,GAAyC,IAAI,GAAG,EAAE,CAAA;IACjE,eAAe,GAA0C,IAAI,GAAG,EAAE,CAAA;IAClE,cAAc,GAAiC,IAAI,GAAG,EAAE,CAAA;IACxD,eAAe,GAAG,CAAC,CAAA;IAE3B,8EAA8E;IACtE,iBAAiB,GAAY,IAAI,CAAA;IAEzC,YAAY,SAAiC,EAAE;QAC7C,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,MAAM,GAAG;YACZ,GAAG,wBAAwB;YAC3B,GAAG,MAAM;YACT,SAAS,EAAE,EAAE,GAAG,wBAAwB,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE;YACzE,gBAAgB,EAAE;gBAChB,GAAG,wBAAwB,CAAC,gBAAgB;gBAC5C,GAAG,MAAM,CAAC,gBAAgB;aACP;SACtB,CAAA;IACH,CAAC;IAED,wCAAwC;IACxC,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAM;QAE5B,wDAAwD;QACxD,KAAK,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACnF,IAAI,cAAc,EAAE,OAAO,EAAE,CAAC;gBAC5B,IAAI,CAAC,eAAe,CAAC,GAAG,CACtB,YAA+B,EAC/B,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAC/C,CAAA;gBACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAA+B,EAAE,CAAC,CAAC,CAAA;gBAC3D,IAAI,CAAC,iBAAiB,CAAC,YAA+B,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QAED,iFAAiF;QACjF,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAA;QACtC,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACvB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACvB,SAAS,EAAE,IAAI,CAAC,mBAAmB,EAAE;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;SAC7C,CAAC,CAAA;IACJ,CAAC;IAED,uCAAuC;IACvC,aAAa;QACX,OAAO,IAAI,CAAC,WAAW,CAAA;IACzB,CAAC;IAED,4BAA4B;IAC5B,mBAAmB;QACjB,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;aACzC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;aACvC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAuB,CAAC,CAAA;IAC7C,CAAC;IAED,yDAAyD;IACzD,qBAAqB;QACnB,OAAO,IAAI,CAAC,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAClD,OAAO,OAAO,EAAE,UAAU,EAAE,IAAI,KAAK,CAAA;QACvC,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,WAAW,CAAC,QAAyB;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;YAC9C,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACrB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAA;YACjF,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAClD,MAAM,OAAO,GAAG,OAAO,EAAE,UAAU,EAAE,IAAI,KAAK,CAAA;YAE9C,OAAO;gBACL,OAAO;gBACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACjC,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;aACjF,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACjC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBAC/D,SAAS,EAAE,IAAI,IAAI,EAAE;aACtB,CAAA;QACH,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,iBAAiB,CAAC,QAAyB;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,cAAc,GAAG,MAAM,EAAE,cAAc,IAAI,EAAE,CAAA;QAEnD,OAAO;YACL,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,KAAK;YACzC,WAAW,EAAE,UAAU,GAAG,cAAc;YACxC,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,UAAU;YAC1B,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,QAAQ;SAC9C,CAAA;IACH,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,QAAQ,CAAC,OAAmB;QAChC,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAA;QACnE,MAAM,aAAa,GAAsB,EAAE,CAAA;QAE3C,IAAI,eAAe,GAAG,QAAQ,CAAA;QAC9B,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW,CAAA;QAE5D,OAAO,QAAQ,GAAG,WAAW,EAAE,CAAC;YAC9B,QAAQ,EAAE,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,OAAO,CAAC,CAAA;gBAEpE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,aAAa,EAAE,CAAA;gBAC1D,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE;oBAClC,OAAO,EAAE,IAAI;oBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBACjC,IAAI,EAAE,QAAQ,CAAC,IAAI;iBACpB,CAAC,CAAA;gBAEF,OAAO;oBACL,GAAG,QAAQ;oBACX,YAAY,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC;oBACtC,aAAa,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;iBACpE,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,aAAa,EAAE,CAAA;gBAC1D,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;gBAC1F,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAA;gBAEjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA;gBAC3E,IAAI,gBAAgB,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;oBAC/C,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;oBACnC,eAAe,GAAG,gBAAgB,CAAA;oBAClC,SAAQ;gBACV,CAAC;gBAED,MAAM,KAAK,CAAA;YACb,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAA;IACvE,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,sBAAsB,CAAC,OAAe;QAC1C,IAAI,CAAC,iBAAiB,EAAE,CAAA;QACxB,OAAO,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/F,CAAC;IAED,2BAA2B;IAC3B,UAAU;QACR,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IACtC,CAAC;IAED,6BAA6B;IAC7B,oBAAoB;QAOlB,OAAO,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IAC/C,CAAC;IAED,wBAAwB;IACxB,KAAK;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;QAC3B,IAAI,CAAC,kBAAkB,EAAE,CAAA;QACzB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAA;QAC7B,IAAI,CAAC,eAAe,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,+EAA+E;IAC/E,kBAAkB;IAClB,+EAA+E;IAEvE,iBAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;QAC/E,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,8EAA8E;IACtE,KAAK,CAAC,uBAAuB;QACnC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAA;QAC7B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAA;IAC7E,CAAC;IAEO,gBAAgB,CAAC,OAAmB;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC9C,MAAM,MAAM,GAAG,cAAc,CAC3B,OAAO,EACP,SAAS,EACT,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,eAAe,EACpB,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAChC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CACnC,CAAA;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,mBAAmB,CAAA;QACjD,OAAO,MAAM,CAAC,QAAQ,CAAA;IACxB,CAAC;IAEO,qBAAqB,CAC3B,eAAgC,EAChC,KAAc;QAEd,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;YAAE,OAAO,IAAI,CAAA;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC9C,OAAO,mBAAmB,CACxB,eAAe,EACf,KAAK,EACL,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAClC,SAAS,CACV,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAyB,EACzB,OAAmB;QAEnB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;QAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,iBAAiB,CAAC,CAAA;QACxD,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,GAAG,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC;YACH,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;YAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACxC,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACpF,MAAM,YAAY,GAAG,GAAG,CAAA,CAAC,YAAY;YAErC,OAAO;gBACL,OAAO,EAAE,4BAA4B,QAAQ,GAAG;gBAChD,QAAQ;gBACR,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK;gBACpC,KAAK,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,EAAE;gBAC7E,IAAI,EAAE,aAAa,CAAC,MAAM,EAAE,WAAW,EAAE,YAAY,CAAC;gBACtD,SAAS;gBACT,YAAY,EAAE,KAAK;aACpB,CAAA;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAA;QAClE,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAyB;QACjD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE;YACjC,QAAQ;YACR,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,YAAY,EAAE,CAAC;YACf,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;IACJ,CAAC;IAEO,aAAa,CACnB,QAAyB,EACzB,MAA8D;QAE9D,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAClD,IAAI,CAAC,OAAO;YAAE,OAAM;QAEpB,OAAO,CAAC,aAAa,EAAE,CAAA;QACvB,OAAO,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;QAE9B,8BAA8B;QAC9B,OAAO,CAAC,YAAY;YAClB,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;gBACvE,OAAO,CAAC,aAAa,CAAA;QAEvB,6BAA6B;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAClD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAA;YAC3C,OAAO,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAA;YAC5C,OAAO,CAAC,WAAW,GAAG,CAAC,GAAG,cAAc,CAAC,SAAS,CAAA;QACpD,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAA;QAClC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QAC/C,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,WAAW,CAAA;QAEjC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAiC,EAAE;IAEnC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAC7C,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAA;IAC3B,OAAO,QAAQ,CAAA;AACjB,CAAC"}

package/dist/testing/MultiLLMProvider.metrics.d.ts

@@ -0,0 +1,33 @@
+/**
+ * @fileoverview Multi-LLM Provider Metrics Aggregation and Compatibility Testing
+ * @module @skillsmith/core/testing/MultiLLMProvider.metrics
+ * @see SMI-1523: Configure multi-LLM provider chain
+ * @see SMI-2741: Split from MultiLLMProvider.ts to meet 500-line standard
+ *
+ * Standalone functions for metrics aggregation and skill compatibility testing,
+ * extracted to keep MultiLLMProvider.ts within the 500-line limit.
+ */
+import type { LLMProviderType, ProviderMetrics, SkillCompatibilityResult, LLMRequest, LLMResponse } from './MultiLLMProvider.types.js';
+/**
+ * Aggregate metrics across all providers into a summary object
+ *
+ * @param providerMetrics - Map of per-provider metrics
+ * @returns Aggregated totals and averages
+ */
+export declare function aggregateMetrics(providerMetrics: Map<LLMProviderType, ProviderMetrics>): {
+    totalRequests: number;
+    totalCost: number;
+    avgLatencyMs: number;
+    avgSuccessRate: number;
+    providerBreakdown: Record<LLMProviderType, number>;
+};
+/**
+ * Run a skill compatibility test across enabled providers
+ *
+ * @param skillId - Skill identifier to test
+ * @param enabledProviders - List of enabled providers to test against
+ * @param completer - Async function that executes an LLM request
+ * @returns Compatibility result with per-provider scores and overall assessment
+ */
+export declare function runCompatibilityTest(skillId: string, enabledProviders: LLMProviderType[], completer: (request: LLMRequest) => Promise<LLMResponse>): Promise<SkillCompatibilityResult>;
+//# sourceMappingURL=MultiLLMProvider.metrics.d.ts.map

package/dist/testing/MultiLLMProvider.metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MultiLLMProvider.metrics.d.ts","sourceRoot":"","sources":["../../src/testing/MultiLLMProvider.metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EACf,wBAAwB,EACxB,UAAU,EACV,WAAW,EACZ,MAAM,6BAA6B,CAAA;AAOpC;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,eAAe,EAAE,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC,GAAG;IACxF,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,iBAAiB,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;CACnD,CAwBA;AAMD;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,gBAAgB,EAAE,eAAe,EAAE,EACnC,SAAS,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,WAAW,CAAC,GACvD,OAAO,CAAC,wBAAwB,CAAC,CA0CnC"}

package/dist/testing/MultiLLMProvider.metrics.js

@@ -0,0 +1,87 @@
+/**
+ * @fileoverview Multi-LLM Provider Metrics Aggregation and Compatibility Testing
+ * @module @skillsmith/core/testing/MultiLLMProvider.metrics
+ * @see SMI-1523: Configure multi-LLM provider chain
+ * @see SMI-2741: Split from MultiLLMProvider.ts to meet 500-line standard
+ *
+ * Standalone functions for metrics aggregation and skill compatibility testing,
+ * extracted to keep MultiLLMProvider.ts within the 500-line limit.
+ */
+import { calculateCompatibilityScore } from './MultiLLMProvider.helpers.js';
+// ============================================================================
+// Aggregated Metrics
+// ============================================================================
+/**
+ * Aggregate metrics across all providers into a summary object
+ *
+ * @param providerMetrics - Map of per-provider metrics
+ * @returns Aggregated totals and averages
+ */
+export function aggregateMetrics(providerMetrics) {
+    let totalRequests = 0, totalCost = 0, totalLatency = 0, totalSuccessRate = 0;
+    const providerBreakdown = {};
+    for (const [provider, metrics] of providerMetrics) {
+        totalRequests += metrics.totalRequests;
+        totalCost += metrics.totalCost;
+        totalLatency += metrics.avgLatencyMs * metrics.totalRequests;
+        totalSuccessRate += metrics.successRate;
+        providerBreakdown[provider] = metrics.totalRequests;
+    }
+    const providerCount = providerMetrics.size;
+    return {
+        totalRequests,
+        totalCost,
+        avgLatencyMs: totalRequests > 0 ? totalLatency / totalRequests : 0,
+        avgSuccessRate: providerCount > 0 ? totalSuccessRate / providerCount : 0,
+        providerBreakdown,
+    };
+}
+// ============================================================================
+// Skill Compatibility Testing
+// ============================================================================
+/**
+ * Run a skill compatibility test across enabled providers
+ *
+ * @param skillId - Skill identifier to test
+ * @param enabledProviders - List of enabled providers to test against
+ * @param completer - Async function that executes an LLM request
+ * @returns Compatibility result with per-provider scores and overall assessment
+ */
+export async function runCompatibilityTest(skillId, enabledProviders, completer) {
+    const results = {};
+    const testPrompt = `Analyze if you can effectively help a user with a skill called "${skillId}".
+    Respond with a brief assessment of your capability.`;
+    for (const provider of enabledProviders) {
+        const startTime = Date.now();
+        try {
+            const response = await completer({
+                messages: [{ role: 'user', content: testPrompt }],
+                provider,
+                maxTokens: 100,
+            });
+            results[provider] = {
+                compatible: true,
+                score: calculateCompatibilityScore(response),
+                latencyMs: response.latencyMs,
+            };
+        }
+        catch (error) {
+            results[provider] = {
+                compatible: false,
+                score: 0,
+                latencyMs: Date.now() - startTime,
+                error: error instanceof Error ? error.message : 'Unknown error',
+            };
+        }
+    }
+    const scores = Object.values(results)
+        .filter((r) => r.compatible)
+        .map((r) => r.score);
+    const overallScore = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+    const recommendedProviders = Object.entries(results)
+        .filter(([, r]) => r.compatible && r.score >= 0.7)
+        .sort(([, a], [, b]) => b.score - a.score)
+        .map(([p]) => p);
+    return { skillId, results, overallScore, recommendedProviders, testedAt: new Date() };
+}
+//# sourceMappingURL=MultiLLMProvider.metrics.js.map

package/dist/testing/MultiLLMProvider.metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MultiLLMProvider.metrics.js","sourceRoot":"","sources":["../../src/testing/MultiLLMProvider.metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAA;AAE3E,+EAA+E;AAC/E,qBAAqB;AACrB,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,eAAsD;IAOrF,IAAI,aAAa,GAAG,CAAC,EACnB,SAAS,GAAG,CAAC,EACb,YAAY,GAAG,CAAC,EAChB,gBAAgB,GAAG,CAAC,CAAA;IACtB,MAAM,iBAAiB,GAAoC,EAAqC,CAAA;IAEhG,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC;QAClD,aAAa,IAAI,OAAO,CAAC,aAAa,CAAA;QACtC,SAAS,IAAI,OAAO,CAAC,SAAS,CAAA;QAC9B,YAAY,IAAI,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,aAAa,CAAA;QAC5D,gBAAgB,IAAI,OAAO,CAAC,WAAW,CAAA;QACvC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,aAAa,CAAA;IACrD,CAAC;IAED,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAA;IAE1C,OAAO;QACL,aAAa;QACb,SAAS;QACT,YAAY,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAClE,cAAc,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACxE,iBAAiB;KAClB,CAAA;AACH,CAAC;AAED,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAe,EACf,gBAAmC,EACnC,SAAwD;IAExD,MAAM,OAAO,GAAwC,EAAyC,CAAA;IAE9F,MAAM,UAAU,GAAG,mEAAmE,OAAO;wDACvC,CAAA;IAEtD,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC;gBAC/B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;gBACjD,QAAQ;gBACR,SAAS,EAAE,GAAG;aACf,CAAC,CAAA;YAEF,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAClB,UAAU,EAAE,IAAI;gBAChB,KAAK,EAAE,2BAA2B,CAAC,QAAQ,CAAC;gBAC5C,SAAS,EAAE,QAAQ,CAAC,SAAS;aAC9B,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAClB,UAAU,EAAE,KAAK;gBACjB,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACjC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAChE,CAAA;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IACtB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAE9F,MAAM,oBAAoB,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SACjD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC;SACjD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACzC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAoB,CAAC,CAAA;IAErC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE,EAAE,CAAA;AACvF,CAAC"}