@skillsmith/core 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/src/analysis/types.d.ts +2 -0
- package/dist/src/analysis/types.d.ts.map +1 -1
- package/dist/src/analysis/types.js +13 -1
- package/dist/src/analysis/types.js.map +1 -1
- package/dist/src/analytics/AnalyticsRepository.d.ts +4 -0
- package/dist/src/analytics/AnalyticsRepository.d.ts.map +1 -1
- package/dist/src/analytics/AnalyticsRepository.js +26 -44
- package/dist/src/analytics/AnalyticsRepository.js.map +1 -1
- package/dist/src/analytics/schema.d.ts +1 -1
- package/dist/src/analytics/schema.d.ts.map +1 -1
- package/dist/src/analytics/schema.js +68 -0
- package/dist/src/analytics/schema.js.map +1 -1
- package/dist/src/api/client.d.ts +33 -29
- package/dist/src/api/client.d.ts.map +1 -1
- package/dist/src/api/client.js +15 -10
- package/dist/src/api/client.js.map +1 -1
- package/dist/src/billing/BillingService.d.ts +139 -0
- package/dist/src/billing/BillingService.d.ts.map +1 -0
- package/dist/src/billing/BillingService.js +393 -0
- package/dist/src/billing/BillingService.js.map +1 -0
- package/dist/src/billing/GDPRComplianceService.d.ts +176 -0
- package/dist/src/billing/GDPRComplianceService.d.ts.map +1 -0
- package/dist/src/billing/GDPRComplianceService.js +361 -0
- package/dist/src/billing/GDPRComplianceService.js.map +1 -0
- package/dist/src/billing/StripeClient.d.ts +177 -0
- package/dist/src/billing/StripeClient.d.ts.map +1 -0
- package/dist/src/billing/StripeClient.js +462 -0
- package/dist/src/billing/StripeClient.js.map +1 -0
- package/dist/src/billing/StripeReconciliationJob.d.ts +95 -0
- package/dist/src/billing/StripeReconciliationJob.d.ts.map +1 -0
- package/dist/src/billing/StripeReconciliationJob.js +405 -0
- package/dist/src/billing/StripeReconciliationJob.js.map +1 -0
- package/dist/src/billing/StripeWebhookHandler.d.ts +92 -0
- package/dist/src/billing/StripeWebhookHandler.d.ts.map +1 -0
- package/dist/src/billing/StripeWebhookHandler.js +409 -0
- package/dist/src/billing/StripeWebhookHandler.js.map +1 -0
- package/dist/src/billing/index.d.ts +18 -0
- package/dist/src/billing/index.d.ts.map +1 -0
- package/dist/src/billing/index.js +19 -0
- package/dist/src/billing/index.js.map +1 -0
- package/dist/src/billing/types.d.ts +266 -0
- package/dist/src/billing/types.d.ts.map +1 -0
- package/dist/src/billing/types.js +23 -0
- package/dist/src/billing/types.js.map +1 -0
- package/dist/src/embeddings/hnsw-store.d.ts +568 -0
- package/dist/src/embeddings/hnsw-store.d.ts.map +1 -0
- package/dist/src/embeddings/hnsw-store.js +805 -0
- package/dist/src/embeddings/hnsw-store.js.map +1 -0
- package/dist/src/embeddings/index.d.ts +2 -0
- package/dist/src/embeddings/index.d.ts.map +1 -1
- package/dist/src/embeddings/index.js +2 -0
- package/dist/src/embeddings/index.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/learning/PatternStore.d.ts +457 -0
- package/dist/src/learning/PatternStore.d.ts.map +1 -0
- package/dist/src/learning/PatternStore.js +893 -0
- package/dist/src/learning/PatternStore.js.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.d.ts +403 -0
- package/dist/src/learning/ReasoningBankIntegration.d.ts.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.js +627 -0
- package/dist/src/learning/ReasoningBankIntegration.js.map +1 -0
- package/dist/src/learning/index.d.ts +15 -0
- package/dist/src/learning/index.d.ts.map +1 -0
- package/dist/src/learning/index.js +15 -0
- package/dist/src/learning/index.js.map +1 -0
- package/dist/src/routing/SONARouter.d.ts +154 -0
- package/dist/src/routing/SONARouter.d.ts.map +1 -0
- package/dist/src/routing/SONARouter.js +679 -0
- package/dist/src/routing/SONARouter.js.map +1 -0
- package/dist/src/routing/index.d.ts +9 -0
- package/dist/src/routing/index.d.ts.map +1 -0
- package/dist/src/routing/index.js +10 -0
- package/dist/src/routing/index.js.map +1 -0
- package/dist/src/routing/types.d.ts +331 -0
- package/dist/src/routing/types.d.ts.map +1 -0
- package/dist/src/routing/types.js +203 -0
- package/dist/src/routing/types.js.map +1 -0
- package/dist/src/scripts/__tests__/scan-imported-skills.test.js +5 -0
- package/dist/src/scripts/__tests__/scan-imported-skills.test.js.map +1 -1
- package/dist/src/security/SkillSandbox.d.ts +156 -0
- package/dist/src/security/SkillSandbox.d.ts.map +1 -0
- package/dist/src/security/SkillSandbox.js +303 -0
- package/dist/src/security/SkillSandbox.js.map +1 -0
- package/dist/src/security/index.d.ts +3 -1
- package/dist/src/security/index.d.ts.map +1 -1
- package/dist/src/security/index.js +5 -1
- package/dist/src/security/index.js.map +1 -1
- package/dist/src/security/rate-limiter/presets.d.ts +12 -0
- package/dist/src/security/rate-limiter/presets.d.ts.map +1 -1
- package/dist/src/security/rate-limiter/presets.js +12 -0
- package/dist/src/security/rate-limiter/presets.js.map +1 -1
- package/dist/src/security/sanitization.d.ts +85 -0
- package/dist/src/security/sanitization.d.ts.map +1 -1
- package/dist/src/security/sanitization.js +133 -0
- package/dist/src/security/sanitization.js.map +1 -1
- package/dist/src/security/scanner/SecurityScanner.d.ts +23 -0
- package/dist/src/security/scanner/SecurityScanner.d.ts.map +1 -1
- package/dist/src/security/scanner/SecurityScanner.js +232 -28
- package/dist/src/security/scanner/SecurityScanner.js.map +1 -1
- package/dist/src/security/scanner/patterns.d.ts +13 -0
- package/dist/src/security/scanner/patterns.d.ts.map +1 -1
- package/dist/src/security/scanner/patterns.js +51 -0
- package/dist/src/security/scanner/patterns.js.map +1 -1
- package/dist/src/security/scanner/types.d.ts +13 -1
- package/dist/src/security/scanner/types.d.ts.map +1 -1
- package/dist/src/security/scanner/weights.d.ts.map +1 -1
- package/dist/src/security/scanner/weights.js +1 -0
- package/dist/src/security/scanner/weights.js.map +1 -1
- package/dist/src/session/SessionManager.d.ts +7 -0
- package/dist/src/session/SessionManager.d.ts.map +1 -1
- package/dist/src/session/SessionManager.js +117 -10
- package/dist/src/session/SessionManager.js.map +1 -1
- package/dist/src/sync/SyncEngine.d.ts.map +1 -1
- package/dist/src/sync/SyncEngine.js +52 -32
- package/dist/src/sync/SyncEngine.js.map +1 -1
- package/dist/src/testing/MultiLLMProvider.d.ts +374 -0
- package/dist/src/testing/MultiLLMProvider.d.ts.map +1 -0
- package/dist/src/testing/MultiLLMProvider.js +720 -0
- package/dist/src/testing/MultiLLMProvider.js.map +1 -0
- package/dist/src/testing/index.d.ts +8 -0
- package/dist/src/testing/index.d.ts.map +1 -0
- package/dist/src/testing/index.js +9 -0
- package/dist/src/testing/index.js.map +1 -0
- package/dist/src/types.d.ts +3 -0
- package/dist/src/types.d.ts.map +1 -1
- package/dist/tests/SecurityScanner.test.js +337 -1
- package/dist/tests/SecurityScanner.test.js.map +1 -1
- package/dist/tests/billing/BillingService.test.d.ts +7 -0
- package/dist/tests/billing/BillingService.test.d.ts.map +1 -0
- package/dist/tests/billing/BillingService.test.js +168 -0
- package/dist/tests/billing/BillingService.test.js.map +1 -0
- package/dist/tests/billing/GDPRCompliance.test.d.ts +7 -0
- package/dist/tests/billing/GDPRCompliance.test.d.ts.map +1 -0
- package/dist/tests/billing/GDPRCompliance.test.js +195 -0
- package/dist/tests/billing/GDPRCompliance.test.js.map +1 -0
- package/dist/tests/billing/StripeReconciliation.test.d.ts +7 -0
- package/dist/tests/billing/StripeReconciliation.test.d.ts.map +1 -0
- package/dist/tests/billing/StripeReconciliation.test.js +266 -0
- package/dist/tests/billing/StripeReconciliation.test.js.map +1 -0
- package/dist/tests/billing/stripe-validators.test.d.ts +7 -0
- package/dist/tests/billing/stripe-validators.test.d.ts.map +1 -0
- package/dist/tests/billing/stripe-validators.test.js +107 -0
- package/dist/tests/billing/stripe-validators.test.js.map +1 -0
- package/dist/tests/embeddings/hnsw-store.test.d.ts +7 -0
- package/dist/tests/embeddings/hnsw-store.test.d.ts.map +1 -0
- package/dist/tests/embeddings/hnsw-store.test.js +295 -0
- package/dist/tests/embeddings/hnsw-store.test.js.map +1 -0
- package/dist/tests/integration/neural/e2e-learning.test.d.ts +17 -0
- package/dist/tests/integration/neural/e2e-learning.test.d.ts.map +1 -0
- package/dist/tests/integration/neural/e2e-learning.test.js +238 -0
- package/dist/tests/integration/neural/e2e-learning.test.js.map +1 -0
- package/dist/tests/integration/neural/helpers.d.ts +132 -0
- package/dist/tests/integration/neural/helpers.d.ts.map +1 -0
- package/dist/tests/integration/neural/helpers.js +287 -0
- package/dist/tests/integration/neural/helpers.js.map +1 -0
- package/dist/tests/integration/neural/personalization.test.d.ts +21 -0
- package/dist/tests/integration/neural/personalization.test.d.ts.map +1 -0
- package/dist/tests/integration/neural/personalization.test.js +304 -0
- package/dist/tests/integration/neural/personalization.test.js.map +1 -0
- package/dist/tests/integration/neural/preference-learner.test.d.ts +23 -0
- package/dist/tests/integration/neural/preference-learner.test.d.ts.map +1 -0
- package/dist/tests/integration/neural/preference-learner.test.js +289 -0
- package/dist/tests/integration/neural/preference-learner.test.js.map +1 -0
- package/dist/tests/integration/neural/privacy.test.d.ts +19 -0
- package/dist/tests/integration/neural/privacy.test.d.ts.map +1 -0
- package/dist/tests/integration/neural/privacy.test.js +249 -0
- package/dist/tests/integration/neural/privacy.test.js.map +1 -0
- package/dist/tests/integration/neural/setup.d.ts +175 -0
- package/dist/tests/integration/neural/setup.d.ts.map +1 -0
- package/dist/tests/integration/neural/setup.js +487 -0
- package/dist/tests/integration/neural/setup.js.map +1 -0
- package/dist/tests/integration/neural/signal-collection.test.d.ts +21 -0
- package/dist/tests/integration/neural/signal-collection.test.d.ts.map +1 -0
- package/dist/tests/integration/neural/signal-collection.test.js +232 -0
- package/dist/tests/integration/neural/signal-collection.test.js.map +1 -0
- package/dist/tests/learning/PatternStore.test.d.ts +8 -0
- package/dist/tests/learning/PatternStore.test.d.ts.map +1 -0
- package/dist/tests/learning/PatternStore.test.js +589 -0
- package/dist/tests/learning/PatternStore.test.js.map +1 -0
- package/dist/tests/learning/ReasoningBankIntegration.test.d.ts +8 -0
- package/dist/tests/learning/ReasoningBankIntegration.test.d.ts.map +1 -0
- package/dist/tests/learning/ReasoningBankIntegration.test.js +269 -0
- package/dist/tests/learning/ReasoningBankIntegration.test.js.map +1 -0
- package/dist/tests/routing/SONARouter.test.d.ts +8 -0
- package/dist/tests/routing/SONARouter.test.d.ts.map +1 -0
- package/dist/tests/routing/SONARouter.test.js +400 -0
- package/dist/tests/routing/SONARouter.test.js.map +1 -0
- package/dist/tests/security/ContinuousSecurity.test.js +10 -12
- package/dist/tests/security/ContinuousSecurity.test.js.map +1 -1
- package/dist/tests/security/SkillSandbox.test.d.ts +8 -0
- package/dist/tests/security/SkillSandbox.test.d.ts.map +1 -0
- package/dist/tests/security/SkillSandbox.test.js +321 -0
- package/dist/tests/security/SkillSandbox.test.js.map +1 -0
- package/dist/tests/sync/SyncEngine.test.js +4 -2
- package/dist/tests/sync/SyncEngine.test.js.map +1 -1
- package/dist/tests/testing/MultiLLMProvider.test.d.ts +14 -0
- package/dist/tests/testing/MultiLLMProvider.test.d.ts.map +1 -0
- package/dist/tests/testing/MultiLLMProvider.test.js +438 -0
- package/dist/tests/testing/MultiLLMProvider.test.js.map +1 -0
- package/package.json +16 -3
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Multi-LLM Provider Chain for Skill Compatibility Testing
|
|
3
|
+
* @module @skillsmith/core/testing/MultiLLMProvider
|
|
4
|
+
* @see SMI-1523: Configure multi-LLM provider chain
|
|
5
|
+
*
|
|
6
|
+
* Provides a unified interface for testing skills across multiple LLM providers:
|
|
7
|
+
* - Claude (Anthropic) - Primary
|
|
8
|
+
* - GPT (OpenAI) - Fallback 1
|
|
9
|
+
* - Gemini (Google) - Fallback 2
|
|
10
|
+
* - Cohere (Command) - Fallback 3
|
|
11
|
+
* - Ollama (Local) - Fallback 4
|
|
12
|
+
*
|
|
13
|
+
* Features:
|
|
14
|
+
* - Automatic failover with configurable strategy
|
|
15
|
+
* - Health monitoring and circuit breaker pattern
|
|
16
|
+
* - Cost-aware provider selection
|
|
17
|
+
* - Skill compatibility testing across providers
|
|
18
|
+
*/
|
|
19
|
+
import { EventEmitter } from 'events';
|
|
20
|
+
/**
|
|
21
|
+
* Supported LLM providers
|
|
22
|
+
*/
|
|
23
|
+
export type LLMProviderType = 'anthropic' | 'openai' | 'google' | 'cohere' | 'ollama';
|
|
24
|
+
/**
|
|
25
|
+
* Provider priority for selection
|
|
26
|
+
*/
|
|
27
|
+
export type ProviderPriority = 'quality' | 'speed' | 'cost' | 'privacy';
|
|
28
|
+
/**
|
|
29
|
+
* Load balancing strategies
|
|
30
|
+
*/
|
|
31
|
+
export type LoadBalanceStrategy = 'round-robin' | 'least-loaded' | 'latency-based' | 'cost-based';
|
|
32
|
+
/**
|
|
33
|
+
* Failover condition triggers
|
|
34
|
+
*/
|
|
35
|
+
export type FailoverCondition = 'error' | 'rate_limit' | 'timeout' | 'cost' | 'unavailable';
|
|
36
|
+
/**
|
|
37
|
+
* Provider configuration
|
|
38
|
+
*/
|
|
39
|
+
export interface ProviderConfig {
|
|
40
|
+
/** Provider type */
|
|
41
|
+
provider: LLMProviderType;
|
|
42
|
+
/** Model identifier */
|
|
43
|
+
model: string;
|
|
44
|
+
/** API key (from environment) */
|
|
45
|
+
apiKey?: string;
|
|
46
|
+
/** API URL (for Ollama/custom) */
|
|
47
|
+
apiUrl?: string;
|
|
48
|
+
/** Whether provider is enabled */
|
|
49
|
+
enabled: boolean;
|
|
50
|
+
/** Provider priority */
|
|
51
|
+
priority: ProviderPriority;
|
|
52
|
+
/** Maximum concurrent requests */
|
|
53
|
+
maxConcurrency?: number;
|
|
54
|
+
/** Request timeout in ms */
|
|
55
|
+
timeoutMs?: number;
|
|
56
|
+
/** Cost per 1K tokens (input) */
|
|
57
|
+
costPerInputToken?: number;
|
|
58
|
+
/** Cost per 1K tokens (output) */
|
|
59
|
+
costPerOutputToken?: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Fallback rule configuration
|
|
63
|
+
*/
|
|
64
|
+
export interface FallbackRule {
|
|
65
|
+
/** Condition that triggers fallback */
|
|
66
|
+
condition: FailoverCondition;
|
|
67
|
+
/** Error codes that trigger this rule */
|
|
68
|
+
errorCodes?: string[];
|
|
69
|
+
/** Providers to try in order */
|
|
70
|
+
fallbackProviders: LLMProviderType[];
|
|
71
|
+
/** Fallback models (optional) */
|
|
72
|
+
fallbackModels?: string[];
|
|
73
|
+
/** Whether to retry original provider after cooling */
|
|
74
|
+
retryOriginal: boolean;
|
|
75
|
+
/** Delay before retry in ms */
|
|
76
|
+
retryDelayMs?: number;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Fallback strategy configuration
|
|
80
|
+
*/
|
|
81
|
+
export interface FallbackStrategy {
|
|
82
|
+
/** Strategy name */
|
|
83
|
+
name: string;
|
|
84
|
+
/** Whether fallback is enabled */
|
|
85
|
+
enabled: boolean;
|
|
86
|
+
/** Fallback rules in priority order */
|
|
87
|
+
rules: FallbackRule[];
|
|
88
|
+
/** Maximum retry attempts */
|
|
89
|
+
maxAttempts: number;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Circuit breaker configuration
|
|
93
|
+
*/
|
|
94
|
+
export interface CircuitBreakerConfig {
|
|
95
|
+
/** Request timeout in ms */
|
|
96
|
+
timeoutMs: number;
|
|
97
|
+
/** Error percentage to open circuit */
|
|
98
|
+
errorThresholdPercentage: number;
|
|
99
|
+
/** Time before trying half-open in ms */
|
|
100
|
+
resetTimeoutMs: number;
|
|
101
|
+
/** Minimum requests before calculating error rate */
|
|
102
|
+
volumeThreshold: number;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Multi-LLM provider configuration
|
|
106
|
+
*/
|
|
107
|
+
export interface MultiLLMProviderConfig {
|
|
108
|
+
/** Provider configurations (optional, defaults available) */
|
|
109
|
+
providers?: Partial<Record<LLMProviderType, ProviderConfig>>;
|
|
110
|
+
/** Default provider (optional, defaults to 'anthropic') */
|
|
111
|
+
defaultProvider?: LLMProviderType;
|
|
112
|
+
/** Fallback strategy */
|
|
113
|
+
fallbackStrategy?: FallbackStrategy;
|
|
114
|
+
/** Load balancing configuration */
|
|
115
|
+
loadBalancing?: {
|
|
116
|
+
enabled: boolean;
|
|
117
|
+
strategy: LoadBalanceStrategy;
|
|
118
|
+
};
|
|
119
|
+
/** Cost optimization */
|
|
120
|
+
costOptimization?: {
|
|
121
|
+
enabled: boolean;
|
|
122
|
+
maxCostPerRequest?: number;
|
|
123
|
+
preferredProviders?: LLMProviderType[];
|
|
124
|
+
};
|
|
125
|
+
/** Circuit breaker configuration */
|
|
126
|
+
circuitBreaker?: CircuitBreakerConfig;
|
|
127
|
+
/** Enable metrics collection */
|
|
128
|
+
enableMetrics?: boolean;
|
|
129
|
+
/** Enable V3 ProviderManager integration */
|
|
130
|
+
useV3Integration?: boolean;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Default multi-LLM provider configuration
|
|
134
|
+
*/
|
|
135
|
+
export declare const DEFAULT_MULTI_LLM_CONFIG: Required<Omit<MultiLLMProviderConfig, 'providers' | 'fallbackStrategy'>> & {
|
|
136
|
+
providers: Partial<Record<LLMProviderType, ProviderConfig>>;
|
|
137
|
+
fallbackStrategy: FallbackStrategy;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Provider health check result
|
|
141
|
+
*/
|
|
142
|
+
export interface HealthCheckResult {
|
|
143
|
+
/** Whether provider is healthy */
|
|
144
|
+
healthy: boolean;
|
|
145
|
+
/** Latency in ms */
|
|
146
|
+
latencyMs?: number;
|
|
147
|
+
/** Error message if unhealthy */
|
|
148
|
+
error?: string;
|
|
149
|
+
/** Check timestamp */
|
|
150
|
+
timestamp: Date;
|
|
151
|
+
/** Additional details */
|
|
152
|
+
details?: Record<string, unknown>;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Provider status
|
|
156
|
+
*/
|
|
157
|
+
export interface ProviderStatus {
|
|
158
|
+
/** Whether provider is available */
|
|
159
|
+
available: boolean;
|
|
160
|
+
/** Current load (0-1) */
|
|
161
|
+
currentLoad: number;
|
|
162
|
+
/** Queue length */
|
|
163
|
+
queueLength: number;
|
|
164
|
+
/** Active requests */
|
|
165
|
+
activeRequests: number;
|
|
166
|
+
/** Rate limit remaining */
|
|
167
|
+
rateLimitRemaining?: number;
|
|
168
|
+
/** Rate limit reset time */
|
|
169
|
+
rateLimitReset?: Date;
|
|
170
|
+
/** Circuit breaker state */
|
|
171
|
+
circuitState: 'closed' | 'open' | 'half-open';
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* LLM request
|
|
175
|
+
*/
|
|
176
|
+
export interface LLMRequest {
|
|
177
|
+
/** Messages */
|
|
178
|
+
messages: Array<{
|
|
179
|
+
role: 'system' | 'user' | 'assistant';
|
|
180
|
+
content: string;
|
|
181
|
+
}>;
|
|
182
|
+
/** Model override */
|
|
183
|
+
model?: string;
|
|
184
|
+
/** Provider override */
|
|
185
|
+
provider?: LLMProviderType;
|
|
186
|
+
/** Max tokens */
|
|
187
|
+
maxTokens?: number;
|
|
188
|
+
/** Temperature */
|
|
189
|
+
temperature?: number;
|
|
190
|
+
/** Cost constraints */
|
|
191
|
+
costConstraints?: {
|
|
192
|
+
maxCost: number;
|
|
193
|
+
};
|
|
194
|
+
/** Request metadata */
|
|
195
|
+
metadata?: Record<string, unknown>;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* LLM response
|
|
199
|
+
*/
|
|
200
|
+
export interface LLMResponse {
|
|
201
|
+
/** Response content */
|
|
202
|
+
content: string;
|
|
203
|
+
/** Provider that handled the request */
|
|
204
|
+
provider: LLMProviderType;
|
|
205
|
+
/** Model used */
|
|
206
|
+
model: string;
|
|
207
|
+
/** Token usage */
|
|
208
|
+
usage: {
|
|
209
|
+
inputTokens: number;
|
|
210
|
+
outputTokens: number;
|
|
211
|
+
totalTokens: number;
|
|
212
|
+
};
|
|
213
|
+
/** Cost estimate */
|
|
214
|
+
cost: number;
|
|
215
|
+
/** Latency in ms */
|
|
216
|
+
latencyMs: number;
|
|
217
|
+
/** Whether fallback was used */
|
|
218
|
+
usedFallback: boolean;
|
|
219
|
+
/** Fallback chain (if used) */
|
|
220
|
+
fallbackChain?: LLMProviderType[];
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Provider metrics
|
|
224
|
+
*/
|
|
225
|
+
export interface ProviderMetrics {
|
|
226
|
+
/** Provider */
|
|
227
|
+
provider: LLMProviderType;
|
|
228
|
+
/** Timestamp */
|
|
229
|
+
timestamp: Date;
|
|
230
|
+
/** Average latency in ms */
|
|
231
|
+
avgLatencyMs: number;
|
|
232
|
+
/** Error rate (0-1) */
|
|
233
|
+
errorRate: number;
|
|
234
|
+
/** Success rate (0-1) */
|
|
235
|
+
successRate: number;
|
|
236
|
+
/** Current load (0-1) */
|
|
237
|
+
load: number;
|
|
238
|
+
/** Total cost */
|
|
239
|
+
totalCost: number;
|
|
240
|
+
/** Total requests */
|
|
241
|
+
totalRequests: number;
|
|
242
|
+
/** Availability (0-1) */
|
|
243
|
+
availability: number;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Skill compatibility test result
|
|
247
|
+
*/
|
|
248
|
+
export interface SkillCompatibilityResult {
|
|
249
|
+
/** Skill ID */
|
|
250
|
+
skillId: string;
|
|
251
|
+
/** Results by provider */
|
|
252
|
+
results: Record<LLMProviderType, {
|
|
253
|
+
compatible: boolean;
|
|
254
|
+
score: number;
|
|
255
|
+
latencyMs: number;
|
|
256
|
+
error?: string;
|
|
257
|
+
notes?: string;
|
|
258
|
+
}>;
|
|
259
|
+
/** Overall compatibility score (0-1) */
|
|
260
|
+
overallScore: number;
|
|
261
|
+
/** Recommended providers */
|
|
262
|
+
recommendedProviders: LLMProviderType[];
|
|
263
|
+
/** Test timestamp */
|
|
264
|
+
testedAt: Date;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Multi-LLM Provider for skill compatibility testing
|
|
268
|
+
*
|
|
269
|
+
* Provides unified access to multiple LLM providers with automatic failover,
|
|
270
|
+
* load balancing, and cost optimization.
|
|
271
|
+
*
|
|
272
|
+
* @example
|
|
273
|
+
* ```typescript
|
|
274
|
+
* const provider = new MultiLLMProvider()
|
|
275
|
+
* await provider.initialize()
|
|
276
|
+
*
|
|
277
|
+
* // Complete a request with automatic failover
|
|
278
|
+
* const response = await provider.complete({
|
|
279
|
+
* messages: [{ role: 'user', content: 'Explain this skill' }]
|
|
280
|
+
* })
|
|
281
|
+
*
|
|
282
|
+
* // Test skill compatibility across providers
|
|
283
|
+
* const compatibility = await provider.testSkillCompatibility('commit')
|
|
284
|
+
* ```
|
|
285
|
+
*/
|
|
286
|
+
export declare class MultiLLMProvider extends EventEmitter {
|
|
287
|
+
private config;
|
|
288
|
+
private initialized;
|
|
289
|
+
private circuitBreakers;
|
|
290
|
+
private providerMetrics;
|
|
291
|
+
private activeRequests;
|
|
292
|
+
private roundRobinIndex;
|
|
293
|
+
private v3ProviderManager;
|
|
294
|
+
constructor(config?: MultiLLMProviderConfig);
|
|
295
|
+
/**
|
|
296
|
+
* Initialize the multi-LLM provider
|
|
297
|
+
*/
|
|
298
|
+
initialize(): Promise<void>;
|
|
299
|
+
/**
|
|
300
|
+
* Check if provider is initialized
|
|
301
|
+
*/
|
|
302
|
+
isInitialized(): boolean;
|
|
303
|
+
/**
|
|
304
|
+
* Attempt to initialize V3 ProviderManager integration
|
|
305
|
+
*/
|
|
306
|
+
private initializeV3Integration;
|
|
307
|
+
/**
|
|
308
|
+
* Get enabled providers
|
|
309
|
+
*/
|
|
310
|
+
getEnabledProviders(): LLMProviderType[];
|
|
311
|
+
/**
|
|
312
|
+
* Get available providers (enabled + circuit closed)
|
|
313
|
+
*/
|
|
314
|
+
getAvailableProviders(): LLMProviderType[];
|
|
315
|
+
/**
|
|
316
|
+
* Check provider health
|
|
317
|
+
*/
|
|
318
|
+
healthCheck(provider: LLMProviderType): Promise<HealthCheckResult>;
|
|
319
|
+
/**
|
|
320
|
+
* Get provider status
|
|
321
|
+
*/
|
|
322
|
+
getProviderStatus(provider: LLMProviderType): ProviderStatus;
|
|
323
|
+
/**
|
|
324
|
+
* Complete a request with automatic failover
|
|
325
|
+
*/
|
|
326
|
+
complete(request: LLMRequest): Promise<LLMResponse>;
|
|
327
|
+
/**
|
|
328
|
+
* Test skill compatibility across providers
|
|
329
|
+
*/
|
|
330
|
+
testSkillCompatibility(skillId: string): Promise<SkillCompatibilityResult>;
|
|
331
|
+
/**
|
|
332
|
+
* Get provider metrics
|
|
333
|
+
*/
|
|
334
|
+
getMetrics(): Map<LLMProviderType, ProviderMetrics>;
|
|
335
|
+
/**
|
|
336
|
+
* Get aggregated metrics
|
|
337
|
+
*/
|
|
338
|
+
getAggregatedMetrics(): {
|
|
339
|
+
totalRequests: number;
|
|
340
|
+
totalCost: number;
|
|
341
|
+
avgLatencyMs: number;
|
|
342
|
+
avgSuccessRate: number;
|
|
343
|
+
providerBreakdown: Record<LLMProviderType, number>;
|
|
344
|
+
};
|
|
345
|
+
/**
|
|
346
|
+
* Close and cleanup
|
|
347
|
+
*
|
|
348
|
+
* After close, the provider cannot be used until initialize() is called again.
|
|
349
|
+
*/
|
|
350
|
+
close(): void;
|
|
351
|
+
private ensureInitialized;
|
|
352
|
+
private selectProvider;
|
|
353
|
+
private selectRoundRobin;
|
|
354
|
+
private selectLeastLoaded;
|
|
355
|
+
private selectByLatency;
|
|
356
|
+
private selectByCost;
|
|
357
|
+
private getFallbackProvider;
|
|
358
|
+
private getErrorCondition;
|
|
359
|
+
private executeRequest;
|
|
360
|
+
private simulateRequest;
|
|
361
|
+
private estimateTokens;
|
|
362
|
+
private calculateCost;
|
|
363
|
+
private calculateCompatibilityScore;
|
|
364
|
+
private initializeMetrics;
|
|
365
|
+
private updateMetrics;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Create and initialize a MultiLLMProvider instance
|
|
369
|
+
*
|
|
370
|
+
* @param config - Configuration options
|
|
371
|
+
* @returns Initialized MultiLLMProvider
|
|
372
|
+
*/
|
|
373
|
+
export declare function createMultiLLMProvider(config?: MultiLLMProviderConfig): Promise<MultiLLMProvider>;
|
|
374
|
+
//# sourceMappingURL=MultiLLMProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MultiLLMProvider.d.ts","sourceRoot":"","sources":["../../../src/testing/MultiLLMProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAMrC;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAA;AAEZ;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAA;AAEvE;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG,cAAc,GAAG,eAAe,GAAG,YAAY,CAAA;AAEjG;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,aAAa,CAAA;AAE3F;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,eAAe,CAAA;IAEzB,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAA;IAEb,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf,kCAAkC;IAClC,OAAO,EAAE,OAAO,CAAA;IAEhB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAA;IAE1B,kCAAkC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,iCAAiC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAE1B,kCAAkC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,SAAS,EAAE,iBAAiB,CAAA;IAE5B,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IAErB,gCAAgC;IAChC,iBAAiB,EAAE,eAAe,EAAE,CAAA;IAEpC,iCAAiC;IACjC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;IAEzB,uDAAuD;IACvD,aAAa,EAAE,OAAO,CAAA;IAEtB,+BAA+B;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAA;IAEZ,kCAAkC;IAClC,OAAO,EAAE,OAAO,CAAA;IAEhB,uCAAuC;IACvC,KAAK,EAAE,YAAY,EAAE,CAAA;IAErB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAA;IAEjB,uCAAuC;IACvC,wBAAwB,EAAE,MAAM,CAAA;IAEhC,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAA;IAEtB,qDAAqD;IACrD,eAAe,EAAE,MAAM,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC,CAAA;IAE5D,2DAA2D;IAC3D,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC,wBAAwB;IACxB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAEnC,mCAAmC;IACnC,aAAa,CAAC,EAAE;QACd,OAAO,EAAE,OAAO,CAAA;QAChB,QAAQ,EAAE,mBAAmB,CAAA;KAC9B,CAAA;IAED,wBAAwB;IACxB,gBAAgB,CAAC,EAAE;QACjB,OAAO,EAAE,OAAO,CAAA;QAChB,iBAAiB,CAAC,EAAE,MAAM,CAAA;QAC1B,kBAAkB,CAAC,EAAE,eAAe,EAAE,CAAA;KACvC,CAAA;IAED,oCAAoC;IACpC,cAAc,CAAC,EAAE,oBAAoB,CAAA;IAErC,gCAAgC;IAChC,aAAa,CAAC,EAAE,OAAO,CAAA;IAEvB,4CAA4C;IAC5C,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAC7C,IAAI,CAAC,sBAAsB,EAAE,WAAW,GAAG,kBAAkB,CAAC,CAC/D,GAAG;IACF,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC,CAAA;IAC3D,gBAAgB,EAAE,gBAAgB,CAAA;CAuGnC,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,OAAO,EAAE,OAAO,CAAA;IAEhB,oBAAoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd,sBAAsB;IACtB,SAAS,EAAE,IAAI,CAAA;IAEf,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oCAAoC;IACpC,SAAS,EAAE,OAAO,CAAA;IAElB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IAEnB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAA;IAEnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAA;IAEtB,2BAA2B;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAE3B,4BAA4B;IAC5B,cAAc,CAAC,EAAE,IAAI,CAAA;IAErB,4BAA4B;IAC5B,YAAY,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAA;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,eAAe;IACf,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAA;QACrC,OAAO,EAAE,MAAM,CAAA;KAChB,CAAC,CAAA;IAEF,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd,wBAAwB;IACxB,QAAQ,CAAC,EAAE,eAAe,CAAA;IAE1B,iBAAiB;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,uBAAuB;IACvB,eAAe,CAAC,EAAE;QAChB,OAAO,EAAE,MAAM,CAAA;KAChB,CAAA;IAED,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAA;IAEf,wCAAwC;IACxC,QAAQ,EAAE,eAAe,CAAA;IAEzB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAA;IAEb,kBAAkB;IAClB,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAA;QACnB,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;IAED,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAA;IAEZ,oBAAoB;IACpB,SAAS,EAAE,MAAM,CAAA;IAEjB,gCAAgC;IAChC,YAAY,EAAE,OAAO,CAAA;IAErB,+BAA+B;IAC/B,aAAa,CAAC,EAAE,eAAe,EAAE,CAAA;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,eAAe;IACf,QAAQ,EAAE,eAAe,CAAA;IAEzB,gBAAgB;IAChB,SAAS,EAAE,IAAI,CAAA;IAEf,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAA;IAEpB,uBAAuB;IACvB,SAAS,EAAE,MAAM,CAAA;IAEjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IAEnB,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAA;IAEZ,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IAEjB,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAA;IAErB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,eAAe;IACf,OAAO,EAAE,MAAM,CAAA;IAEf,0BAA0B;IAC1B,OAAO,EAAE,MAAM,CACb,eAAe,EACf;QACE,UAAU,EAAE,OAAO,CAAA;QACnB,KAAK,EAAE,MAAM,CAAA;QACb,SAAS,EAAE,MAAM,CAAA;QACjB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,CACF,CAAA;IAED,wCAAwC;IACxC,YAAY,EAAE,MAAM,CAAA;IAEpB,4BAA4B;IAC5B,oBAAoB,EAAE,eAAe,EAAE,CAAA;IAEvC,qBAAqB;IACrB,QAAQ,EAAE,IAAI,CAAA;CACf;AA6ED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,MAAM,CAGb;IACD,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,eAAe,CAAkD;IACzE,OAAO,CAAC,eAAe,CAAmD;IAC1E,OAAO,CAAC,cAAc,CAA0C;IAChE,OAAO,CAAC,eAAe,CAAI;IAG3B,OAAO,CAAC,iBAAiB,CAAgB;gBAE7B,MAAM,GAAE,sBAA2B;IAa/C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjC;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;YACW,uBAAuB;IAsBrC;;OAEG;IACH,mBAAmB,IAAI,eAAe,EAAE;IAMxC;;OAEG;IACH,qBAAqB,IAAI,eAAe,EAAE;IAO1C;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAsCxE;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,eAAe,GAAG,cAAc;IAe5D;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAuDzD;;OAEG;IACG,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAuDhF;;OAEG;IACH,UAAU,IAAI,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC;IAInD;;OAEG;IACH,oBAAoB,IAAI;QACtB,aAAa,EAAE,MAAM,CAAA;QACrB,SAAS,EAAE,MAAM,CAAA;QACjB,YAAY,EAAE,MAAM,CAAA;QACpB,cAAc,EAAE,MAAM,CAAA;QACtB,iBAAiB,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;KACnD;IA0BD;;;;OAIG;IACH,KAAK,IAAI,IAAI;IAcb,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,cAAc;IAwCtB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,iBAAiB;YAoBX,cAAc;YA4Cd,eAAe;IAY7B,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,aAAa;IAMrB,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,aAAa;CAkCtB;AAMD;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,GAAE,sBAA2B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAI3B"}
|