@ai-orchestration/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -5
- package/dist/core/interfaces.d.ts +80 -1
- package/dist/core/interfaces.d.ts.map +1 -1
- package/dist/core/metrics.d.ts +161 -0
- package/dist/core/metrics.d.ts.map +1 -0
- package/dist/core/metrics.js +276 -0
- package/dist/core/metrics.js.map +1 -0
- package/dist/core/orchestrator.d.ts +45 -3
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +271 -11
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/types.d.ts +39 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/factory/index.d.ts.map +1 -1
- package/dist/factory/index.js +27 -7
- package/dist/factory/index.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/providers/cerebras.d.ts.map +1 -1
- package/dist/providers/cerebras.js +22 -4
- package/dist/providers/cerebras.js.map +1 -1
- package/dist/providers/groq.d.ts.map +1 -1
- package/dist/providers/groq.js +22 -4
- package/dist/providers/groq.js.map +1 -1
- package/dist/providers/local.d.ts.map +1 -1
- package/dist/providers/local.js +9 -1
- package/dist/providers/local.js.map +1 -1
- package/dist/providers/openrouter.d.ts.map +1 -1
- package/dist/providers/openrouter.js +18 -2
- package/dist/providers/openrouter.js.map +1 -1
- package/docs/CHANGELOG.md +36 -0
- package/examples/README.md +205 -0
- package/examples/basic.ts +51 -17
- package/examples/chat-app/config.json +20 -1
- package/examples/chat-app/index.ts +41 -2
- package/examples/config.example.json +26 -4
- package/examples/language.ts +160 -0
- package/examples/metrics.ts +150 -0
- package/examples/strategies.ts +4 -2
- package/examples/test-local.ts +6 -6
- package/package.json +6 -2
package/README.md
CHANGED
|
@@ -267,20 +267,58 @@ For local models that expose an OpenAI-compatible API:
|
|
|
267
267
|
}
|
|
268
268
|
```
|
|
269
269
|
|
|
270
|
-
##
|
|
270
|
+
## Advanced Configuration
|
|
271
271
|
|
|
272
|
-
|
|
272
|
+
### Retry and Timeout Configuration
|
|
273
273
|
|
|
274
274
|
```typescript
|
|
275
275
|
const orchestrator = createOrchestrator({
|
|
276
276
|
providers: [...],
|
|
277
277
|
strategy: {...},
|
|
278
|
-
|
|
279
|
-
|
|
278
|
+
maxRetries: 3, // Maximum retry attempts (default: number of providers)
|
|
279
|
+
requestTimeout: 30000, // Global timeout in milliseconds (default: 30000)
|
|
280
|
+
retryDelay: 'exponential', // or number in milliseconds (default: 1000)
|
|
280
281
|
});
|
|
281
282
|
```
|
|
282
283
|
|
|
283
|
-
|
|
284
|
+
### Circuit Breaker
|
|
285
|
+
|
|
286
|
+
Automatically disable providers after consecutive failures:
|
|
287
|
+
|
|
288
|
+
```typescript
|
|
289
|
+
const orchestrator = createOrchestrator({
|
|
290
|
+
providers: [...],
|
|
291
|
+
strategy: {...},
|
|
292
|
+
circuitBreaker: {
|
|
293
|
+
enabled: true,
|
|
294
|
+
failureThreshold: 5, // Open circuit after 5 failures
|
|
295
|
+
resetTimeout: 60000, // Reset after 60 seconds
|
|
296
|
+
},
|
|
297
|
+
});
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
### Health Checks
|
|
301
|
+
|
|
302
|
+
Enhanced health check configuration:
|
|
303
|
+
|
|
304
|
+
```typescript
|
|
305
|
+
const orchestrator = createOrchestrator({
|
|
306
|
+
providers: [...],
|
|
307
|
+
strategy: {...},
|
|
308
|
+
healthCheck: {
|
|
309
|
+
enabled: true,
|
|
310
|
+
interval: 60000, // Check every 60 seconds
|
|
311
|
+
timeout: 5000, // Health check timeout (default: 5000ms)
|
|
312
|
+
maxConsecutiveFailures: 3, // Mark unhealthy after 3 failures (default: 3)
|
|
313
|
+
latencyThreshold: 10000, // Max latency in ms (default: 10000ms)
|
|
314
|
+
},
|
|
315
|
+
// Legacy format still supported:
|
|
316
|
+
// enableHealthChecks: true,
|
|
317
|
+
// healthCheckInterval: 60000,
|
|
318
|
+
});
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
Or manually check health:
|
|
284
322
|
|
|
285
323
|
```typescript
|
|
286
324
|
const health = await provider.checkHealth();
|
|
@@ -296,9 +334,95 @@ const response = await orchestrator.chat(messages, {
|
|
|
296
334
|
topP: 0.9,
|
|
297
335
|
topK: 40,
|
|
298
336
|
stopSequences: ['\n\n'],
|
|
337
|
+
responseLanguage: 'es', // Force response in Spanish
|
|
338
|
+
frequencyPenalty: 0.5, // Reduce repetition
|
|
339
|
+
presencePenalty: 0.3, // Encourage new topics
|
|
340
|
+
seed: 42, // For reproducible outputs
|
|
341
|
+
timeout: 30000, // Request timeout in milliseconds
|
|
342
|
+
user: 'user-123', // User identifier for tracking
|
|
343
|
+
});
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
### Available Chat Options
|
|
347
|
+
|
|
348
|
+
- **`temperature`**: Controls randomness (0.0 to 2.0)
|
|
349
|
+
- **`maxTokens`**: Maximum tokens in response
|
|
350
|
+
- **`topP`**: Nucleus sampling threshold
|
|
351
|
+
- **`topK`**: Top-K sampling
|
|
352
|
+
- **`stopSequences`**: Stop generation on these sequences
|
|
353
|
+
- **`responseLanguage`**: Force response language (see below)
|
|
354
|
+
- **`frequencyPenalty`**: Penalize frequent tokens (-2.0 to 2.0)
|
|
355
|
+
- **`presencePenalty`**: Penalize existing tokens (-2.0 to 2.0)
|
|
356
|
+
- **`seed`**: Seed for reproducible outputs
|
|
357
|
+
- **`timeout`**: Request timeout in milliseconds (overrides global timeout)
|
|
358
|
+
- **`user`**: User identifier for tracking/rate limiting
|
|
359
|
+
|
|
360
|
+
### Forcing Response Language
|
|
361
|
+
|
|
362
|
+
You can force the AI to respond in a specific language using the `responseLanguage` option:
|
|
363
|
+
|
|
364
|
+
```typescript
|
|
365
|
+
// Using ISO 639-1 language codes
|
|
366
|
+
const response = await orchestrator.chat(messages, {
|
|
367
|
+
responseLanguage: 'es', // Spanish
|
|
368
|
+
// or 'en', 'fr', 'de', 'it', 'pt', 'ja', 'zh', 'ru', etc.
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
// Using full language names
|
|
372
|
+
const response2 = await orchestrator.chat(messages, {
|
|
373
|
+
responseLanguage: 'spanish', // Also works
|
|
374
|
+
// or 'english', 'french', 'german', 'italian', etc.
|
|
299
375
|
});
|
|
300
376
|
```
|
|
301
377
|
|
|
378
|
+
**How it works**: When `responseLanguage` is specified, the framework automatically prepends a system message instructing the model to respond in the specified language. If you already have a system message, the language instruction will be prepended to it.
|
|
379
|
+
|
|
380
|
+
**Supported languages**: Spanish, English, French, German, Italian, Portuguese, Japanese, Chinese, Russian, Korean, Arabic, Hindi, Dutch, Polish, Swedish, Turkish (and more via ISO 639-1 codes).
|
|
381
|
+
|
|
382
|
+
## Metrics and Analytics
|
|
383
|
+
|
|
384
|
+
Track provider usage, costs, and strategy effectiveness:
|
|
385
|
+
|
|
386
|
+
```typescript
|
|
387
|
+
const orchestrator = createOrchestrator({
|
|
388
|
+
providers: [...],
|
|
389
|
+
strategy: {...},
|
|
390
|
+
enableMetrics: true, // Enabled by default
|
|
391
|
+
onMetricsEvent: (event) => {
|
|
392
|
+
// Optional: Real-time event tracking
|
|
393
|
+
console.log('Event:', event.type, event.providerId);
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
// Make some requests...
|
|
398
|
+
|
|
399
|
+
// Get overall metrics
|
|
400
|
+
const metrics = orchestrator.getMetrics().getOrchestratorMetrics();
|
|
401
|
+
console.log('Total Requests:', metrics.totalRequests);
|
|
402
|
+
console.log('Total Cost:', metrics.totalCost);
|
|
403
|
+
console.log('Error Rate:', metrics.errorRate);
|
|
404
|
+
|
|
405
|
+
// Get provider-specific metrics
|
|
406
|
+
const providerMetrics = orchestrator.getMetrics().getProviderMetrics('groq-1');
|
|
407
|
+
console.log('Provider Requests:', providerMetrics?.totalRequests);
|
|
408
|
+
console.log('Provider Cost:', providerMetrics?.totalCost);
|
|
409
|
+
console.log('Success Rate:', providerMetrics?.successfulRequests / providerMetrics?.totalRequests);
|
|
410
|
+
|
|
411
|
+
// Get strategy metrics
|
|
412
|
+
const strategyMetrics = orchestrator.getMetrics().getStrategyMetrics();
|
|
413
|
+
console.log('Selections by Provider:', strategyMetrics.selectionsByProvider);
|
|
414
|
+
console.log('Average Selection Time:', strategyMetrics.averageSelectionTime);
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
### Available Metrics
|
|
418
|
+
|
|
419
|
+
- **Provider Metrics**: Requests, success/failure rates, latency, token usage, costs
|
|
420
|
+
- **Strategy Metrics**: Selection counts, distribution, selection time
|
|
421
|
+
- **Overall Metrics**: Total requests, costs, error rates, requests per minute
|
|
422
|
+
- **Request History**: Detailed history with filtering options
|
|
423
|
+
|
|
424
|
+
See `examples/metrics.ts` for a complete example.
|
|
425
|
+
|
|
302
426
|
## Extensibility
|
|
303
427
|
|
|
304
428
|
### Adding a New Provider
|
|
@@ -74,6 +74,48 @@ export interface StrategyConfig {
|
|
|
74
74
|
type: string;
|
|
75
75
|
[key: string]: unknown;
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Circuit breaker configuration
|
|
79
|
+
*/
|
|
80
|
+
export interface CircuitBreakerConfig {
|
|
81
|
+
/**
|
|
82
|
+
* Enable circuit breaker functionality
|
|
83
|
+
*/
|
|
84
|
+
enabled?: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Number of consecutive failures before opening the circuit
|
|
87
|
+
*/
|
|
88
|
+
failureThreshold?: number;
|
|
89
|
+
/**
|
|
90
|
+
* Time in milliseconds before attempting to reset the circuit
|
|
91
|
+
*/
|
|
92
|
+
resetTimeout?: number;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Health check configuration
|
|
96
|
+
*/
|
|
97
|
+
export interface HealthCheckConfig {
|
|
98
|
+
/**
|
|
99
|
+
* Interval between health checks in milliseconds
|
|
100
|
+
*/
|
|
101
|
+
interval?: number;
|
|
102
|
+
/**
|
|
103
|
+
* Timeout for individual health checks in milliseconds
|
|
104
|
+
*/
|
|
105
|
+
timeout?: number;
|
|
106
|
+
/**
|
|
107
|
+
* Maximum consecutive failures before marking provider as unhealthy
|
|
108
|
+
*/
|
|
109
|
+
maxConsecutiveFailures?: number;
|
|
110
|
+
/**
|
|
111
|
+
* Maximum latency threshold in milliseconds. Providers exceeding this will be marked unhealthy.
|
|
112
|
+
*/
|
|
113
|
+
latencyThreshold?: number;
|
|
114
|
+
/**
|
|
115
|
+
* Enable periodic health checks
|
|
116
|
+
*/
|
|
117
|
+
enabled?: boolean;
|
|
118
|
+
}
|
|
77
119
|
/**
|
|
78
120
|
* Main orchestrator configuration
|
|
79
121
|
*/
|
|
@@ -81,7 +123,44 @@ export interface OrchestratorConfig {
|
|
|
81
123
|
providers: ProviderConfig[];
|
|
82
124
|
strategy: StrategyConfig;
|
|
83
125
|
defaultOptions?: ChatOptions;
|
|
84
|
-
|
|
126
|
+
/**
|
|
127
|
+
* Maximum number of retry attempts before giving up.
|
|
128
|
+
* Defaults to the number of providers if not specified.
|
|
129
|
+
*/
|
|
130
|
+
maxRetries?: number;
|
|
131
|
+
/**
|
|
132
|
+
* Global timeout for all requests in milliseconds.
|
|
133
|
+
* Defaults to 30000 (30 seconds) if not specified.
|
|
134
|
+
*/
|
|
135
|
+
requestTimeout?: number;
|
|
136
|
+
/**
|
|
137
|
+
* Delay between retries in milliseconds, or 'exponential' for exponential backoff.
|
|
138
|
+
* Defaults to 1000 (1 second) if not specified.
|
|
139
|
+
*/
|
|
140
|
+
retryDelay?: number | 'exponential';
|
|
141
|
+
/**
|
|
142
|
+
* Circuit breaker configuration
|
|
143
|
+
*/
|
|
144
|
+
circuitBreaker?: CircuitBreakerConfig;
|
|
145
|
+
/**
|
|
146
|
+
* Health check configuration
|
|
147
|
+
*/
|
|
148
|
+
healthCheck?: HealthCheckConfig;
|
|
149
|
+
/**
|
|
150
|
+
* Enable metrics collection (default: true)
|
|
151
|
+
*/
|
|
152
|
+
enableMetrics?: boolean;
|
|
153
|
+
/**
|
|
154
|
+
* Callback for metrics events
|
|
155
|
+
*/
|
|
156
|
+
onMetricsEvent?: import('./metrics.js').MetricsCallback;
|
|
157
|
+
/**
|
|
158
|
+
* @deprecated Use healthCheck.enabled instead
|
|
159
|
+
*/
|
|
85
160
|
enableHealthChecks?: boolean;
|
|
161
|
+
/**
|
|
162
|
+
* @deprecated Use healthCheck.interval instead
|
|
163
|
+
*/
|
|
164
|
+
healthCheckInterval?: number;
|
|
86
165
|
}
|
|
87
166
|
//# sourceMappingURL=interfaces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../src/core/interfaces.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAGpB,YAAY,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,gBAAgB,GACjB,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC;IAEvC;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAE5E;;OAEG;IACH,UAAU,CACR,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,CACJ,SAAS,EAAE,SAAS,EAAE,EACtB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IAE7B;;OAEG;IACH,MAAM,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CAC1E;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,QAAQ,EAAE,cAAc,CAAC;IACzB,cAAc,CAAC,EAAE,WAAW,CAAC;IAC7B,
|
|
1
|
+
{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../src/core/interfaces.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAGpB,YAAY,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,gBAAgB,GACjB,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC;IAEvC;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAE5E;;OAEG;IACH,UAAU,CACR,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,CACJ,SAAS,EAAE,SAAS,EAAE,EACtB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IAE7B;;OAEG;IACH,MAAM,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CAC1E;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,QAAQ,EAAE,cAAc,CAAC;IACzB,cAAc,CAAC,EAAE,WAAW,CAAC;IAC7B;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IACpC;;OAEG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC;;OAEG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,cAAc,EAAE,eAAe,CAAC;IACxD;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metrics and analytics system for the AI Orchestration Framework
|
|
3
|
+
*/
|
|
4
|
+
import type { AIService, ChatResponse } from './interfaces.js';
|
|
5
|
+
import type { ProviderHealth } from './types.js';
|
|
6
|
+
/**
|
|
7
|
+
* Metrics for a single provider
|
|
8
|
+
*/
|
|
9
|
+
export interface ProviderMetrics {
|
|
10
|
+
providerId: string;
|
|
11
|
+
providerName: string;
|
|
12
|
+
model?: string;
|
|
13
|
+
totalRequests: number;
|
|
14
|
+
successfulRequests: number;
|
|
15
|
+
failedRequests: number;
|
|
16
|
+
totalTokens: {
|
|
17
|
+
prompt: number;
|
|
18
|
+
completion: number;
|
|
19
|
+
total: number;
|
|
20
|
+
};
|
|
21
|
+
totalCost: number;
|
|
22
|
+
averageLatency: number;
|
|
23
|
+
lastUsed?: Date;
|
|
24
|
+
lastSuccess?: Date;
|
|
25
|
+
lastFailure?: Date;
|
|
26
|
+
healthHistory: Array<{
|
|
27
|
+
timestamp: Date;
|
|
28
|
+
healthy: boolean;
|
|
29
|
+
latency?: number;
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Strategy selection metrics
|
|
34
|
+
*/
|
|
35
|
+
export interface StrategyMetrics {
|
|
36
|
+
totalSelections: number;
|
|
37
|
+
selectionsByProvider: Map<string, number>;
|
|
38
|
+
selectionsByStrategy: Map<string, number>;
|
|
39
|
+
averageSelectionTime: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Overall orchestrator metrics
|
|
43
|
+
*/
|
|
44
|
+
export interface OrchestratorMetrics {
|
|
45
|
+
totalRequests: number;
|
|
46
|
+
successfulRequests: number;
|
|
47
|
+
failedRequests: number;
|
|
48
|
+
totalCost: number;
|
|
49
|
+
providerMetrics: Map<string, ProviderMetrics>;
|
|
50
|
+
strategyMetrics: StrategyMetrics;
|
|
51
|
+
averageRequestLatency: number;
|
|
52
|
+
requestsPerMinute: number;
|
|
53
|
+
errorRate: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Event types for metrics tracking
|
|
57
|
+
*/
|
|
58
|
+
export type MetricsEvent = {
|
|
59
|
+
type: 'provider_selected';
|
|
60
|
+
providerId: string;
|
|
61
|
+
strategy: string;
|
|
62
|
+
timestamp: Date;
|
|
63
|
+
selectionTime?: number;
|
|
64
|
+
} | {
|
|
65
|
+
type: 'request_started';
|
|
66
|
+
providerId: string;
|
|
67
|
+
timestamp: Date;
|
|
68
|
+
} | {
|
|
69
|
+
type: 'request_success';
|
|
70
|
+
providerId: string;
|
|
71
|
+
response: ChatResponse;
|
|
72
|
+
latency: number;
|
|
73
|
+
timestamp: Date;
|
|
74
|
+
} | {
|
|
75
|
+
type: 'request_failure';
|
|
76
|
+
providerId: string;
|
|
77
|
+
error: Error;
|
|
78
|
+
latency?: number;
|
|
79
|
+
timestamp: Date;
|
|
80
|
+
} | {
|
|
81
|
+
type: 'health_check';
|
|
82
|
+
providerId: string;
|
|
83
|
+
health: ProviderHealth;
|
|
84
|
+
timestamp: Date;
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Callback function for metrics events
|
|
88
|
+
*/
|
|
89
|
+
export type MetricsCallback = (event: MetricsEvent) => void;
|
|
90
|
+
/**
|
|
91
|
+
* Metrics collector for tracking orchestrator usage
|
|
92
|
+
*/
|
|
93
|
+
export declare class MetricsCollector {
|
|
94
|
+
private providerMetrics;
|
|
95
|
+
private strategyMetrics;
|
|
96
|
+
private requestHistory;
|
|
97
|
+
private callbacks;
|
|
98
|
+
private selectionTimes;
|
|
99
|
+
/**
|
|
100
|
+
* Register a callback for metrics events
|
|
101
|
+
*/
|
|
102
|
+
onEvent(callback: MetricsCallback): () => void;
|
|
103
|
+
/**
|
|
104
|
+
* Emit a metrics event
|
|
105
|
+
*/
|
|
106
|
+
private emit;
|
|
107
|
+
/**
|
|
108
|
+
* Record provider selection
|
|
109
|
+
*/
|
|
110
|
+
recordSelection(provider: AIService, strategy: string, selectionTime?: number): void;
|
|
111
|
+
/**
|
|
112
|
+
* Record request start
|
|
113
|
+
*/
|
|
114
|
+
recordRequestStart(provider: AIService): void;
|
|
115
|
+
/**
|
|
116
|
+
* Record successful request
|
|
117
|
+
*/
|
|
118
|
+
recordSuccess(provider: AIService, response: ChatResponse, latency: number): void;
|
|
119
|
+
/**
|
|
120
|
+
* Record failed request
|
|
121
|
+
*/
|
|
122
|
+
recordFailure(provider: AIService, error: Error, latency?: number): void;
|
|
123
|
+
/**
|
|
124
|
+
* Record health check
|
|
125
|
+
*/
|
|
126
|
+
recordHealthCheck(provider: AIService, health: ProviderHealth): void;
|
|
127
|
+
/**
|
|
128
|
+
* Get or create provider metrics
|
|
129
|
+
*/
|
|
130
|
+
private getOrCreateProviderMetrics;
|
|
131
|
+
/**
|
|
132
|
+
* Get metrics for a specific provider
|
|
133
|
+
*/
|
|
134
|
+
getProviderMetrics(providerId: string): ProviderMetrics | undefined;
|
|
135
|
+
/**
|
|
136
|
+
* Get all provider metrics
|
|
137
|
+
*/
|
|
138
|
+
getAllProviderMetrics(): Map<string, ProviderMetrics>;
|
|
139
|
+
/**
|
|
140
|
+
* Get strategy metrics
|
|
141
|
+
*/
|
|
142
|
+
getStrategyMetrics(): StrategyMetrics;
|
|
143
|
+
/**
|
|
144
|
+
* Get overall orchestrator metrics
|
|
145
|
+
*/
|
|
146
|
+
getOrchestratorMetrics(): OrchestratorMetrics;
|
|
147
|
+
/**
|
|
148
|
+
* Reset all metrics
|
|
149
|
+
*/
|
|
150
|
+
reset(): void;
|
|
151
|
+
/**
|
|
152
|
+
* Get request history (optionally filtered)
|
|
153
|
+
*/
|
|
154
|
+
getRequestHistory(options?: {
|
|
155
|
+
providerId?: string;
|
|
156
|
+
startTime?: Date;
|
|
157
|
+
endTime?: Date;
|
|
158
|
+
limit?: number;
|
|
159
|
+
}): typeof this.requestHistory;
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/core/metrics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,aAAa,EAAE,KAAK,CAAC;QACnB,SAAS,EAAE,IAAI,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,oBAAoB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9C,eAAe,EAAE,eAAe,CAAC;IACjC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IACE,IAAI,EAAE,iBAAiB,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;CACjB,GACD;IACE,IAAI,EAAE,iBAAiB,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB,GACD;IACE,IAAI,EAAE,iBAAiB,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;CACjB,GACD;IACE,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,EAAE,IAAI,CAAC;CACjB,CAAC;AAEN;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;AAE5D;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,eAAe,CAA2C;IAClE,OAAO,CAAC,eAAe,CAKrB;IACF,OAAO,CAAC,cAAc,CAOd;IACR,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,cAAc,CAAgB;IAEtC;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,IAAI;IAK9C;;OAEG;IACH,OAAO,CAAC,IAAI;IAUZ;;OAEG;IACH,eAAe,CACb,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,MAAM,EAChB,aAAa,CAAC,EAAE,MAAM,GACrB,IAAI;IA6BP;;OAEG;IACH,kBAAkB,CAAC,QAAQ,EAAE,SAAS,GAAG,IAAI;IAY7C;;OAEG;IACH,aAAa,CACX,QAAQ,EAAE,SAAS,EACnB,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,MAAM,GACd,IAAI;IA6DP;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAsBxE;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,IAAI;IAqBpE;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;OAEG;IACH,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAInE;;OAEG;IACH,qBAAqB,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC;IAIrD;;OAEG;IACH,kBAAkB,IAAI,eAAe;IAIrC;;OAEG;IACH,sBAAsB,IAAI,mBAAmB;IAuC7C;;OAEG;IACH,KAAK,IAAI,IAAI;IAYb;;OAEG;IACH,iBAAiB,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,OAAO,CAAC,EAAE,IAAI,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,IAAI,CAAC,cAAc;CAuB/B"}
|