mcp-agent-foundry 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -2
- package/dist/cli/setup-wizard.d.ts.map +1 -1
- package/dist/cli/setup-wizard.js +873 -8
- package/dist/cli/setup-wizard.js.map +1 -1
- package/dist/cli/test-connection.d.ts +28 -0
- package/dist/cli/test-connection.d.ts.map +1 -1
- package/dist/cli/test-connection.js +335 -1
- package/dist/cli/test-connection.js.map +1 -1
- package/dist/cli.d.ts +13 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +169 -1
- package/dist/cli.js.map +1 -1
- package/dist/config/validator.d.ts +113 -0
- package/dist/config/validator.d.ts.map +1 -1
- package/dist/config/validator.js +113 -0
- package/dist/config/validator.js.map +1 -1
- package/dist/failover/health-tracker.d.ts +175 -0
- package/dist/failover/health-tracker.d.ts.map +1 -0
- package/dist/failover/health-tracker.js +350 -0
- package/dist/failover/health-tracker.js.map +1 -0
- package/dist/failover/index.d.ts +9 -0
- package/dist/failover/index.d.ts.map +1 -0
- package/dist/failover/index.js +9 -0
- package/dist/failover/index.js.map +1 -0
- package/dist/failover/orchestrator.d.ts +189 -0
- package/dist/failover/orchestrator.d.ts.map +1 -0
- package/dist/failover/orchestrator.js +488 -0
- package/dist/failover/orchestrator.js.map +1 -0
- package/dist/failover/pricing.d.ts +115 -0
- package/dist/failover/pricing.d.ts.map +1 -0
- package/dist/failover/pricing.js +283 -0
- package/dist/failover/pricing.js.map +1 -0
- package/dist/persistence/state-schema.d.ts +50 -0
- package/dist/persistence/state-schema.d.ts.map +1 -1
- package/dist/persistence/state-schema.js +2 -0
- package/dist/persistence/state-schema.js.map +1 -1
- package/dist/providers/fireworks.d.ts +23 -0
- package/dist/providers/fireworks.d.ts.map +1 -0
- package/dist/providers/fireworks.js +31 -0
- package/dist/providers/fireworks.js.map +1 -0
- package/dist/providers/groq.d.ts +23 -0
- package/dist/providers/groq.d.ts.map +1 -0
- package/dist/providers/groq.js +31 -0
- package/dist/providers/groq.js.map +1 -0
- package/dist/providers/kimi-code.d.ts +32 -0
- package/dist/providers/kimi-code.d.ts.map +1 -0
- package/dist/providers/kimi-code.js +46 -0
- package/dist/providers/kimi-code.js.map +1 -0
- package/dist/providers/kimi.d.ts +1 -1
- package/dist/providers/kimi.js +1 -1
- package/dist/providers/openrouter.d.ts +23 -0
- package/dist/providers/openrouter.d.ts.map +1 -0
- package/dist/providers/openrouter.js +31 -0
- package/dist/providers/openrouter.js.map +1 -0
- package/dist/providers/perplexity.d.ts +29 -0
- package/dist/providers/perplexity.d.ts.map +1 -0
- package/dist/providers/perplexity.js +51 -0
- package/dist/providers/perplexity.js.map +1 -0
- package/dist/providers/together.d.ts +23 -0
- package/dist/providers/together.d.ts.map +1 -0
- package/dist/providers/together.js +31 -0
- package/dist/providers/together.js.map +1 -0
- package/dist/router/engine.d.ts +21 -0
- package/dist/router/engine.d.ts.map +1 -1
- package/dist/router/engine.js +81 -21
- package/dist/router/engine.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +49 -0
- package/dist/server.js.map +1 -1
- package/dist/types.d.ts +52 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +14 -0
- package/dist/types.js.map +1 -1
- package/package.json +5 -2
- package/scripts/postinstall.js +78 -0
- package/scripts/preuninstall.js +67 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Health Tracker
|
|
3
|
+
*
|
|
4
|
+
* Tracks the health status of LLM providers to enable intelligent failover.
|
|
5
|
+
* Monitors success/failure rates, latency, and manages cooldown periods.
|
|
6
|
+
*/
|
|
7
|
+
import type { Logger } from '../observability/logger.js';
|
|
8
|
+
/**
|
|
9
|
+
* Health status of a provider.
|
|
10
|
+
*/
|
|
11
|
+
export type ProviderHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
|
|
12
|
+
/**
|
|
13
|
+
* Health information for a single provider.
|
|
14
|
+
*/
|
|
15
|
+
export interface ProviderHealth {
|
|
16
|
+
/** Provider name */
|
|
17
|
+
provider: string;
|
|
18
|
+
/** Current health status */
|
|
19
|
+
status: ProviderHealthStatus;
|
|
20
|
+
/** Timestamp of last health check */
|
|
21
|
+
lastCheck: Date;
|
|
22
|
+
/** Timestamp of last successful request */
|
|
23
|
+
lastSuccess: Date | null;
|
|
24
|
+
/** Last error details */
|
|
25
|
+
lastError: {
|
|
26
|
+
code: number | undefined;
|
|
27
|
+
message: string;
|
|
28
|
+
timestamp: Date;
|
|
29
|
+
} | null;
|
|
30
|
+
/** Number of consecutive failures */
|
|
31
|
+
consecutiveFailures: number;
|
|
32
|
+
/** Cooldown end time (provider should not be used until this time) */
|
|
33
|
+
cooldownUntil: Date | null;
|
|
34
|
+
/** Rolling average latency in ms */
|
|
35
|
+
latencyMs: number;
|
|
36
|
+
/** Total successful requests */
|
|
37
|
+
successCount: number;
|
|
38
|
+
/** Total failed requests */
|
|
39
|
+
failureCount: number;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Configuration for the health tracker.
|
|
43
|
+
*/
|
|
44
|
+
export interface HealthTrackerConfig {
|
|
45
|
+
/** Number of consecutive failures before marking unhealthy (default: 3) */
|
|
46
|
+
failureThreshold?: number;
|
|
47
|
+
/** Number of consecutive failures before marking degraded (default: 1) */
|
|
48
|
+
degradedThreshold?: number;
|
|
49
|
+
/** Cooldown duration in ms after marking unhealthy (default: 300000 = 5 min) */
|
|
50
|
+
cooldownMs?: number;
|
|
51
|
+
/** How often to run health checks in ms (default: 60000 = 1 min) */
|
|
52
|
+
healthCheckIntervalMs?: number;
|
|
53
|
+
/** Window size for rolling latency average (default: 10) */
|
|
54
|
+
latencyWindowSize?: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Tracks provider health for intelligent failover decisions.
|
|
58
|
+
*
|
|
59
|
+
* Features:
|
|
60
|
+
* - Tracks success/failure rates per provider
|
|
61
|
+
* - Manages cooldown periods for unhealthy providers
|
|
62
|
+
* - Provides rolling average latency metrics
|
|
63
|
+
* - Supports periodic health check callbacks
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* ```typescript
|
|
67
|
+
* const tracker = new ProviderHealthTracker(logger);
|
|
68
|
+
*
|
|
69
|
+
* // Record outcomes
|
|
70
|
+
* tracker.markSuccess('openai', 150);
|
|
71
|
+
* tracker.markFailure('deepseek', new Error('Rate limited'), 429);
|
|
72
|
+
*
|
|
73
|
+
* // Check before using
|
|
74
|
+
* if (tracker.isHealthy('openai')) {
|
|
75
|
+
* // Safe to use
|
|
76
|
+
* }
|
|
77
|
+
*
|
|
78
|
+
* // Get cooldown info
|
|
79
|
+
* const remaining = tracker.getCooldownRemaining('deepseek');
|
|
80
|
+
* ```
|
|
81
|
+
*/
|
|
82
|
+
export declare class ProviderHealthTracker {
|
|
83
|
+
private readonly health;
|
|
84
|
+
private readonly latencyHistory;
|
|
85
|
+
private readonly config;
|
|
86
|
+
private readonly logger;
|
|
87
|
+
private healthCheckInterval;
|
|
88
|
+
private healthCheckCallback;
|
|
89
|
+
constructor(logger: Logger, config?: HealthTrackerConfig);
|
|
90
|
+
/**
|
|
91
|
+
* Record a successful request for a provider.
|
|
92
|
+
*/
|
|
93
|
+
markSuccess(provider: string, latencyMs: number): void;
|
|
94
|
+
/**
|
|
95
|
+
* Record a failed request for a provider.
|
|
96
|
+
*/
|
|
97
|
+
markFailure(provider: string, error: Error, statusCode?: number): void;
|
|
98
|
+
/**
|
|
99
|
+
* Check if a provider is healthy and not in cooldown.
|
|
100
|
+
*/
|
|
101
|
+
isHealthy(provider: string): boolean;
|
|
102
|
+
/**
|
|
103
|
+
* Check if a provider is available (healthy or degraded, not in cooldown).
|
|
104
|
+
*/
|
|
105
|
+
isAvailable(provider: string): boolean;
|
|
106
|
+
/**
|
|
107
|
+
* Get remaining cooldown time in ms (0 if not in cooldown).
|
|
108
|
+
*/
|
|
109
|
+
getCooldownRemaining(provider: string): number;
|
|
110
|
+
/**
|
|
111
|
+
* Get health info for a provider.
|
|
112
|
+
*/
|
|
113
|
+
getHealth(provider: string): ProviderHealth | undefined;
|
|
114
|
+
/**
|
|
115
|
+
* Get health info for all tracked providers.
|
|
116
|
+
*/
|
|
117
|
+
getAllHealth(): Map<string, ProviderHealth>;
|
|
118
|
+
/**
|
|
119
|
+
* Get providers sorted by health and latency (best first).
|
|
120
|
+
*/
|
|
121
|
+
getHealthyProviders(providerNames: string[]): string[];
|
|
122
|
+
/**
|
|
123
|
+
* Reset health for a provider (e.g., after config change).
|
|
124
|
+
*/
|
|
125
|
+
resetHealth(provider: string): void;
|
|
126
|
+
/**
|
|
127
|
+
* Clear all health tracking data.
|
|
128
|
+
*/
|
|
129
|
+
clear(): void;
|
|
130
|
+
/**
|
|
131
|
+
* Start periodic health checks.
|
|
132
|
+
*/
|
|
133
|
+
startHealthCheckLoop(callback: (provider: string) => Promise<void>): void;
|
|
134
|
+
/**
|
|
135
|
+
* Stop periodic health checks.
|
|
136
|
+
*/
|
|
137
|
+
stopHealthCheckLoop(): void;
|
|
138
|
+
/**
|
|
139
|
+
* Serialize health state for persistence.
|
|
140
|
+
*/
|
|
141
|
+
serialize(): Array<{
|
|
142
|
+
provider: string;
|
|
143
|
+
status: ProviderHealthStatus;
|
|
144
|
+
consecutiveFailures: number;
|
|
145
|
+
cooldownUntil: number | null;
|
|
146
|
+
latencyMs: number;
|
|
147
|
+
successCount: number;
|
|
148
|
+
failureCount: number;
|
|
149
|
+
lastSuccessAt: number | null;
|
|
150
|
+
lastErrorAt: number | null;
|
|
151
|
+
lastErrorMessage: string | null;
|
|
152
|
+
lastErrorCode: number | null;
|
|
153
|
+
}>;
|
|
154
|
+
/**
|
|
155
|
+
* Restore health state from persistence.
|
|
156
|
+
*/
|
|
157
|
+
restore(data: Array<{
|
|
158
|
+
provider: string;
|
|
159
|
+
status: ProviderHealthStatus;
|
|
160
|
+
consecutiveFailures: number;
|
|
161
|
+
cooldownUntil: number | null;
|
|
162
|
+
latencyMs: number;
|
|
163
|
+
successCount: number;
|
|
164
|
+
failureCount: number;
|
|
165
|
+
lastSuccessAt: number | null;
|
|
166
|
+
lastErrorAt: number | null;
|
|
167
|
+
lastErrorMessage: string | null;
|
|
168
|
+
lastErrorCode: number | null;
|
|
169
|
+
}>): void;
|
|
170
|
+
private getOrCreateHealth;
|
|
171
|
+
private updateLatency;
|
|
172
|
+
private getAverageLatency;
|
|
173
|
+
private runHealthChecks;
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=health-tracker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"health-tracker.d.ts","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAMzD;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,qCAAqC;IACrC,SAAS,EAAE,IAAI,CAAC;IAChB,2CAA2C;IAC3C,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,yBAAyB;IACzB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,IAAI,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,qCAAqC;IACrC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sEAAsE;IACtE,aAAa,EAAE,IAAI,GAAG,IAAI,CAAC;IAC3B,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAkBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;IACjE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoC;IACnE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgC;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E,OAAO,CAAC,mBAAmB,CAAsD;gBAErE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB;IAKxD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IA6BtD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAmCtE;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAcpC;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IActC;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAU9C;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAIvD;;OAEG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAI3C;;OAEG;IACH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAkBtD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAOnC;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAgBzE;;OAEG;IACH,mBAAmB,IAAI,IAAI;IAU3B;;OAEG;IACH,SAAS,IAAI,KAAK,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,oBAAoB,CAAC;QAC7B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC;IAkCF;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,oBAAoB,CAAC;QAC7B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,GAAG,IAAI;IAiCT,OAAO,CAAC,iBAAiB;IAoBzB,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,iBAAiB;YAUX,eAAe;CA8B9B"}
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Health Tracker
|
|
3
|
+
*
|
|
4
|
+
* Tracks the health status of LLM providers to enable intelligent failover.
|
|
5
|
+
* Monitors success/failure rates, latency, and manages cooldown periods.
|
|
6
|
+
*/
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// Constants
|
|
9
|
+
// ============================================================================
|
|
10
|
+
const DEFAULT_CONFIG = {
|
|
11
|
+
failureThreshold: 3,
|
|
12
|
+
degradedThreshold: 1,
|
|
13
|
+
cooldownMs: 300000, // 5 minutes
|
|
14
|
+
healthCheckIntervalMs: 60000, // 1 minute
|
|
15
|
+
latencyWindowSize: 10,
|
|
16
|
+
};
|
|
17
|
+
// ============================================================================
|
|
18
|
+
// ProviderHealthTracker Class
|
|
19
|
+
// ============================================================================
|
|
20
|
+
/**
|
|
21
|
+
* Tracks provider health for intelligent failover decisions.
|
|
22
|
+
*
|
|
23
|
+
* Features:
|
|
24
|
+
* - Tracks success/failure rates per provider
|
|
25
|
+
* - Manages cooldown periods for unhealthy providers
|
|
26
|
+
* - Provides rolling average latency metrics
|
|
27
|
+
* - Supports periodic health check callbacks
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* ```typescript
|
|
31
|
+
* const tracker = new ProviderHealthTracker(logger);
|
|
32
|
+
*
|
|
33
|
+
* // Record outcomes
|
|
34
|
+
* tracker.markSuccess('openai', 150);
|
|
35
|
+
* tracker.markFailure('deepseek', new Error('Rate limited'), 429);
|
|
36
|
+
*
|
|
37
|
+
* // Check before using
|
|
38
|
+
* if (tracker.isHealthy('openai')) {
|
|
39
|
+
* // Safe to use
|
|
40
|
+
* }
|
|
41
|
+
*
|
|
42
|
+
* // Get cooldown info
|
|
43
|
+
* const remaining = tracker.getCooldownRemaining('deepseek');
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export class ProviderHealthTracker {
|
|
47
|
+
health = new Map();
|
|
48
|
+
latencyHistory = new Map();
|
|
49
|
+
config;
|
|
50
|
+
logger;
|
|
51
|
+
healthCheckInterval = null;
|
|
52
|
+
healthCheckCallback = null;
|
|
53
|
+
constructor(logger, config) {
|
|
54
|
+
this.logger = logger;
|
|
55
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Record a successful request for a provider.
|
|
59
|
+
*/
|
|
60
|
+
markSuccess(provider, latencyMs) {
|
|
61
|
+
const health = this.getOrCreateHealth(provider);
|
|
62
|
+
const now = new Date();
|
|
63
|
+
health.lastSuccess = now;
|
|
64
|
+
health.lastCheck = now;
|
|
65
|
+
health.consecutiveFailures = 0;
|
|
66
|
+
health.successCount++;
|
|
67
|
+
// Update rolling latency average
|
|
68
|
+
this.updateLatency(provider, latencyMs);
|
|
69
|
+
health.latencyMs = this.getAverageLatency(provider);
|
|
70
|
+
// Recover from unhealthy/degraded if in cooldown
|
|
71
|
+
if (health.cooldownUntil && now >= health.cooldownUntil) {
|
|
72
|
+
health.cooldownUntil = null;
|
|
73
|
+
}
|
|
74
|
+
// Update status based on recent performance
|
|
75
|
+
health.status = 'healthy';
|
|
76
|
+
this.logger.debug('Provider health: success', {
|
|
77
|
+
provider,
|
|
78
|
+
latencyMs,
|
|
79
|
+
status: health.status,
|
|
80
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Record a failed request for a provider.
|
|
85
|
+
*/
|
|
86
|
+
markFailure(provider, error, statusCode) {
|
|
87
|
+
const health = this.getOrCreateHealth(provider);
|
|
88
|
+
const now = new Date();
|
|
89
|
+
health.lastError = {
|
|
90
|
+
code: statusCode,
|
|
91
|
+
message: error.message,
|
|
92
|
+
timestamp: now,
|
|
93
|
+
};
|
|
94
|
+
health.lastCheck = now;
|
|
95
|
+
health.consecutiveFailures++;
|
|
96
|
+
health.failureCount++;
|
|
97
|
+
// Update status based on consecutive failures
|
|
98
|
+
if (health.consecutiveFailures >= this.config.failureThreshold) {
|
|
99
|
+
health.status = 'unhealthy';
|
|
100
|
+
health.cooldownUntil = new Date(now.getTime() + this.config.cooldownMs);
|
|
101
|
+
this.logger.warn('Provider marked unhealthy', {
|
|
102
|
+
provider,
|
|
103
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
104
|
+
cooldownUntil: health.cooldownUntil.toISOString(),
|
|
105
|
+
lastError: error.message,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
else if (health.consecutiveFailures >= this.config.degradedThreshold) {
|
|
109
|
+
health.status = 'degraded';
|
|
110
|
+
this.logger.info('Provider marked degraded', {
|
|
111
|
+
provider,
|
|
112
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
113
|
+
lastError: error.message,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Check if a provider is healthy and not in cooldown.
|
|
119
|
+
*/
|
|
120
|
+
isHealthy(provider) {
|
|
121
|
+
const health = this.health.get(provider);
|
|
122
|
+
if (!health) {
|
|
123
|
+
return true; // Assume healthy if never tracked
|
|
124
|
+
}
|
|
125
|
+
// Check cooldown
|
|
126
|
+
if (health.cooldownUntil && new Date() < health.cooldownUntil) {
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
return health.status === 'healthy';
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Check if a provider is available (healthy or degraded, not in cooldown).
|
|
133
|
+
*/
|
|
134
|
+
isAvailable(provider) {
|
|
135
|
+
const health = this.health.get(provider);
|
|
136
|
+
if (!health) {
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
// Check cooldown
|
|
140
|
+
if (health.cooldownUntil && new Date() < health.cooldownUntil) {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
return health.status !== 'unhealthy';
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Get remaining cooldown time in ms (0 if not in cooldown).
|
|
147
|
+
*/
|
|
148
|
+
getCooldownRemaining(provider) {
|
|
149
|
+
const health = this.health.get(provider);
|
|
150
|
+
if (!health?.cooldownUntil) {
|
|
151
|
+
return 0;
|
|
152
|
+
}
|
|
153
|
+
const remaining = health.cooldownUntil.getTime() - Date.now();
|
|
154
|
+
return Math.max(0, remaining);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Get health info for a provider.
|
|
158
|
+
*/
|
|
159
|
+
getHealth(provider) {
|
|
160
|
+
return this.health.get(provider);
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Get health info for all tracked providers.
|
|
164
|
+
*/
|
|
165
|
+
getAllHealth() {
|
|
166
|
+
return new Map(this.health);
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Get providers sorted by health and latency (best first).
|
|
170
|
+
*/
|
|
171
|
+
getHealthyProviders(providerNames) {
|
|
172
|
+
return providerNames
|
|
173
|
+
.filter((p) => this.isAvailable(p))
|
|
174
|
+
.sort((a, b) => {
|
|
175
|
+
const healthA = this.health.get(a);
|
|
176
|
+
const healthB = this.health.get(b);
|
|
177
|
+
// Healthy before degraded
|
|
178
|
+
if (healthA?.status === 'healthy' && healthB?.status !== 'healthy')
|
|
179
|
+
return -1;
|
|
180
|
+
if (healthA?.status !== 'healthy' && healthB?.status === 'healthy')
|
|
181
|
+
return 1;
|
|
182
|
+
// Lower latency first
|
|
183
|
+
const latencyA = healthA?.latencyMs ?? 0;
|
|
184
|
+
const latencyB = healthB?.latencyMs ?? 0;
|
|
185
|
+
return latencyA - latencyB;
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Reset health for a provider (e.g., after config change).
|
|
190
|
+
*/
|
|
191
|
+
resetHealth(provider) {
|
|
192
|
+
this.health.delete(provider);
|
|
193
|
+
this.latencyHistory.delete(provider);
|
|
194
|
+
this.logger.info('Provider health reset', { provider });
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Clear all health tracking data.
|
|
198
|
+
*/
|
|
199
|
+
clear() {
|
|
200
|
+
this.health.clear();
|
|
201
|
+
this.latencyHistory.clear();
|
|
202
|
+
this.logger.info('All provider health data cleared');
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Start periodic health checks.
|
|
206
|
+
*/
|
|
207
|
+
startHealthCheckLoop(callback) {
|
|
208
|
+
if (this.healthCheckInterval) {
|
|
209
|
+
return; // Already running
|
|
210
|
+
}
|
|
211
|
+
this.healthCheckCallback = callback;
|
|
212
|
+
this.healthCheckInterval = setInterval(() => void this.runHealthChecks(), this.config.healthCheckIntervalMs);
|
|
213
|
+
this.logger.info('Health check loop started', {
|
|
214
|
+
intervalMs: this.config.healthCheckIntervalMs,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Stop periodic health checks.
|
|
219
|
+
*/
|
|
220
|
+
stopHealthCheckLoop() {
|
|
221
|
+
if (this.healthCheckInterval) {
|
|
222
|
+
clearInterval(this.healthCheckInterval);
|
|
223
|
+
this.healthCheckInterval = null;
|
|
224
|
+
this.healthCheckCallback = null;
|
|
225
|
+
this.logger.info('Health check loop stopped');
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Serialize health state for persistence.
|
|
230
|
+
*/
|
|
231
|
+
serialize() {
|
|
232
|
+
const result = [];
|
|
233
|
+
for (const [provider, health] of this.health) {
|
|
234
|
+
result.push({
|
|
235
|
+
provider,
|
|
236
|
+
status: health.status,
|
|
237
|
+
consecutiveFailures: health.consecutiveFailures,
|
|
238
|
+
cooldownUntil: health.cooldownUntil?.getTime() ?? null,
|
|
239
|
+
latencyMs: health.latencyMs,
|
|
240
|
+
successCount: health.successCount,
|
|
241
|
+
failureCount: health.failureCount,
|
|
242
|
+
lastSuccessAt: health.lastSuccess?.getTime() ?? null,
|
|
243
|
+
lastErrorAt: health.lastError?.timestamp.getTime() ?? null,
|
|
244
|
+
lastErrorMessage: health.lastError?.message ?? null,
|
|
245
|
+
lastErrorCode: health.lastError?.code ?? null,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
return result;
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Restore health state from persistence.
|
|
252
|
+
*/
|
|
253
|
+
restore(data) {
|
|
254
|
+
for (const item of data) {
|
|
255
|
+
const health = {
|
|
256
|
+
provider: item.provider,
|
|
257
|
+
status: item.status,
|
|
258
|
+
lastCheck: new Date(),
|
|
259
|
+
lastSuccess: item.lastSuccessAt ? new Date(item.lastSuccessAt) : null,
|
|
260
|
+
lastError: item.lastErrorAt
|
|
261
|
+
? {
|
|
262
|
+
code: item.lastErrorCode ?? undefined,
|
|
263
|
+
message: item.lastErrorMessage ?? 'Unknown error',
|
|
264
|
+
timestamp: new Date(item.lastErrorAt),
|
|
265
|
+
}
|
|
266
|
+
: null,
|
|
267
|
+
consecutiveFailures: item.consecutiveFailures,
|
|
268
|
+
cooldownUntil: item.cooldownUntil ? new Date(item.cooldownUntil) : null,
|
|
269
|
+
latencyMs: item.latencyMs,
|
|
270
|
+
successCount: item.successCount,
|
|
271
|
+
failureCount: item.failureCount,
|
|
272
|
+
};
|
|
273
|
+
this.health.set(item.provider, health);
|
|
274
|
+
}
|
|
275
|
+
this.logger.info('Provider health restored', {
|
|
276
|
+
providerCount: data.length,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
// ==========================================================================
|
|
280
|
+
// Private Methods
|
|
281
|
+
// ==========================================================================
|
|
282
|
+
getOrCreateHealth(provider) {
|
|
283
|
+
let health = this.health.get(provider);
|
|
284
|
+
if (!health) {
|
|
285
|
+
health = {
|
|
286
|
+
provider,
|
|
287
|
+
status: 'healthy',
|
|
288
|
+
lastCheck: new Date(),
|
|
289
|
+
lastSuccess: null,
|
|
290
|
+
lastError: null,
|
|
291
|
+
consecutiveFailures: 0,
|
|
292
|
+
cooldownUntil: null,
|
|
293
|
+
latencyMs: 0,
|
|
294
|
+
successCount: 0,
|
|
295
|
+
failureCount: 0,
|
|
296
|
+
};
|
|
297
|
+
this.health.set(provider, health);
|
|
298
|
+
}
|
|
299
|
+
return health;
|
|
300
|
+
}
|
|
301
|
+
updateLatency(provider, latencyMs) {
|
|
302
|
+
let history = this.latencyHistory.get(provider);
|
|
303
|
+
if (!history) {
|
|
304
|
+
history = [];
|
|
305
|
+
this.latencyHistory.set(provider, history);
|
|
306
|
+
}
|
|
307
|
+
history.push(latencyMs);
|
|
308
|
+
// Keep only the last N samples
|
|
309
|
+
if (history.length > this.config.latencyWindowSize) {
|
|
310
|
+
history.shift();
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
getAverageLatency(provider) {
|
|
314
|
+
const history = this.latencyHistory.get(provider);
|
|
315
|
+
if (!history || history.length === 0) {
|
|
316
|
+
return 0;
|
|
317
|
+
}
|
|
318
|
+
const sum = history.reduce((a, b) => a + b, 0);
|
|
319
|
+
return Math.round(sum / history.length);
|
|
320
|
+
}
|
|
321
|
+
async runHealthChecks() {
|
|
322
|
+
if (!this.healthCheckCallback) {
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
// Check providers that are in cooldown and might be ready to recover
|
|
326
|
+
const now = Date.now();
|
|
327
|
+
for (const [provider, health] of this.health) {
|
|
328
|
+
if (health.status === 'unhealthy' && health.cooldownUntil) {
|
|
329
|
+
if (now >= health.cooldownUntil.getTime()) {
|
|
330
|
+
this.logger.debug('Running health check for provider exiting cooldown', {
|
|
331
|
+
provider,
|
|
332
|
+
});
|
|
333
|
+
try {
|
|
334
|
+
await this.healthCheckCallback(provider);
|
|
335
|
+
// If the callback doesn't throw, consider it a success
|
|
336
|
+
this.logger.info('Provider recovered after cooldown', { provider });
|
|
337
|
+
}
|
|
338
|
+
catch (error) {
|
|
339
|
+
// Failed, will stay unhealthy and cooldown will be extended
|
|
340
|
+
this.logger.debug('Provider health check failed', {
|
|
341
|
+
provider,
|
|
342
|
+
error: error instanceof Error ? error.message : String(error),
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
//# sourceMappingURL=health-tracker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"health-tracker.js","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA2DH,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,cAAc,GAAkC;IACpD,gBAAgB,EAAE,CAAC;IACnB,iBAAiB,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,YAAY;IAChC,qBAAqB,EAAE,KAAK,EAAE,WAAW;IACzC,iBAAiB,EAAE,EAAE;CACtB,CAAC;AAEF,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,qBAAqB;IACf,MAAM,GAAgC,IAAI,GAAG,EAAE,CAAC;IAChD,cAAc,GAA0B,IAAI,GAAG,EAAE,CAAC;IAClD,MAAM,CAAgC;IACtC,MAAM,CAAS;IACxB,mBAAmB,GAA0C,IAAI,CAAC;IAClE,mBAAmB,GAAiD,IAAI,CAAC;IAEjF,YAAY,MAAc,EAAE,MAA4B;QACtD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,SAAiB;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;QACzB,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,iCAAiC;QACjC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEpD,iDAAiD;QACjD,IAAI,MAAM,CAAC,aAAa,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QAED,4CAA4C;QAC5C,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;QAE1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;YAC5C,QAAQ;YACR,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,KAAY,EAAE,UAAmB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,SAAS,GAAG;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,GAAG;SACf,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,8CAA8C;QAC9C,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC/D,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBAC5C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE;gBACjD,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAE3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC,kCAAkC;QACjD,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAAgB;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,aAAuB;QACzC,OAAO,aAAa;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEnC,0BAA0B;YAC1B,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC;YAE7E,sBAAsB;YACtB,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,OAAO,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAA6C;QAChE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;QACpC,IAAI,CAAC,mBAAmB,GAAG,WAAW,CACpC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,eAAe,EAAE,EACjC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAClC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;SAC9C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QAaP,MAAM,MAAM,GAYP,EAAE,CAAC;QAER,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,IAAI;gBACtD,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,IAAI;gBACpD,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,IAAI,IAAI;gBACnD,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,IAAI,IAAI;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAYN;QACA,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,MAAM,GAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,IAAI,CAAC,WAAW;oBACzB,CAAC,CAAC;wBACE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,SAAS;wBACrC,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;wBACjD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;qBACtC;oBACH,CAAC,CAAC,IAAI;gBACR,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;gBAC7C,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvE,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,iBAAiB,CAAC,QAAgB;QACxC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG;gBACP,QAAQ;gBACR,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,mBAAmB,EAAE,CAAC;gBACtB,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,CAAC;gBACZ,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;aAChB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAExB,+BAA+B;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1D,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,EAAE;wBACtE,QAAQ;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;wBACzC,uDAAuD;wBACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACtE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,4DAA4D;wBAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;4BAChD,QAAQ;4BACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAC9D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/failover/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/failover/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}
|