mcp-agent-foundry 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +350 -52
- package/dist/background/index.d.ts +33 -0
- package/dist/background/index.d.ts.map +1 -0
- package/dist/background/index.js +33 -0
- package/dist/background/index.js.map +1 -0
- package/dist/background/task-runner.d.ts +177 -0
- package/dist/background/task-runner.d.ts.map +1 -0
- package/dist/background/task-runner.js +551 -0
- package/dist/background/task-runner.js.map +1 -0
- package/dist/background/types.d.ts +135 -0
- package/dist/background/types.d.ts.map +1 -0
- package/dist/background/types.js +8 -0
- package/dist/background/types.js.map +1 -0
- package/dist/failover/health-tracker.d.ts +81 -1
- package/dist/failover/health-tracker.d.ts.map +1 -1
- package/dist/failover/health-tracker.js +229 -1
- package/dist/failover/health-tracker.js.map +1 -1
- package/dist/hooks/hook-executor.d.ts +77 -0
- package/dist/hooks/hook-executor.d.ts.map +1 -0
- package/dist/hooks/hook-executor.js +308 -0
- package/dist/hooks/hook-executor.js.map +1 -0
- package/dist/hooks/hook-manager.d.ts +140 -0
- package/dist/hooks/hook-manager.d.ts.map +1 -0
- package/dist/hooks/hook-manager.js +520 -0
- package/dist/hooks/hook-manager.js.map +1 -0
- package/dist/hooks/index.d.ts +10 -0
- package/dist/hooks/index.d.ts.map +1 -0
- package/dist/hooks/index.js +10 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/types.d.ts +221 -0
- package/dist/hooks/types.d.ts.map +1 -0
- package/dist/hooks/types.js +31 -0
- package/dist/hooks/types.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/mcp/auto-mode.d.ts +221 -0
- package/dist/mcp/auto-mode.d.ts.map +1 -0
- package/dist/mcp/auto-mode.js +436 -0
- package/dist/mcp/auto-mode.js.map +1 -0
- package/dist/mcp/index.d.ts +14 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +22 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/tools/tasks/delete-task.d.ts +25 -0
- package/dist/mcp/tools/tasks/delete-task.d.ts.map +1 -0
- package/dist/mcp/tools/tasks/delete-task.js +148 -0
- package/dist/mcp/tools/tasks/delete-task.js.map +1 -0
- package/dist/mcp/tools/tasks/index.d.ts +2 -0
- package/dist/mcp/tools/tasks/index.d.ts.map +1 -1
- package/dist/mcp/tools/tasks/index.js +6 -0
- package/dist/mcp/tools/tasks/index.js.map +1 -1
- package/dist/observability/debug-logger.d.ts +209 -0
- package/dist/observability/debug-logger.d.ts.map +1 -0
- package/dist/observability/debug-logger.js +430 -0
- package/dist/observability/debug-logger.js.map +1 -0
- package/dist/observability/index.d.ts +12 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +12 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/logger.d.ts +180 -0
- package/dist/observability/logger.d.ts.map +1 -1
- package/dist/observability/logger.js +158 -0
- package/dist/observability/logger.js.map +1 -1
- package/dist/router/context-manager.d.ts +214 -1
- package/dist/router/context-manager.d.ts.map +1 -1
- package/dist/router/context-manager.js +759 -2
- package/dist/router/context-manager.js.map +1 -1
- package/dist/router/context-types.d.ts +182 -0
- package/dist/router/context-types.d.ts.map +1 -0
- package/dist/router/context-types.js +8 -0
- package/dist/router/context-types.js.map +1 -0
- package/dist/server.d.ts +41 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +165 -0
- package/dist/server.js.map +1 -1
- package/dist/skills/hot-reloader.d.ts +104 -0
- package/dist/skills/hot-reloader.d.ts.map +1 -0
- package/dist/skills/hot-reloader.js +314 -0
- package/dist/skills/hot-reloader.js.map +1 -0
- package/dist/skills/index.d.ts +14 -0
- package/dist/skills/index.d.ts.map +1 -0
- package/dist/skills/index.js +16 -0
- package/dist/skills/index.js.map +1 -0
- package/dist/skills/skill-executor.d.ts +96 -0
- package/dist/skills/skill-executor.d.ts.map +1 -0
- package/dist/skills/skill-executor.js +289 -0
- package/dist/skills/skill-executor.js.map +1 -0
- package/dist/skills/skill-loader.d.ts +147 -0
- package/dist/skills/skill-loader.d.ts.map +1 -0
- package/dist/skills/skill-loader.js +579 -0
- package/dist/skills/skill-loader.js.map +1 -0
- package/dist/skills/types.d.ts +198 -0
- package/dist/skills/types.d.ts.map +1 -0
- package/dist/skills/types.js +21 -0
- package/dist/skills/types.js.map +1 -0
- package/dist/tasks/coordinator.d.ts +22 -1
- package/dist/tasks/coordinator.d.ts.map +1 -1
- package/dist/tasks/coordinator.js +83 -0
- package/dist/tasks/coordinator.js.map +1 -1
- package/dist/tasks/state-coordinator.d.ts +19 -0
- package/dist/tasks/state-coordinator.d.ts.map +1 -1
- package/dist/tasks/state-coordinator.js +40 -0
- package/dist/tasks/state-coordinator.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Background Task Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for background task execution.
|
|
5
|
+
* Enables non-blocking agent invocations and pipeline steps.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Background task status.
|
|
9
|
+
*/
|
|
10
|
+
export type BackgroundTaskStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
|
11
|
+
/**
|
|
12
|
+
* Type of background task.
|
|
13
|
+
*/
|
|
14
|
+
export type BackgroundTaskType = 'agent_invocation' | 'pipeline_step' | 'custom';
|
|
15
|
+
/**
|
|
16
|
+
* Background task definition.
|
|
17
|
+
*/
|
|
18
|
+
export interface BackgroundTask {
|
|
19
|
+
/** Unique task ID */
|
|
20
|
+
id: string;
|
|
21
|
+
/** Task type */
|
|
22
|
+
type: BackgroundTaskType;
|
|
23
|
+
/** Current status */
|
|
24
|
+
status: BackgroundTaskStatus;
|
|
25
|
+
/** Role being invoked (for agent tasks) */
|
|
26
|
+
role?: string;
|
|
27
|
+
/** Provider being used */
|
|
28
|
+
provider?: string;
|
|
29
|
+
/** Model being used */
|
|
30
|
+
model?: string;
|
|
31
|
+
/** Task description/prompt */
|
|
32
|
+
prompt: string;
|
|
33
|
+
/** When task was created */
|
|
34
|
+
createdAt: Date;
|
|
35
|
+
/** When task started running */
|
|
36
|
+
startedAt?: Date;
|
|
37
|
+
/** When task completed/failed */
|
|
38
|
+
completedAt?: Date;
|
|
39
|
+
/** Result if completed */
|
|
40
|
+
result?: unknown;
|
|
41
|
+
/** Error if failed */
|
|
42
|
+
error?: Error;
|
|
43
|
+
/** Progress percentage (0-100) */
|
|
44
|
+
progress?: number;
|
|
45
|
+
/** Progress message */
|
|
46
|
+
progressMessage?: string;
|
|
47
|
+
/** Associated worktree ID */
|
|
48
|
+
worktreeId?: string;
|
|
49
|
+
/** Parent task ID (for pipeline steps) */
|
|
50
|
+
parentTaskId?: string;
|
|
51
|
+
/** Abort controller for cancellation */
|
|
52
|
+
abortController?: AbortController;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Notification type for background task events.
|
|
56
|
+
*/
|
|
57
|
+
export type TaskNotificationType = 'started' | 'progress' | 'completed' | 'failed';
|
|
58
|
+
/**
|
|
59
|
+
* Notification for background task events.
|
|
60
|
+
*/
|
|
61
|
+
export interface TaskNotification {
|
|
62
|
+
/** ID of the task this notification is for */
|
|
63
|
+
taskId: string;
|
|
64
|
+
/** Type of notification */
|
|
65
|
+
type: TaskNotificationType;
|
|
66
|
+
/** When the event occurred */
|
|
67
|
+
timestamp: Date;
|
|
68
|
+
/** Human-readable message */
|
|
69
|
+
message: string;
|
|
70
|
+
/** Result data (for completed notifications) */
|
|
71
|
+
result?: unknown;
|
|
72
|
+
/** Error message (for failed notifications) */
|
|
73
|
+
error?: string;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Background task runner configuration.
|
|
77
|
+
*/
|
|
78
|
+
export interface BackgroundTaskConfig {
|
|
79
|
+
/** Maximum concurrent background tasks (default: 5) */
|
|
80
|
+
maxConcurrent: number;
|
|
81
|
+
/** Default timeout for background tasks in ms (default: 300000 = 5 minutes) */
|
|
82
|
+
defaultTimeout: number;
|
|
83
|
+
/** Enable task notifications (default: true) */
|
|
84
|
+
enableNotifications: boolean;
|
|
85
|
+
/** Notification callback */
|
|
86
|
+
onNotification?: (notification: TaskNotification) => void;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Events emitted by the BackgroundTaskRunner.
|
|
90
|
+
*/
|
|
91
|
+
export interface BackgroundTaskEvents {
|
|
92
|
+
'task:submitted': (task: BackgroundTask) => void;
|
|
93
|
+
'task:started': (task: BackgroundTask) => void;
|
|
94
|
+
'task:progress': (task: BackgroundTask, progress: number, message?: string) => void;
|
|
95
|
+
'task:completed': (task: BackgroundTask) => void;
|
|
96
|
+
'task:failed': (task: BackgroundTask, error: Error) => void;
|
|
97
|
+
'task:cancelled': (task: BackgroundTask) => void;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Options for submitting a background task.
|
|
101
|
+
*/
|
|
102
|
+
export interface SubmitTaskOptions {
|
|
103
|
+
/** Role being invoked (for agent tasks) */
|
|
104
|
+
role?: string;
|
|
105
|
+
/** Provider being used */
|
|
106
|
+
provider?: string;
|
|
107
|
+
/** Model being used */
|
|
108
|
+
model?: string;
|
|
109
|
+
/** Timeout in ms (overrides default) */
|
|
110
|
+
timeout?: number;
|
|
111
|
+
/** Associated worktree ID */
|
|
112
|
+
worktreeId?: string;
|
|
113
|
+
/** Parent task ID (for pipeline steps) */
|
|
114
|
+
parentTaskId?: string;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Serialized background task for persistence.
|
|
118
|
+
*/
|
|
119
|
+
export interface SerializedBackgroundTask {
|
|
120
|
+
id: string;
|
|
121
|
+
type: BackgroundTaskType;
|
|
122
|
+
status: BackgroundTaskStatus;
|
|
123
|
+
role?: string;
|
|
124
|
+
provider?: string;
|
|
125
|
+
model?: string;
|
|
126
|
+
prompt: string;
|
|
127
|
+
createdAt: number;
|
|
128
|
+
startedAt?: number;
|
|
129
|
+
completedAt?: number;
|
|
130
|
+
progress?: number;
|
|
131
|
+
progressMessage?: string;
|
|
132
|
+
worktreeId?: string;
|
|
133
|
+
parentTaskId?: string;
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/background/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B,SAAS,GACT,SAAS,GACT,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAMhB;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,kBAAkB,GAAG,eAAe,GAAG,QAAQ,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qBAAqB;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,qBAAqB;IACrB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,SAAS,EAAE,IAAI,CAAC;IAChB,gCAAgC;IAChC,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,iCAAiC;IACjC,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,0BAA0B;IAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sBAAsB;IACtB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8CAA8C;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,IAAI,EAAE,oBAAoB,CAAC;IAC3B,8BAA8B;IAC9B,SAAS,EAAE,IAAI,CAAC;IAChB,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB,gDAAgD;IAChD,mBAAmB,EAAE,OAAO,CAAC;IAC7B,4BAA4B;IAC5B,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,gBAAgB,KAAK,IAAI,CAAC;CAC3D;AAMD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,gBAAgB,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;IACjD,cAAc,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;IAC/C,eAAe,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IACpF,gBAAgB,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;IACjD,aAAa,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAC5D,gBAAgB,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;CAClD;AAMD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CAEvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/background/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -2,13 +2,28 @@
|
|
|
2
2
|
* Provider Health Tracker
|
|
3
3
|
*
|
|
4
4
|
* Tracks the health status of LLM providers to enable intelligent failover.
|
|
5
|
-
* Monitors success/failure rates, latency, and manages cooldown periods.
|
|
5
|
+
* Monitors success/failure rates, latency, rate limits, and manages cooldown periods.
|
|
6
6
|
*/
|
|
7
7
|
import type { Logger } from '../observability/logger.js';
|
|
8
8
|
/**
|
|
9
9
|
* Health status of a provider.
|
|
10
10
|
*/
|
|
11
11
|
export type ProviderHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
|
|
12
|
+
/**
|
|
13
|
+
* Rate limit information extracted from provider response headers.
|
|
14
|
+
*/
|
|
15
|
+
export interface RateLimitInfo {
|
|
16
|
+
/** Total requests allowed in period */
|
|
17
|
+
limit: number;
|
|
18
|
+
/** Remaining requests in period */
|
|
19
|
+
remaining: number;
|
|
20
|
+
/** When the period resets (Unix timestamp in ms) */
|
|
21
|
+
resetAt: number;
|
|
22
|
+
/** Tokens remaining (for token-based limits) */
|
|
23
|
+
tokensRemaining?: number;
|
|
24
|
+
/** Token limit */
|
|
25
|
+
tokenLimit?: number;
|
|
26
|
+
}
|
|
12
27
|
/**
|
|
13
28
|
* Health information for a single provider.
|
|
14
29
|
*/
|
|
@@ -37,6 +52,12 @@ export interface ProviderHealth {
|
|
|
37
52
|
successCount: number;
|
|
38
53
|
/** Total failed requests */
|
|
39
54
|
failureCount: number;
|
|
55
|
+
/** Current rate limit info (if available) */
|
|
56
|
+
rateLimit: RateLimitInfo | null;
|
|
57
|
+
/** Percentage of rate limit used (0-100) */
|
|
58
|
+
rateLimitUsedPercentage: number;
|
|
59
|
+
/** Is rate limit warning active (above threshold) */
|
|
60
|
+
rateLimitWarning: boolean;
|
|
40
61
|
}
|
|
41
62
|
/**
|
|
42
63
|
* Configuration for the health tracker.
|
|
@@ -52,6 +73,10 @@ export interface HealthTrackerConfig {
|
|
|
52
73
|
healthCheckIntervalMs?: number;
|
|
53
74
|
/** Window size for rolling latency average (default: 10) */
|
|
54
75
|
latencyWindowSize?: number;
|
|
76
|
+
/** Threshold to trigger rate limit warning (0-1, default 0.7 = 70% used) */
|
|
77
|
+
rateLimitWarningThreshold?: number;
|
|
78
|
+
/** Callback when rate limit warning triggers */
|
|
79
|
+
onRateLimitWarning?: (provider: string, info: RateLimitInfo) => void;
|
|
55
80
|
}
|
|
56
81
|
/**
|
|
57
82
|
* Tracks provider health for intelligent failover decisions.
|
|
@@ -135,6 +160,43 @@ export declare class ProviderHealthTracker {
|
|
|
135
160
|
* Stop periodic health checks.
|
|
136
161
|
*/
|
|
137
162
|
stopHealthCheckLoop(): void;
|
|
163
|
+
/**
|
|
164
|
+
* Update rate limit info from response headers.
|
|
165
|
+
*
|
|
166
|
+
* Parses rate limit headers from various providers and updates the health state.
|
|
167
|
+
* Triggers warning callback if rate limit usage exceeds threshold.
|
|
168
|
+
*/
|
|
169
|
+
updateRateLimits(provider: string, headers: Record<string, string>): void;
|
|
170
|
+
/**
|
|
171
|
+
* Parse rate limit headers from various provider formats.
|
|
172
|
+
*
|
|
173
|
+
* Supports:
|
|
174
|
+
* - OpenAI: x-ratelimit-limit-requests, x-ratelimit-remaining-requests, etc.
|
|
175
|
+
* - Anthropic: anthropic-ratelimit-requests-limit, etc.
|
|
176
|
+
* - Generic: x-ratelimit-limit, ratelimit-limit, etc.
|
|
177
|
+
*/
|
|
178
|
+
parseRateLimitHeaders(headers: Record<string, string>): RateLimitInfo | null;
|
|
179
|
+
/**
|
|
180
|
+
* Check if provider is approaching rate limit (above warning threshold).
|
|
181
|
+
*/
|
|
182
|
+
isApproachingRateLimit(provider: string): boolean;
|
|
183
|
+
/**
|
|
184
|
+
* Get time until rate limit resets (in ms).
|
|
185
|
+
* Returns 0 if no rate limit info or already reset.
|
|
186
|
+
*/
|
|
187
|
+
getRateLimitResetTime(provider: string): number;
|
|
188
|
+
/**
|
|
189
|
+
* Get all providers with active rate limit warnings.
|
|
190
|
+
*/
|
|
191
|
+
getProvidersWithRateLimitWarnings(): string[];
|
|
192
|
+
/**
|
|
193
|
+
* Parse various reset time formats to Unix timestamp (ms).
|
|
194
|
+
*/
|
|
195
|
+
private parseResetTime;
|
|
196
|
+
/**
|
|
197
|
+
* Serialized rate limit info for persistence.
|
|
198
|
+
*/
|
|
199
|
+
private serializeRateLimitInfo;
|
|
138
200
|
/**
|
|
139
201
|
* Serialize health state for persistence.
|
|
140
202
|
*/
|
|
@@ -150,6 +212,15 @@ export declare class ProviderHealthTracker {
|
|
|
150
212
|
lastErrorAt: number | null;
|
|
151
213
|
lastErrorMessage: string | null;
|
|
152
214
|
lastErrorCode: number | null;
|
|
215
|
+
rateLimit: {
|
|
216
|
+
limit: number;
|
|
217
|
+
remaining: number;
|
|
218
|
+
resetAt: number;
|
|
219
|
+
tokensRemaining: number | null;
|
|
220
|
+
tokenLimit: number | null;
|
|
221
|
+
} | null;
|
|
222
|
+
rateLimitUsedPercentage: number;
|
|
223
|
+
rateLimitWarning: boolean;
|
|
153
224
|
}>;
|
|
154
225
|
/**
|
|
155
226
|
* Restore health state from persistence.
|
|
@@ -166,6 +237,15 @@ export declare class ProviderHealthTracker {
|
|
|
166
237
|
lastErrorAt: number | null;
|
|
167
238
|
lastErrorMessage: string | null;
|
|
168
239
|
lastErrorCode: number | null;
|
|
240
|
+
rateLimit?: {
|
|
241
|
+
limit: number;
|
|
242
|
+
remaining: number;
|
|
243
|
+
resetAt: number;
|
|
244
|
+
tokensRemaining: number | null;
|
|
245
|
+
tokenLimit: number | null;
|
|
246
|
+
} | null;
|
|
247
|
+
rateLimitUsedPercentage?: number;
|
|
248
|
+
rateLimitWarning?: boolean;
|
|
169
249
|
}>): void;
|
|
170
250
|
private getOrCreateHealth;
|
|
171
251
|
private updateLatency;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"health-tracker.d.ts","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAMzD;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,qCAAqC;IACrC,SAAS,EAAE,IAAI,CAAC;IAChB,2CAA2C;IAC3C,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,yBAAyB;IACzB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,IAAI,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,qCAAqC;IACrC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sEAAsE;IACtE,aAAa,EAAE,IAAI,GAAG,IAAI,CAAC;IAC3B,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"health-tracker.d.ts","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAMzD;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,MAAM,EAAE,oBAAoB,CAAC;IAC7B,qCAAqC;IACrC,SAAS,EAAE,IAAI,CAAC;IAChB,2CAA2C;IAC3C,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,yBAAyB;IACzB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,IAAI,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,qCAAqC;IACrC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sEAAsE;IACtE,aAAa,EAAE,IAAI,GAAG,IAAI,CAAC;IAC3B,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IAChC,4CAA4C;IAC5C,uBAAuB,EAAE,MAAM,CAAC;IAChC,qDAAqD;IACrD,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,4EAA4E;IAC5E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,KAAK,IAAI,CAAC;CACtE;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;IACjE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoC;IACnE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiI;IACxJ,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,mBAAmB,CAA+C;IAC1E,OAAO,CAAC,mBAAmB,CAAsD;gBAErE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,mBAAmB;IAKxD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IA6BtD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAmCtE;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAcpC;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IActC;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAU9C;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAIvD;;OAEG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAI3C;;OAEG;IACH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAkBtD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAOnC;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAgBzE;;OAEG;IACH,mBAAmB,IAAI,IAAI;IAc3B;;;;;OAKG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAsDzE;;;;;;;OAOG;IACH,qBAAqB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,aAAa,GAAG,IAAI;IA2E5E;;OAEG;IACH,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAKjD;;;OAGG;IACH,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAU/C;;OAEG;IACH,iCAAiC,IAAI,MAAM,EAAE;IAU7C;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmB9B;;OAEG;IACH,SAAS,IAAI,KAAK,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,oBAAoB,CAAC;QAC7B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,EAAE;YACT,KAAK,EAAE,MAAM,CAAC;YACd,SAAS,EAAE,MAAM,CAAC;YAClB,OAAO,EAAE,MAAM,CAAC;YAChB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;YAC/B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B,GAAG,IAAI,CAAC;QACT,uBAAuB,EAAE,MAAM,CAAC;QAChC,gBAAgB,EAAE,OAAO,CAAC;KAC3B,CAAC;IA8CF;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,oBAAoB,CAAC;QAC7B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,SAAS,CAAC,EAAE;YACV,KAAK,EAAE,MAAM,CAAC;YACd,SAAS,EAAE,MAAM,CAAC;YAClB,OAAO,EAAE,MAAM,CAAC;YAChB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;YAC/B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;SAC3B,GAAG,IAAI,CAAC;QACT,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,CAAC,GAAG,IAAI;IAgDT,OAAO,CAAC,iBAAiB;IAuBzB,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,iBAAiB;YAUX,eAAe;CA8B9B"}
|
|
@@ -2,17 +2,19 @@
|
|
|
2
2
|
* Provider Health Tracker
|
|
3
3
|
*
|
|
4
4
|
* Tracks the health status of LLM providers to enable intelligent failover.
|
|
5
|
-
* Monitors success/failure rates, latency, and manages cooldown periods.
|
|
5
|
+
* Monitors success/failure rates, latency, rate limits, and manages cooldown periods.
|
|
6
6
|
*/
|
|
7
7
|
// ============================================================================
|
|
8
8
|
// Constants
|
|
9
9
|
// ============================================================================
|
|
10
|
+
// Note: onRateLimitWarning is optional and not included in Required<>
|
|
10
11
|
const DEFAULT_CONFIG = {
|
|
11
12
|
failureThreshold: 3,
|
|
12
13
|
degradedThreshold: 1,
|
|
13
14
|
cooldownMs: 300000, // 5 minutes
|
|
14
15
|
healthCheckIntervalMs: 60000, // 1 minute
|
|
15
16
|
latencyWindowSize: 10,
|
|
17
|
+
rateLimitWarningThreshold: 0.7, // 70% used triggers warning
|
|
16
18
|
};
|
|
17
19
|
// ============================================================================
|
|
18
20
|
// ProviderHealthTracker Class
|
|
@@ -225,6 +227,212 @@ export class ProviderHealthTracker {
|
|
|
225
227
|
this.logger.info('Health check loop stopped');
|
|
226
228
|
}
|
|
227
229
|
}
|
|
230
|
+
// ==========================================================================
|
|
231
|
+
// Rate Limit Methods
|
|
232
|
+
// ==========================================================================
|
|
233
|
+
/**
|
|
234
|
+
* Update rate limit info from response headers.
|
|
235
|
+
*
|
|
236
|
+
* Parses rate limit headers from various providers and updates the health state.
|
|
237
|
+
* Triggers warning callback if rate limit usage exceeds threshold.
|
|
238
|
+
*/
|
|
239
|
+
updateRateLimits(provider, headers) {
|
|
240
|
+
const health = this.getOrCreateHealth(provider);
|
|
241
|
+
const rateLimitInfo = this.parseRateLimitHeaders(headers);
|
|
242
|
+
if (!rateLimitInfo) {
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
health.rateLimit = rateLimitInfo;
|
|
246
|
+
// Calculate percentage used
|
|
247
|
+
const usedPercentage = rateLimitInfo.limit > 0
|
|
248
|
+
? ((rateLimitInfo.limit - rateLimitInfo.remaining) / rateLimitInfo.limit) * 100
|
|
249
|
+
: 0;
|
|
250
|
+
health.rateLimitUsedPercentage = Math.round(usedPercentage * 100) / 100;
|
|
251
|
+
// Check if warning threshold exceeded
|
|
252
|
+
const threshold = this.config.rateLimitWarningThreshold ?? 0.7;
|
|
253
|
+
const wasWarning = health.rateLimitWarning;
|
|
254
|
+
health.rateLimitWarning = usedPercentage / 100 >= threshold;
|
|
255
|
+
// Log and trigger callback on new warning
|
|
256
|
+
if (health.rateLimitWarning && !wasWarning) {
|
|
257
|
+
const resetsInSeconds = Math.ceil((rateLimitInfo.resetAt - Date.now()) / 1000);
|
|
258
|
+
this.logger.warn('Rate limit warning', {
|
|
259
|
+
provider,
|
|
260
|
+
remaining: rateLimitInfo.remaining,
|
|
261
|
+
limit: rateLimitInfo.limit,
|
|
262
|
+
usedPercentage: health.rateLimitUsedPercentage,
|
|
263
|
+
threshold: threshold * 100,
|
|
264
|
+
resetsIn: `${resetsInSeconds}s`,
|
|
265
|
+
tokensRemaining: rateLimitInfo.tokensRemaining,
|
|
266
|
+
tokenLimit: rateLimitInfo.tokenLimit,
|
|
267
|
+
});
|
|
268
|
+
// Trigger callback if configured
|
|
269
|
+
if (this.config.onRateLimitWarning) {
|
|
270
|
+
this.config.onRateLimitWarning(provider, rateLimitInfo);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
// Check for rate limit exceeded (remaining is 0 or negative)
|
|
274
|
+
if (rateLimitInfo.remaining <= 0) {
|
|
275
|
+
const retryAfter = headers['retry-after'] ?? headers['Retry-After'];
|
|
276
|
+
this.logger.error('Rate limit exceeded', {
|
|
277
|
+
provider,
|
|
278
|
+
retryAfter,
|
|
279
|
+
resetAt: new Date(rateLimitInfo.resetAt).toISOString(),
|
|
280
|
+
tokensRemaining: rateLimitInfo.tokensRemaining,
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Parse rate limit headers from various provider formats.
|
|
286
|
+
*
|
|
287
|
+
* Supports:
|
|
288
|
+
* - OpenAI: x-ratelimit-limit-requests, x-ratelimit-remaining-requests, etc.
|
|
289
|
+
* - Anthropic: anthropic-ratelimit-requests-limit, etc.
|
|
290
|
+
* - Generic: x-ratelimit-limit, ratelimit-limit, etc.
|
|
291
|
+
*/
|
|
292
|
+
parseRateLimitHeaders(headers) {
|
|
293
|
+
// Normalize header keys to lowercase for consistent access
|
|
294
|
+
const normalizedHeaders = {};
|
|
295
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
296
|
+
normalizedHeaders[key.toLowerCase()] = value;
|
|
297
|
+
}
|
|
298
|
+
let limit;
|
|
299
|
+
let remaining;
|
|
300
|
+
let resetAt;
|
|
301
|
+
let tokensRemaining;
|
|
302
|
+
let tokenLimit;
|
|
303
|
+
// Try OpenAI format
|
|
304
|
+
if (normalizedHeaders['x-ratelimit-limit-requests']) {
|
|
305
|
+
limit = parseInt(normalizedHeaders['x-ratelimit-limit-requests'], 10);
|
|
306
|
+
remaining = parseInt(normalizedHeaders['x-ratelimit-remaining-requests'] ?? '0', 10);
|
|
307
|
+
resetAt = this.parseResetTime(normalizedHeaders['x-ratelimit-reset-requests']);
|
|
308
|
+
tokenLimit = normalizedHeaders['x-ratelimit-limit-tokens']
|
|
309
|
+
? parseInt(normalizedHeaders['x-ratelimit-limit-tokens'], 10)
|
|
310
|
+
: undefined;
|
|
311
|
+
tokensRemaining = normalizedHeaders['x-ratelimit-remaining-tokens']
|
|
312
|
+
? parseInt(normalizedHeaders['x-ratelimit-remaining-tokens'], 10)
|
|
313
|
+
: undefined;
|
|
314
|
+
}
|
|
315
|
+
// Try Anthropic format
|
|
316
|
+
else if (normalizedHeaders['anthropic-ratelimit-requests-limit']) {
|
|
317
|
+
limit = parseInt(normalizedHeaders['anthropic-ratelimit-requests-limit'], 10);
|
|
318
|
+
remaining = parseInt(normalizedHeaders['anthropic-ratelimit-requests-remaining'] ?? '0', 10);
|
|
319
|
+
resetAt = this.parseResetTime(normalizedHeaders['anthropic-ratelimit-requests-reset']);
|
|
320
|
+
tokenLimit = normalizedHeaders['anthropic-ratelimit-tokens-limit']
|
|
321
|
+
? parseInt(normalizedHeaders['anthropic-ratelimit-tokens-limit'], 10)
|
|
322
|
+
: undefined;
|
|
323
|
+
tokensRemaining = normalizedHeaders['anthropic-ratelimit-tokens-remaining']
|
|
324
|
+
? parseInt(normalizedHeaders['anthropic-ratelimit-tokens-remaining'], 10)
|
|
325
|
+
: undefined;
|
|
326
|
+
}
|
|
327
|
+
// Try generic x-ratelimit format
|
|
328
|
+
else if (normalizedHeaders['x-ratelimit-limit']) {
|
|
329
|
+
limit = parseInt(normalizedHeaders['x-ratelimit-limit'], 10);
|
|
330
|
+
remaining = parseInt(normalizedHeaders['x-ratelimit-remaining'] ?? '0', 10);
|
|
331
|
+
resetAt = this.parseResetTime(normalizedHeaders['x-ratelimit-reset']);
|
|
332
|
+
}
|
|
333
|
+
// Try generic ratelimit format (without x- prefix)
|
|
334
|
+
else if (normalizedHeaders['ratelimit-limit']) {
|
|
335
|
+
limit = parseInt(normalizedHeaders['ratelimit-limit'], 10);
|
|
336
|
+
remaining = parseInt(normalizedHeaders['ratelimit-remaining'] ?? '0', 10);
|
|
337
|
+
resetAt = this.parseResetTime(normalizedHeaders['ratelimit-reset']);
|
|
338
|
+
}
|
|
339
|
+
// Try retry-after only (minimal info)
|
|
340
|
+
else if (normalizedHeaders['retry-after']) {
|
|
341
|
+
const retryAfterSeconds = parseInt(normalizedHeaders['retry-after'], 10);
|
|
342
|
+
if (!isNaN(retryAfterSeconds)) {
|
|
343
|
+
return {
|
|
344
|
+
limit: 0,
|
|
345
|
+
remaining: 0,
|
|
346
|
+
resetAt: Date.now() + retryAfterSeconds * 1000,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
// If we found limit and remaining, return the info
|
|
351
|
+
if (limit !== undefined && remaining !== undefined) {
|
|
352
|
+
return {
|
|
353
|
+
limit,
|
|
354
|
+
remaining,
|
|
355
|
+
resetAt: resetAt ?? Date.now() + 60000, // Default to 1 minute if not specified
|
|
356
|
+
tokensRemaining,
|
|
357
|
+
tokenLimit,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Check if provider is approaching rate limit (above warning threshold).
|
|
364
|
+
*/
|
|
365
|
+
isApproachingRateLimit(provider) {
|
|
366
|
+
const health = this.health.get(provider);
|
|
367
|
+
return health?.rateLimitWarning ?? false;
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Get time until rate limit resets (in ms).
|
|
371
|
+
* Returns 0 if no rate limit info or already reset.
|
|
372
|
+
*/
|
|
373
|
+
getRateLimitResetTime(provider) {
|
|
374
|
+
const health = this.health.get(provider);
|
|
375
|
+
if (!health?.rateLimit) {
|
|
376
|
+
return 0;
|
|
377
|
+
}
|
|
378
|
+
const remaining = health.rateLimit.resetAt - Date.now();
|
|
379
|
+
return Math.max(0, remaining);
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Get all providers with active rate limit warnings.
|
|
383
|
+
*/
|
|
384
|
+
getProvidersWithRateLimitWarnings() {
|
|
385
|
+
const result = [];
|
|
386
|
+
for (const [provider, health] of this.health) {
|
|
387
|
+
if (health.rateLimitWarning) {
|
|
388
|
+
result.push(provider);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return result;
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Parse various reset time formats to Unix timestamp (ms).
|
|
395
|
+
*/
|
|
396
|
+
parseResetTime(value) {
|
|
397
|
+
if (!value) {
|
|
398
|
+
return undefined;
|
|
399
|
+
}
|
|
400
|
+
// Try parsing as Unix timestamp (seconds)
|
|
401
|
+
const asNumber = parseInt(value, 10);
|
|
402
|
+
if (!isNaN(asNumber)) {
|
|
403
|
+
// If the number is less than a reasonable date (year 2000), treat as seconds from now
|
|
404
|
+
if (asNumber < 946684800) {
|
|
405
|
+
return Date.now() + asNumber * 1000;
|
|
406
|
+
}
|
|
407
|
+
// If reasonable timestamp size, assume it's seconds since epoch
|
|
408
|
+
if (asNumber < 10000000000) {
|
|
409
|
+
return asNumber * 1000;
|
|
410
|
+
}
|
|
411
|
+
// Already in milliseconds
|
|
412
|
+
return asNumber;
|
|
413
|
+
}
|
|
414
|
+
// Try parsing as ISO date string
|
|
415
|
+
const asDate = Date.parse(value);
|
|
416
|
+
if (!isNaN(asDate)) {
|
|
417
|
+
return asDate;
|
|
418
|
+
}
|
|
419
|
+
return undefined;
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Serialized rate limit info for persistence.
|
|
423
|
+
*/
|
|
424
|
+
serializeRateLimitInfo(info) {
|
|
425
|
+
if (!info) {
|
|
426
|
+
return null;
|
|
427
|
+
}
|
|
428
|
+
return {
|
|
429
|
+
limit: info.limit,
|
|
430
|
+
remaining: info.remaining,
|
|
431
|
+
resetAt: info.resetAt,
|
|
432
|
+
tokensRemaining: info.tokensRemaining ?? null,
|
|
433
|
+
tokenLimit: info.tokenLimit ?? null,
|
|
434
|
+
};
|
|
435
|
+
}
|
|
228
436
|
/**
|
|
229
437
|
* Serialize health state for persistence.
|
|
230
438
|
*/
|
|
@@ -243,6 +451,9 @@ export class ProviderHealthTracker {
|
|
|
243
451
|
lastErrorAt: health.lastError?.timestamp.getTime() ?? null,
|
|
244
452
|
lastErrorMessage: health.lastError?.message ?? null,
|
|
245
453
|
lastErrorCode: health.lastError?.code ?? null,
|
|
454
|
+
rateLimit: this.serializeRateLimitInfo(health.rateLimit),
|
|
455
|
+
rateLimitUsedPercentage: health.rateLimitUsedPercentage,
|
|
456
|
+
rateLimitWarning: health.rateLimitWarning,
|
|
246
457
|
});
|
|
247
458
|
}
|
|
248
459
|
return result;
|
|
@@ -252,6 +463,17 @@ export class ProviderHealthTracker {
|
|
|
252
463
|
*/
|
|
253
464
|
restore(data) {
|
|
254
465
|
for (const item of data) {
|
|
466
|
+
// Restore rate limit info, handling optional fields for backward compatibility
|
|
467
|
+
let rateLimit = null;
|
|
468
|
+
if (item.rateLimit) {
|
|
469
|
+
rateLimit = {
|
|
470
|
+
limit: item.rateLimit.limit,
|
|
471
|
+
remaining: item.rateLimit.remaining,
|
|
472
|
+
resetAt: item.rateLimit.resetAt,
|
|
473
|
+
tokensRemaining: item.rateLimit.tokensRemaining ?? undefined,
|
|
474
|
+
tokenLimit: item.rateLimit.tokenLimit ?? undefined,
|
|
475
|
+
};
|
|
476
|
+
}
|
|
255
477
|
const health = {
|
|
256
478
|
provider: item.provider,
|
|
257
479
|
status: item.status,
|
|
@@ -269,6 +491,9 @@ export class ProviderHealthTracker {
|
|
|
269
491
|
latencyMs: item.latencyMs,
|
|
270
492
|
successCount: item.successCount,
|
|
271
493
|
failureCount: item.failureCount,
|
|
494
|
+
rateLimit,
|
|
495
|
+
rateLimitUsedPercentage: item.rateLimitUsedPercentage ?? 0,
|
|
496
|
+
rateLimitWarning: item.rateLimitWarning ?? false,
|
|
272
497
|
};
|
|
273
498
|
this.health.set(item.provider, health);
|
|
274
499
|
}
|
|
@@ -293,6 +518,9 @@ export class ProviderHealthTracker {
|
|
|
293
518
|
latencyMs: 0,
|
|
294
519
|
successCount: 0,
|
|
295
520
|
failureCount: 0,
|
|
521
|
+
rateLimit: null,
|
|
522
|
+
rateLimitUsedPercentage: 0,
|
|
523
|
+
rateLimitWarning: false,
|
|
296
524
|
};
|
|
297
525
|
this.health.set(provider, health);
|
|
298
526
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"health-tracker.js","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA2DH,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,cAAc,GAAkC;IACpD,gBAAgB,EAAE,CAAC;IACnB,iBAAiB,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,YAAY;IAChC,qBAAqB,EAAE,KAAK,EAAE,WAAW;IACzC,iBAAiB,EAAE,EAAE;CACtB,CAAC;AAEF,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,qBAAqB;IACf,MAAM,GAAgC,IAAI,GAAG,EAAE,CAAC;IAChD,cAAc,GAA0B,IAAI,GAAG,EAAE,CAAC;IAClD,MAAM,CAAgC;IACtC,MAAM,CAAS;IACxB,mBAAmB,GAA0C,IAAI,CAAC;IAClE,mBAAmB,GAAiD,IAAI,CAAC;IAEjF,YAAY,MAAc,EAAE,MAA4B;QACtD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,SAAiB;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;QACzB,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,iCAAiC;QACjC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEpD,iDAAiD;QACjD,IAAI,MAAM,CAAC,aAAa,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QAED,4CAA4C;QAC5C,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;QAE1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;YAC5C,QAAQ;YACR,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,KAAY,EAAE,UAAmB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,SAAS,GAAG;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,GAAG;SACf,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,8CAA8C;QAC9C,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC/D,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBAC5C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE;gBACjD,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAE3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC,kCAAkC;QACjD,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAAgB;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,aAAuB;QACzC,OAAO,aAAa;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEnC,0BAA0B;YAC1B,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC;YAE7E,sBAAsB;YACtB,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,OAAO,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAA6C;QAChE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;QACpC,IAAI,CAAC,mBAAmB,GAAG,WAAW,CACpC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,eAAe,EAAE,EACjC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAClC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;SAC9C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QAaP,MAAM,MAAM,GAYP,EAAE,CAAC;QAER,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,IAAI;gBACtD,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,IAAI;gBACpD,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,IAAI,IAAI;gBACnD,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,IAAI,IAAI;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAYN;QACA,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,MAAM,GAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,IAAI,CAAC,WAAW;oBACzB,CAAC,CAAC;wBACE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,SAAS;wBACrC,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;wBACjD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;qBACtC;oBACH,CAAC,CAAC,IAAI;gBACR,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;gBAC7C,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvE,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,iBAAiB,CAAC,QAAgB;QACxC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG;gBACP,QAAQ;gBACR,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,mBAAmB,EAAE,CAAC;gBACtB,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,CAAC;gBACZ,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;aAChB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAExB,+BAA+B;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1D,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,EAAE;wBACtE,QAAQ;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;wBACzC,uDAAuD;wBACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACtE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,4DAA4D;wBAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;4BAChD,QAAQ;4BACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAC9D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"health-tracker.js","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAqFH,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,sEAAsE;AACtE,MAAM,cAAc,GAAmI;IACrJ,gBAAgB,EAAE,CAAC;IACnB,iBAAiB,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,YAAY;IAChC,qBAAqB,EAAE,KAAK,EAAE,WAAW;IACzC,iBAAiB,EAAE,EAAE;IACrB,yBAAyB,EAAE,GAAG,EAAE,4BAA4B;CAC7D,CAAC;AAEF,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,qBAAqB;IACf,MAAM,GAAgC,IAAI,GAAG,EAAE,CAAC;IAChD,cAAc,GAA0B,IAAI,GAAG,EAAE,CAAC;IAClD,MAAM,CAAiI;IACvI,MAAM,CAAS;IACxB,mBAAmB,GAA0C,IAAI,CAAC;IAClE,mBAAmB,GAAiD,IAAI,CAAC;IAEjF,YAAY,MAAc,EAAE,MAA4B;QACtD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,SAAiB;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;QACzB,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,iCAAiC;QACjC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEpD,iDAAiD;QACjD,IAAI,MAAM,CAAC,aAAa,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QAED,4CAA4C;QAC5C,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;QAE1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;YAC5C,QAAQ;YACR,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,KAAY,EAAE,UAAmB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,SAAS,GAAG;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,GAAG;SACf,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,8CAA8C;QAC9C,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC/D,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBAC5C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE;gBACjD,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAE3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC,kCAAkC;QACjD,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAAgB;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,aAAuB;QACzC,OAAO,aAAa;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEnC,0BAA0B;YAC1B,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC;YAE7E,sBAAsB;YACtB,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,OAAO,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAA6C;QAChE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;QACpC,IAAI,CAAC,mBAAmB,GAAG,WAAW,CACpC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,eAAe,EAAE,EACjC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAClC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;SAC9C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,qBAAqB;IACrB,6EAA6E;IAE7E;;;;;OAKG;IACH,gBAAgB,CAAC,QAAgB,EAAE,OAA+B;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAE1D,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,MAAM,CAAC,SAAS,GAAG,aAAa,CAAC;QAEjC,4BAA4B;QAC5B,MAAM,cAAc,GAAG,aAAa,CAAC,KAAK,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,GAAG;YAC/E,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAExE,sCAAsC;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,yBAAyB,IAAI,GAAG,CAAC;QAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAC3C,MAAM,CAAC,gBAAgB,GAAG,cAAc,GAAG,GAAG,IAAI,SAAS,CAAC;QAE5D,0CAA0C;QAC1C,IAAI,MAAM,CAAC,gBAAgB,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YAE/E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;gBACrC,QAAQ;gBACR,SAAS,EAAE,aAAa,CAAC,SAAS;gBAClC,KAAK,EAAE,aAAa,CAAC,KAAK;gBAC1B,cAAc,EAAE,MAAM,CAAC,uBAAuB;gBAC9C,SAAS,EAAE,SAAS,GAAG,GAAG;gBAC1B,QAAQ,EAAE,GAAG,eAAe,GAAG;gBAC/B,eAAe,EAAE,aAAa,CAAC,eAAe;gBAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;aACrC,CAAC,CAAC;YAEH,iCAAiC;YACjC,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,IAAI,aAAa,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;YACpE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBACvC,QAAQ;gBACR,UAAU;gBACV,OAAO,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE;gBACtD,eAAe,EAAE,aAAa,CAAC,eAAe;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,qBAAqB,CAAC,OAA+B;QACnD,2DAA2D;QAC3D,MAAM,iBAAiB,GAA2B,EAAE,CAAC;QACrD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,iBAAiB,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,IAAI,KAAyB,CAAC;QAC9B,IAAI,SAA6B,CAAC;QAClC,IAAI,OAA2B,CAAC;QAChC,IAAI,eAAmC,CAAC;QACxC,IAAI,UAA8B,CAAC;QAEnC,oBAAoB;QACpB,IAAI,iBAAiB,CAAC,4BAA4B,CAAC,EAAE,CAAC;YACpD,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,EAAE,EAAE,CAAC,CAAC;YACtE,SAAS,GAAG,QAAQ,CAAC,iBAAiB,CAAC,gCAAgC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YACrF,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,4BAA4B,CAAC,CAAC,CAAC;YAC/E,UAAU,GAAG,iBAAiB,CAAC,0BAA0B,CAAC;gBACxD,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,0BAA0B,CAAC,EAAE,EAAE,CAAC;gBAC7D,CAAC,CAAC,SAAS,CAAC;YACd,eAAe,GAAG,iBAAiB,CAAC,8BAA8B,CAAC;gBACjE,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,8BAA8B,CAAC,EAAE,EAAE,CAAC;gBACjE,CAAC,CAAC,SAAS,CAAC;QAChB,CAAC;QACD,uBAAuB;aAClB,IAAI,iBAAiB,CAAC,oCAAoC,CAAC,EAAE,CAAC;YACjE,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,oCAAoC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9E,SAAS,GAAG,QAAQ,CAAC,iBAAiB,CAAC,wCAAwC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7F,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,oCAAoC,CAAC,CAAC,CAAC;YACvF,UAAU,GAAG,iBAAiB,CAAC,kCAAkC,CAAC;gBAChE,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,kCAAkC,CAAC,EAAE,EAAE,CAAC;gBACrE,CAAC,CAAC,SAAS,CAAC;YACd,eAAe,GAAG,iBAAiB,CAAC,sCAAsC,CAAC;gBACzE,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,sCAAsC,CAAC,EAAE,EAAE,CAAC;gBACzE,CAAC,CAAC,SAAS,CAAC;QAChB,CAAC;QACD,iCAAiC;aAC5B,IAAI,iBAAiB,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAChD,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7D,SAAS,GAAG,QAAQ,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5E,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC,CAAC;QACxE,CAAC;QACD,mDAAmD;aAC9C,IAAI,iBAAiB,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC9C,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,SAAS,GAAG,QAAQ,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1E,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,sCAAsC;aACjC,IAAI,iBAAiB,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC9B,OAAO;oBACL,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB,GAAG,IAAI;iBAC/C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,IAAI,KAAK,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACnD,OAAO;gBACL,KAAK;gBACL,SAAS;gBACT,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,uCAAuC;gBAC/E,eAAe;gBACf,UAAU;aACX,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,QAAgB;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,MAAM,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,QAAgB;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,iCAAiC;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAyB;QAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,0CAA0C;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrB,sFAAsF;YACtF,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC;YACtC,CAAC;YACD,gEAAgE;YAChE,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;gBAC3B,OAAO,QAAQ,GAAG,IAAI,CAAC;YACzB,CAAC;YACD,0BAA0B;YAC1B,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,iCAAiC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACnB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,IAA0B;QAOvD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,IAAI;YAC7C,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;SACpC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS;QAsBP,MAAM,MAAM,GAqBP,EAAE,CAAC;QAER,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,IAAI;gBACtD,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,IAAI;gBACpD,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,IAAI,IAAI;gBACnD,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,IAAI,IAAI;gBAC7C,SAAS,EAAE,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,SAAS,CAAC;gBACxD,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;gBACvD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;aAC1C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAqBN;QACA,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,+EAA+E;YAC/E,IAAI,SAAS,GAAyB,IAAI,CAAC;YAC3C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,SAAS,GAAG;oBACV,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;oBAC3B,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;oBACnC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO;oBAC/B,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,IAAI,SAAS;oBAC5D,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,IAAI,SAAS;iBACnD,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,GAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,IAAI,CAAC,WAAW;oBACzB,CAAC,CAAC;wBACE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,SAAS;wBACrC,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;wBACjD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;qBACtC;oBACH,CAAC,CAAC,IAAI;gBACR,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;gBAC7C,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvE,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS;gBACT,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,IAAI,CAAC;gBAC1D,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,KAAK;aACjD,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,iBAAiB,CAAC,QAAgB;QACxC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG;gBACP,QAAQ;gBACR,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,mBAAmB,EAAE,CAAC;gBACtB,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,CAAC;gBACZ,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,IAAI;gBACf,uBAAuB,EAAE,CAAC;gBAC1B,gBAAgB,EAAE,KAAK;aACxB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAExB,+BAA+B;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1D,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,EAAE;wBACtE,QAAQ;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;wBACzC,uDAAuD;wBACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACtE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,4DAA4D;wBAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;4BAChD,QAAQ;4BACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAC9D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|