@azumag/opencode-rate-limit-fallback 1.30.0 → 1.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -47
- package/dist/index.d.ts +1 -1
- package/dist/index.js +8 -2
- package/dist/src/circuitbreaker/CircuitBreaker.d.ts +53 -0
- package/dist/src/circuitbreaker/CircuitBreaker.js +208 -0
- package/dist/src/circuitbreaker/CircuitState.d.ts +44 -0
- package/dist/src/circuitbreaker/CircuitState.js +128 -0
- package/dist/src/circuitbreaker/index.d.ts +8 -0
- package/dist/src/circuitbreaker/index.js +8 -0
- package/dist/src/fallback/FallbackHandler.d.ts +1 -0
- package/dist/src/fallback/FallbackHandler.js +26 -1
- package/dist/src/fallback/ModelSelector.d.ts +7 -1
- package/dist/src/fallback/ModelSelector.js +17 -4
- package/dist/src/metrics/MetricsManager.d.ts +10 -1
- package/dist/src/metrics/MetricsManager.js +137 -0
- package/dist/src/types/index.d.ts +46 -0
- package/dist/src/types/index.js +10 -0
- package/dist/src/utils/config.d.ts +8 -1
- package/dist/src/utils/config.js +26 -11
- package/dist/src/utils/errorDetection.js +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,10 +17,11 @@ OpenCode plugin that automatically switches to fallback models when rate limited
|
|
|
17
17
|
- Jitter to prevent thundering herd problem
|
|
18
18
|
- Configurable retry limits and timeouts
|
|
19
19
|
- Retry statistics tracking
|
|
20
|
-
- Toast notifications for user feedback
|
|
21
|
-
- Subagent session support with automatic fallback propagation to parent sessions
|
|
22
|
-
- Configurable maximum subagent nesting depth
|
|
23
|
-
- **
|
|
20
|
+
- Toast notifications for user feedback
|
|
21
|
+
- Subagent session support with automatic fallback propagation to parent sessions
|
|
22
|
+
- Configurable maximum subagent nesting depth
|
|
23
|
+
- **Circuit breaker pattern** to prevent cascading failures from consistently failing models
|
|
24
|
+
- **Metrics collection** to track rate limits, fallbacks, and model performance
|
|
24
25
|
|
|
25
26
|
## Installation
|
|
26
27
|
|
|
@@ -85,21 +86,29 @@ Create a configuration file at one of these locations:
|
|
|
85
86
|
"format": "pretty"
|
|
86
87
|
},
|
|
87
88
|
"resetInterval": "daily"
|
|
89
|
+
},
|
|
90
|
+
"circuitBreaker": {
|
|
91
|
+
"enabled": true,
|
|
92
|
+
"failureThreshold": 5,
|
|
93
|
+
"recoveryTimeoutMs": 60000,
|
|
94
|
+
"halfOpenMaxCalls": 1,
|
|
95
|
+
"successThreshold": 2
|
|
88
96
|
}
|
|
89
97
|
}
|
|
90
98
|
```
|
|
91
99
|
|
|
92
100
|
### Configuration Options
|
|
93
101
|
|
|
94
|
-
| Option | Type | Default | Description |
|
|
95
|
-
|--------|------|---------|-------------|
|
|
96
|
-
| `enabled` | boolean | `true` | Enable/disable the plugin |
|
|
97
|
-
| `cooldownMs` | number | `60000` | Cooldown period (ms) before retrying a rate-limited model |
|
|
98
|
-
| `fallbackMode` | string | `"cycle"` | Behavior when all models are exhausted (see below) |
|
|
99
|
-
| `fallbackModels` | array | See below | List of fallback models in priority order |
|
|
100
|
-
| `maxSubagentDepth` | number | `10` | Maximum nesting depth for subagent hierarchies |
|
|
101
|
-
| `enableSubagentFallback` | boolean | `true` | Enable/disable fallback for subagent sessions |
|
|
102
|
-
| `retryPolicy` | object | See below | Retry policy configuration (see below) |
|
|
102
|
+
| Option | Type | Default | Description |
|
|
103
|
+
|--------|------|---------|-------------|
|
|
104
|
+
| `enabled` | boolean | `true` | Enable/disable the plugin |
|
|
105
|
+
| `cooldownMs` | number | `60000` | Cooldown period (ms) before retrying a rate-limited model |
|
|
106
|
+
| `fallbackMode` | string | `"cycle"` | Behavior when all models are exhausted (see below) |
|
|
107
|
+
| `fallbackModels` | array | See below | List of fallback models in priority order |
|
|
108
|
+
| `maxSubagentDepth` | number | `10` | Maximum nesting depth for subagent hierarchies |
|
|
109
|
+
| `enableSubagentFallback` | boolean | `true` | Enable/disable fallback for subagent sessions |
|
|
110
|
+
| `retryPolicy` | object | See below | Retry policy configuration (see below) |
|
|
111
|
+
| `circuitBreaker` | object | See below | Circuit breaker configuration (see below) |
|
|
103
112
|
|
|
104
113
|
### Fallback Modes
|
|
105
114
|
|
|
@@ -163,11 +172,56 @@ Example with `baseDelayMs: 1000` and `maxDelayMs: 5000`:
|
|
|
163
172
|
|
|
164
173
|
Jitter adds random variation to delay times to prevent the "thundering herd" problem, where multiple clients retry simultaneously and overwhelm the API.
|
|
165
174
|
|
|
166
|
-
- Recommended for production environments with multiple concurrent users
|
|
167
|
-
- `jitterFactor: 0.1` adds ±10% variance to delay times
|
|
168
|
-
- Example: With base delay of 1000ms and jitterFactor 0.1, actual delay will be 900-1100ms
|
|
175
|
+
- Recommended for production environments with multiple concurrent users
|
|
176
|
+
- `jitterFactor: 0.1` adds ±10% variance to delay times
|
|
177
|
+
- Example: With base delay of 1000ms and jitterFactor 0.1, actual delay will be 900-1100ms
|
|
178
|
+
|
|
179
|
+
### Circuit Breaker
|
|
180
|
+
|
|
181
|
+
The circuit breaker pattern prevents cascading failures by temporarily disabling models that are consistently failing (not due to rate limits).
|
|
182
|
+
|
|
183
|
+
| Option | Type | Default | Description |
|
|
184
|
+
|--------|------|---------|-------------|
|
|
185
|
+
| `circuitBreaker.enabled` | boolean | `false` | Enable/disable circuit breaker |
|
|
186
|
+
| `circuitBreaker.failureThreshold` | number | `5` | Consecutive failures before opening circuit |
|
|
187
|
+
| `circuitBreaker.recoveryTimeoutMs` | number | `60000` | Wait time before attempting recovery (ms) |
|
|
188
|
+
| `circuitBreaker.halfOpenMaxCalls` | number | `1` | Max calls allowed in HALF_OPEN state |
|
|
189
|
+
| `circuitBreaker.successThreshold` | number | `2` | Successes needed to close circuit |
|
|
190
|
+
|
|
191
|
+
#### How It Works
|
|
192
|
+
|
|
193
|
+
The circuit breaker maintains three states for each model:
|
|
194
|
+
|
|
195
|
+
1. **CLOSED State**: Normal operation, requests pass through
|
|
196
|
+
- Failures are counted until the threshold is reached
|
|
197
|
+
- On threshold breach, transitions to OPEN state
|
|
169
198
|
|
|
170
|
-
|
|
199
|
+
2. **OPEN State**: Model is failing, requests fail fast
|
|
200
|
+
- The circuit is "open" to prevent unnecessary API calls
|
|
201
|
+
- No requests are allowed through
|
|
202
|
+
- After the recovery timeout, transitions to HALF_OPEN state
|
|
203
|
+
|
|
204
|
+
3. **HALF_OPEN State**: Testing if model recovered after timeout
|
|
205
|
+
- A limited number of test requests are allowed
|
|
206
|
+
- On success, transitions back to CLOSED state
|
|
207
|
+
- On failure, returns to OPEN state
|
|
208
|
+
|
|
209
|
+
#### Important Notes
|
|
210
|
+
|
|
211
|
+
- **Rate limit errors are NOT counted as failures**: The circuit breaker only tracks actual failures, not rate limit errors
|
|
212
|
+
- **Disabled by default**: Set `circuitBreaker.enabled: true` to activate this feature
|
|
213
|
+
- **Per-model tracking**: Each model has its own circuit state
|
|
214
|
+
- **Toast notifications**: Users are notified when circuits open/close for awareness
|
|
215
|
+
|
|
216
|
+
#### Configuration Recommendations
|
|
217
|
+
|
|
218
|
+
| Environment | failureThreshold | recoveryTimeoutMs | halfOpenMaxCalls |
|
|
219
|
+
|-------------|------------------|-------------------|------------------|
|
|
220
|
+
| Development | 3 | 30000 | 1 |
|
|
221
|
+
| Production | 5 | 60000 | 1 |
|
|
222
|
+
| High Availability | 10 | 30000 | 2 |
|
|
223
|
+
|
|
224
|
+
### Default Fallback Models
|
|
171
225
|
|
|
172
226
|
If no configuration is provided, the following models are used:
|
|
173
227
|
|
|
@@ -206,11 +260,12 @@ When OpenCode uses subagents (e.g., for complex tasks requiring specialized agen
|
|
|
206
260
|
|
|
207
261
|
## Metrics
|
|
208
262
|
|
|
209
|
-
The plugin includes a metrics collection feature that tracks:
|
|
210
|
-
- Rate limit events per provider/model
|
|
211
|
-
- Fallback statistics (total, successful, failed, average duration)
|
|
212
|
-
- **Retry statistics** (total attempts, successes, failures, average delay)
|
|
213
|
-
- Model performance (requests, successes, failures, response time)
|
|
263
|
+
The plugin includes a metrics collection feature that tracks:
|
|
264
|
+
- Rate limit events per provider/model
|
|
265
|
+
- Fallback statistics (total, successful, failed, average duration)
|
|
266
|
+
- **Retry statistics** (total attempts, successes, failures, average delay)
|
|
267
|
+
- Model performance (requests, successes, failures, response time)
|
|
268
|
+
- **Circuit breaker statistics** (state transitions, open/closed counts)
|
|
214
269
|
|
|
215
270
|
### Metrics Configuration
|
|
216
271
|
|
|
@@ -284,15 +339,28 @@ Retries:
|
|
|
284
339
|
|
|
285
340
|
Model Performance:
|
|
286
341
|
----------------------------------------
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
342
|
+
google/gemini-2.5-pro:
|
|
343
|
+
Requests: 10
|
|
344
|
+
Successes: 9
|
|
345
|
+
Failures: 1
|
|
346
|
+
Avg Response: 0.85s
|
|
347
|
+
Success Rate: 90.0%
|
|
348
|
+
|
|
349
|
+
Circuit Breaker:
|
|
350
|
+
----------------------------------------
|
|
351
|
+
anthropic/claude-3-5-sonnet-20250514:
|
|
352
|
+
State: OPEN
|
|
353
|
+
Failures: 5
|
|
354
|
+
Successes: 0
|
|
355
|
+
State Transitions: 2
|
|
356
|
+
google/gemini-2.5-pro:
|
|
357
|
+
State: CLOSED
|
|
358
|
+
Failures: 2
|
|
359
|
+
Successes: 8
|
|
360
|
+
State Transitions: 3
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
**JSON** (machine-readable):
|
|
296
364
|
```json
|
|
297
365
|
{
|
|
298
366
|
"rateLimits": {
|
|
@@ -332,18 +400,32 @@ Model Performance:
|
|
|
332
400
|
}
|
|
333
401
|
}
|
|
334
402
|
},
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
403
|
+
"modelPerformance": {
|
|
404
|
+
"google/gemini-2.5-pro": {
|
|
405
|
+
"requests": 10,
|
|
406
|
+
"successes": 9,
|
|
407
|
+
"failures": 1,
|
|
408
|
+
"averageResponseTime": 850
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
"circuitBreaker": {
|
|
412
|
+
"anthropic/claude-3-5-sonnet-20250514": {
|
|
413
|
+
"currentState": "OPEN",
|
|
414
|
+
"failures": 5,
|
|
415
|
+
"successes": 0,
|
|
416
|
+
"stateTransitions": 2
|
|
417
|
+
},
|
|
418
|
+
"google/gemini-2.5-pro": {
|
|
419
|
+
"currentState": "CLOSED",
|
|
420
|
+
"failures": 2,
|
|
421
|
+
"successes": 8,
|
|
422
|
+
"stateTransitions": 3
|
|
423
|
+
}
|
|
424
|
+
},
|
|
425
|
+
"startedAt": 1739148000000,
|
|
426
|
+
"generatedAt": 1739149800000
|
|
427
|
+
}
|
|
428
|
+
```
|
|
347
429
|
|
|
348
430
|
**CSV** (spreadsheet-friendly):
|
|
349
431
|
```
|
|
@@ -364,10 +446,15 @@ model,attempts,successes,success_rate
|
|
|
364
446
|
anthropic/claude-3-5-sonnet-20250514,5,3,60.0
|
|
365
447
|
google/gemini-2.5-pro,7,5,71.4
|
|
366
448
|
|
|
367
|
-
=== MODEL_PERFORMANCE ===
|
|
368
|
-
model,requests,successes,failures,avg_response_time_ms,success_rate
|
|
369
|
-
google/gemini-2.5-pro,10,9,1,850,90.0
|
|
370
|
-
|
|
449
|
+
=== MODEL_PERFORMANCE ===
|
|
450
|
+
model,requests,successes,failures,avg_response_time_ms,success_rate
|
|
451
|
+
google/gemini-2.5-pro,10,9,1,850,90.0
|
|
452
|
+
|
|
453
|
+
=== CIRCUIT_BREAKER ===
|
|
454
|
+
model,current_state,failures,successes,state_transitions
|
|
455
|
+
anthropic/claude-3-5-sonnet-20250514,OPEN,5,0,2
|
|
456
|
+
google/gemini-2.5-pro,CLOSED,2,8,3
|
|
457
|
+
```
|
|
371
458
|
|
|
372
459
|
## License
|
|
373
460
|
|
package/dist/index.d.ts
CHANGED
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
import type { Plugin } from "@opencode-ai/plugin";
|
|
7
7
|
export declare const RateLimitFallback: Plugin;
|
|
8
8
|
export default RateLimitFallback;
|
|
9
|
-
export type { PluginConfig, MetricsConfig, FallbackModel, FallbackMode } from "./src/types/index.js";
|
|
9
|
+
export type { PluginConfig, MetricsConfig, FallbackModel, FallbackMode, CircuitBreakerConfig, CircuitBreakerState, CircuitBreakerStateType } from "./src/types/index.js";
|
|
10
10
|
export type { LogConfig, Logger } from "./logger.js";
|
package/dist/index.js
CHANGED
|
@@ -53,8 +53,8 @@ function isSubagentSessionCreatedEvent(event) {
|
|
|
53
53
|
// ============================================================================
|
|
54
54
|
// Main Plugin Export
|
|
55
55
|
// ============================================================================
|
|
56
|
-
export const RateLimitFallback = async ({ client, directory }) => {
|
|
57
|
-
const config = loadConfig(directory);
|
|
56
|
+
export const RateLimitFallback = async ({ client, directory, worktree }) => {
|
|
57
|
+
const { config, source: configSource } = loadConfig(directory, worktree);
|
|
58
58
|
// Detect headless mode (no TUI)
|
|
59
59
|
const isHeadless = !client.tui;
|
|
60
60
|
// Auto-adjust log level for headless mode to ensure visibility
|
|
@@ -64,6 +64,12 @@ export const RateLimitFallback = async ({ client, directory }) => {
|
|
|
64
64
|
};
|
|
65
65
|
// Create logger instance
|
|
66
66
|
const logger = createLogger(logConfig, "RateLimitFallback");
|
|
67
|
+
if (configSource) {
|
|
68
|
+
logger.info(`Config loaded from ${configSource}`);
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
logger.info("No config file found, using defaults");
|
|
72
|
+
}
|
|
67
73
|
if (!config.enabled) {
|
|
68
74
|
return {};
|
|
69
75
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit Breaker - Manages circuit breakers for multiple models
|
|
3
|
+
*/
|
|
4
|
+
import type { Logger } from '../../logger.js';
|
|
5
|
+
import type { CircuitBreakerConfig, CircuitBreakerState, OpenCodeClient } from '../types/index.js';
|
|
6
|
+
import type { MetricsManager } from '../metrics/MetricsManager.js';
|
|
7
|
+
/**
|
|
8
|
+
* CircuitBreaker class - Manages circuit breaker logic for models
|
|
9
|
+
*/
|
|
10
|
+
export declare class CircuitBreaker {
|
|
11
|
+
private circuits;
|
|
12
|
+
private config;
|
|
13
|
+
private logger;
|
|
14
|
+
private metricsManager?;
|
|
15
|
+
private client?;
|
|
16
|
+
constructor(config: CircuitBreakerConfig, logger: Logger, metricsManager?: MetricsManager, client?: OpenCodeClient);
|
|
17
|
+
/**
|
|
18
|
+
* Check if a request should be allowed for a model
|
|
19
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
20
|
+
* @returns true if request is allowed, false otherwise
|
|
21
|
+
*/
|
|
22
|
+
canExecute(modelKey: string): boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Record a successful request for a model
|
|
25
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
26
|
+
*/
|
|
27
|
+
recordSuccess(modelKey: string): void;
|
|
28
|
+
/**
|
|
29
|
+
* Record a failed request for a model
|
|
30
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
31
|
+
* @param isRateLimit - true if the failure was due to rate limiting
|
|
32
|
+
*/
|
|
33
|
+
recordFailure(modelKey: string, isRateLimit: boolean): void;
|
|
34
|
+
/**
|
|
35
|
+
* Get the current state of a circuit
|
|
36
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
37
|
+
* @returns The current circuit state
|
|
38
|
+
*/
|
|
39
|
+
getState(modelKey: string): CircuitBreakerState;
|
|
40
|
+
/**
|
|
41
|
+
* Clean up stale entries from the circuits map
|
|
42
|
+
*/
|
|
43
|
+
cleanupStaleEntries(): void;
|
|
44
|
+
/**
|
|
45
|
+
* Get or create a circuit for a model
|
|
46
|
+
* @private
|
|
47
|
+
*/
|
|
48
|
+
private getOrCreateCircuit;
|
|
49
|
+
/**
|
|
50
|
+
* Destroy the circuit breaker and clean up resources
|
|
51
|
+
*/
|
|
52
|
+
destroy(): void;
|
|
53
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit Breaker - Manages circuit breakers for multiple models
|
|
3
|
+
*/
|
|
4
|
+
import { CircuitState } from './CircuitState.js';
|
|
5
|
+
import { safeShowToast } from '../utils/helpers.js';
|
|
6
|
+
/**
|
|
7
|
+
* CircuitBreaker class - Manages circuit breaker logic for models
|
|
8
|
+
*/
|
|
9
|
+
export class CircuitBreaker {
|
|
10
|
+
circuits;
|
|
11
|
+
config;
|
|
12
|
+
logger;
|
|
13
|
+
metricsManager;
|
|
14
|
+
client;
|
|
15
|
+
constructor(config, logger, metricsManager, client) {
|
|
16
|
+
this.config = config;
|
|
17
|
+
this.logger = logger;
|
|
18
|
+
this.metricsManager = metricsManager;
|
|
19
|
+
this.client = client;
|
|
20
|
+
this.circuits = new Map();
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Check if a request should be allowed for a model
|
|
24
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
25
|
+
* @returns true if request is allowed, false otherwise
|
|
26
|
+
*/
|
|
27
|
+
canExecute(modelKey) {
|
|
28
|
+
if (!this.config.enabled) {
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
const circuit = this.getOrCreateCircuit(modelKey);
|
|
32
|
+
const { allowed, transition } = circuit.canExecute();
|
|
33
|
+
const state = circuit.getState();
|
|
34
|
+
this.logger.debug(`Circuit breaker check for ${modelKey}`, {
|
|
35
|
+
state: state.state,
|
|
36
|
+
allowed,
|
|
37
|
+
failureCount: state.failureCount,
|
|
38
|
+
});
|
|
39
|
+
// Log and record transition if occurred
|
|
40
|
+
if (transition) {
|
|
41
|
+
const oldStateType = transition.from;
|
|
42
|
+
const newStateType = transition.to;
|
|
43
|
+
this.logger.info(`Circuit breaker state changed for ${modelKey}`, {
|
|
44
|
+
oldState: oldStateType,
|
|
45
|
+
newState: newStateType,
|
|
46
|
+
});
|
|
47
|
+
// Show toast notification for HALF_OPEN transition (recovery attempt)
|
|
48
|
+
if (newStateType === 'HALF_OPEN' && this.client) {
|
|
49
|
+
safeShowToast(this.client, {
|
|
50
|
+
body: {
|
|
51
|
+
title: "Circuit Recovery Attempt",
|
|
52
|
+
message: `Attempting recovery for ${modelKey} after ${this.config.recoveryTimeoutMs}ms`,
|
|
53
|
+
variant: "info",
|
|
54
|
+
duration: 3000,
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
// Record metrics
|
|
59
|
+
if (this.metricsManager) {
|
|
60
|
+
this.metricsManager.recordCircuitBreakerStateTransition(modelKey, oldStateType, newStateType);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return allowed;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Record a successful request for a model
|
|
67
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
68
|
+
*/
|
|
69
|
+
recordSuccess(modelKey) {
|
|
70
|
+
if (!this.config.enabled) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const circuit = this.getOrCreateCircuit(modelKey);
|
|
74
|
+
const oldState = circuit.getState().state;
|
|
75
|
+
circuit.onSuccess();
|
|
76
|
+
const newState = circuit.getState().state;
|
|
77
|
+
// Log state transition
|
|
78
|
+
if (oldState !== newState) {
|
|
79
|
+
this.logger.info(`Circuit breaker state changed for ${modelKey}`, {
|
|
80
|
+
oldState,
|
|
81
|
+
newState,
|
|
82
|
+
});
|
|
83
|
+
// Show toast notification for circuit close
|
|
84
|
+
if (newState === 'CLOSED' && oldState !== 'CLOSED' && this.client) {
|
|
85
|
+
safeShowToast(this.client, {
|
|
86
|
+
body: {
|
|
87
|
+
title: "Circuit Closed",
|
|
88
|
+
message: `Circuit breaker closed for ${modelKey} - service recovered`,
|
|
89
|
+
variant: "success",
|
|
90
|
+
duration: 3000,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Record metrics
|
|
95
|
+
if (this.metricsManager) {
|
|
96
|
+
this.metricsManager.recordCircuitBreakerStateTransition(modelKey, oldState, newState);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Record a failed request for a model
|
|
102
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
103
|
+
* @param isRateLimit - true if the failure was due to rate limiting
|
|
104
|
+
*/
|
|
105
|
+
recordFailure(modelKey, isRateLimit) {
|
|
106
|
+
if (!this.config.enabled) {
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
// Rate limit errors don't count as circuit failures
|
|
110
|
+
if (isRateLimit) {
|
|
111
|
+
this.logger.debug(`Rate limit error for ${modelKey}, not counting as circuit failure`);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const circuit = this.getOrCreateCircuit(modelKey);
|
|
115
|
+
const oldState = circuit.getState().state;
|
|
116
|
+
circuit.onFailure();
|
|
117
|
+
const newState = circuit.getState().state;
|
|
118
|
+
// Log state transition
|
|
119
|
+
if (oldState !== newState) {
|
|
120
|
+
this.logger.warn(`Circuit breaker state changed for ${modelKey}`, {
|
|
121
|
+
oldState,
|
|
122
|
+
newState,
|
|
123
|
+
failureCount: circuit.getState().failureCount,
|
|
124
|
+
});
|
|
125
|
+
// Show toast notification for circuit open
|
|
126
|
+
if (newState === 'OPEN' && this.client) {
|
|
127
|
+
safeShowToast(this.client, {
|
|
128
|
+
body: {
|
|
129
|
+
title: "Circuit Opened",
|
|
130
|
+
message: `Circuit breaker opened for ${modelKey} after failure threshold`,
|
|
131
|
+
variant: "warning",
|
|
132
|
+
duration: 5000,
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
// Show toast notification for circuit close
|
|
137
|
+
if (newState === 'CLOSED' && oldState !== 'CLOSED' && this.client) {
|
|
138
|
+
safeShowToast(this.client, {
|
|
139
|
+
body: {
|
|
140
|
+
title: "Circuit Closed",
|
|
141
|
+
message: `Circuit breaker closed for ${modelKey} - service recovered`,
|
|
142
|
+
variant: "success",
|
|
143
|
+
duration: 3000,
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
// Record metrics
|
|
148
|
+
if (this.metricsManager) {
|
|
149
|
+
this.metricsManager.recordCircuitBreakerStateTransition(modelKey, oldState, newState);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Get the current state of a circuit
|
|
155
|
+
* @param modelKey - The model key (providerID/modelID)
|
|
156
|
+
* @returns The current circuit state
|
|
157
|
+
*/
|
|
158
|
+
getState(modelKey) {
|
|
159
|
+
const circuit = this.circuits.get(modelKey);
|
|
160
|
+
if (!circuit) {
|
|
161
|
+
return {
|
|
162
|
+
state: 'CLOSED',
|
|
163
|
+
failureCount: 0,
|
|
164
|
+
successCount: 0,
|
|
165
|
+
lastFailureTime: 0,
|
|
166
|
+
lastSuccessTime: 0,
|
|
167
|
+
nextAttemptTime: 0,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
return circuit.getState();
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Clean up stale entries from the circuits map
|
|
174
|
+
*/
|
|
175
|
+
cleanupStaleEntries() {
|
|
176
|
+
const now = Date.now();
|
|
177
|
+
const cutoffTime = now - (24 * 60 * 60 * 1000); // 24 hours
|
|
178
|
+
for (const [key, circuit] of this.circuits.entries()) {
|
|
179
|
+
const state = circuit.getState();
|
|
180
|
+
const lastActivity = Math.max(state.lastFailureTime, state.lastSuccessTime);
|
|
181
|
+
// Remove circuits that haven't been active for 24 hours
|
|
182
|
+
if (lastActivity < cutoffTime) {
|
|
183
|
+
this.circuits.delete(key);
|
|
184
|
+
this.logger.debug(`Cleaned up stale circuit for ${key}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Get or create a circuit for a model
|
|
190
|
+
* @private
|
|
191
|
+
*/
|
|
192
|
+
getOrCreateCircuit(modelKey) {
|
|
193
|
+
let circuit = this.circuits.get(modelKey);
|
|
194
|
+
if (!circuit) {
|
|
195
|
+
circuit = new CircuitState(this.config);
|
|
196
|
+
this.circuits.set(modelKey, circuit);
|
|
197
|
+
this.logger.debug(`Created new circuit for ${modelKey}`);
|
|
198
|
+
}
|
|
199
|
+
return circuit;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Destroy the circuit breaker and clean up resources
|
|
203
|
+
*/
|
|
204
|
+
destroy() {
|
|
205
|
+
this.circuits.clear();
|
|
206
|
+
this.logger.debug('Circuit breaker destroyed');
|
|
207
|
+
}
|
|
208
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit State - State machine for individual circuit breaker state
|
|
3
|
+
*/
|
|
4
|
+
import type { CircuitBreakerConfig, CircuitBreakerState, CircuitBreakerStateType } from '../types/index.js';
|
|
5
|
+
/**
|
|
6
|
+
* Return type for canExecute method
|
|
7
|
+
*/
|
|
8
|
+
export interface CanExecuteResult {
|
|
9
|
+
allowed: boolean;
|
|
10
|
+
transition?: {
|
|
11
|
+
from: CircuitBreakerStateType;
|
|
12
|
+
to: CircuitBreakerStateType;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* CircuitState class - Manages state transitions for a single circuit
|
|
17
|
+
*/
|
|
18
|
+
export declare class CircuitState {
|
|
19
|
+
state: CircuitBreakerState;
|
|
20
|
+
private halfOpenCalls;
|
|
21
|
+
private config;
|
|
22
|
+
constructor(config: CircuitBreakerConfig);
|
|
23
|
+
/**
|
|
24
|
+
* Handle a successful request
|
|
25
|
+
*/
|
|
26
|
+
onSuccess(): void;
|
|
27
|
+
/**
|
|
28
|
+
* Handle a failed request
|
|
29
|
+
*/
|
|
30
|
+
onFailure(): void;
|
|
31
|
+
/**
|
|
32
|
+
* Check if a request can be executed through this circuit
|
|
33
|
+
* @returns Object with allowed flag and optional transition info
|
|
34
|
+
*/
|
|
35
|
+
canExecute(): CanExecuteResult;
|
|
36
|
+
/**
|
|
37
|
+
* Get the current state
|
|
38
|
+
*/
|
|
39
|
+
getState(): CircuitBreakerState;
|
|
40
|
+
/**
|
|
41
|
+
* Reset the circuit to CLOSED state
|
|
42
|
+
*/
|
|
43
|
+
reset(): void;
|
|
44
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit State - State machine for individual circuit breaker state
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* CircuitState class - Manages state transitions for a single circuit
|
|
6
|
+
*/
|
|
7
|
+
export class CircuitState {
|
|
8
|
+
state;
|
|
9
|
+
halfOpenCalls = 0;
|
|
10
|
+
config;
|
|
11
|
+
constructor(config) {
|
|
12
|
+
this.config = config;
|
|
13
|
+
this.state = {
|
|
14
|
+
state: 'CLOSED',
|
|
15
|
+
failureCount: 0,
|
|
16
|
+
successCount: 0,
|
|
17
|
+
lastFailureTime: 0,
|
|
18
|
+
lastSuccessTime: 0,
|
|
19
|
+
nextAttemptTime: 0,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Handle a successful request
|
|
24
|
+
*/
|
|
25
|
+
onSuccess() {
|
|
26
|
+
const now = Date.now();
|
|
27
|
+
this.state.lastSuccessTime = now;
|
|
28
|
+
switch (this.state.state) {
|
|
29
|
+
case 'CLOSED':
|
|
30
|
+
// Reset failure count on success
|
|
31
|
+
this.state.failureCount = 0;
|
|
32
|
+
break;
|
|
33
|
+
case 'HALF_OPEN':
|
|
34
|
+
this.state.successCount++;
|
|
35
|
+
this.state.failureCount = 0;
|
|
36
|
+
// Close circuit if success threshold reached
|
|
37
|
+
if (this.state.successCount >= this.config.successThreshold) {
|
|
38
|
+
this.state.state = 'CLOSED';
|
|
39
|
+
this.state.successCount = 0;
|
|
40
|
+
this.halfOpenCalls = 0;
|
|
41
|
+
}
|
|
42
|
+
break;
|
|
43
|
+
case 'OPEN':
|
|
44
|
+
// Should not receive success in OPEN state
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Handle a failed request
|
|
50
|
+
*/
|
|
51
|
+
onFailure() {
|
|
52
|
+
const now = Date.now();
|
|
53
|
+
this.state.lastFailureTime = now;
|
|
54
|
+
switch (this.state.state) {
|
|
55
|
+
case 'CLOSED':
|
|
56
|
+
this.state.failureCount++;
|
|
57
|
+
// Open circuit if failure threshold reached
|
|
58
|
+
if (this.state.failureCount >= this.config.failureThreshold) {
|
|
59
|
+
this.state.state = 'OPEN';
|
|
60
|
+
this.state.nextAttemptTime = now + this.config.recoveryTimeoutMs;
|
|
61
|
+
this.state.successCount = 0;
|
|
62
|
+
}
|
|
63
|
+
break;
|
|
64
|
+
case 'HALF_OPEN':
|
|
65
|
+
// Re-open circuit on failure
|
|
66
|
+
this.state.state = 'OPEN';
|
|
67
|
+
this.state.nextAttemptTime = now + this.config.recoveryTimeoutMs;
|
|
68
|
+
this.state.failureCount++;
|
|
69
|
+
this.state.successCount = 0;
|
|
70
|
+
this.halfOpenCalls = 0;
|
|
71
|
+
break;
|
|
72
|
+
case 'OPEN':
|
|
73
|
+
// Already open, just update count
|
|
74
|
+
this.state.failureCount++;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Check if a request can be executed through this circuit
|
|
80
|
+
* @returns Object with allowed flag and optional transition info
|
|
81
|
+
*/
|
|
82
|
+
canExecute() {
|
|
83
|
+
const now = Date.now();
|
|
84
|
+
switch (this.state.state) {
|
|
85
|
+
case 'CLOSED':
|
|
86
|
+
return { allowed: true };
|
|
87
|
+
case 'OPEN':
|
|
88
|
+
// Check if recovery timeout has elapsed
|
|
89
|
+
if (now >= this.state.nextAttemptTime) {
|
|
90
|
+
// Transition to HALF_OPEN for test request
|
|
91
|
+
const transition = { from: 'OPEN', to: 'HALF_OPEN' };
|
|
92
|
+
this.state.state = 'HALF_OPEN';
|
|
93
|
+
this.halfOpenCalls = 0;
|
|
94
|
+
return { allowed: true, transition };
|
|
95
|
+
}
|
|
96
|
+
return { allowed: false };
|
|
97
|
+
case 'HALF_OPEN':
|
|
98
|
+
// Limit calls in HALF_OPEN state
|
|
99
|
+
if (this.halfOpenCalls < this.config.halfOpenMaxCalls) {
|
|
100
|
+
this.halfOpenCalls++;
|
|
101
|
+
return { allowed: true };
|
|
102
|
+
}
|
|
103
|
+
return { allowed: false };
|
|
104
|
+
default:
|
|
105
|
+
return { allowed: false };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Get the current state
|
|
110
|
+
*/
|
|
111
|
+
getState() {
|
|
112
|
+
return { ...this.state };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Reset the circuit to CLOSED state
|
|
116
|
+
*/
|
|
117
|
+
reset() {
|
|
118
|
+
this.state = {
|
|
119
|
+
state: 'CLOSED',
|
|
120
|
+
failureCount: 0,
|
|
121
|
+
successCount: 0,
|
|
122
|
+
lastFailureTime: 0,
|
|
123
|
+
lastSuccessTime: 0,
|
|
124
|
+
nextAttemptTime: 0,
|
|
125
|
+
};
|
|
126
|
+
this.halfOpenCalls = 0;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit Breaker Module
|
|
3
|
+
*
|
|
4
|
+
* Provides circuit breaker pattern implementation to prevent cascading failures
|
|
5
|
+
* by automatically disabling models that are consistently failing.
|
|
6
|
+
*/
|
|
7
|
+
export { CircuitBreaker } from './CircuitBreaker.js';
|
|
8
|
+
export { CircuitState, type CanExecuteResult } from './CircuitState.js';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Circuit Breaker Module
|
|
3
|
+
*
|
|
4
|
+
* Provides circuit breaker pattern implementation to prevent cascading failures
|
|
5
|
+
* by automatically disabling models that are consistently failing.
|
|
6
|
+
*/
|
|
7
|
+
export { CircuitBreaker } from './CircuitBreaker.js';
|
|
8
|
+
export { CircuitState } from './CircuitState.js';
|
|
@@ -21,6 +21,7 @@ export declare class FallbackHandler {
|
|
|
21
21
|
private metricsManager;
|
|
22
22
|
private subagentTracker;
|
|
23
23
|
private retryManager;
|
|
24
|
+
private circuitBreaker?;
|
|
24
25
|
constructor(config: PluginConfig, client: OpenCodeClient, logger: Logger, metricsManager: MetricsManager, subagentTracker: SubagentTracker);
|
|
25
26
|
/**
|
|
26
27
|
* Check and mark fallback in progress for deduplication
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { SESSION_ENTRY_TTL_MS } from '../types/index.js';
|
|
5
5
|
import { ModelSelector } from './ModelSelector.js';
|
|
6
|
+
import { CircuitBreaker } from '../circuitbreaker/CircuitBreaker.js';
|
|
6
7
|
import { extractMessageParts, convertPartsToSDKFormat, safeShowToast, getStateKey, getModelKey, DEDUP_WINDOW_MS, STATE_TIMEOUT_MS } from '../utils/helpers.js';
|
|
7
8
|
import { RetryManager } from '../retry/RetryManager.js';
|
|
8
9
|
/**
|
|
@@ -24,13 +25,19 @@ export class FallbackHandler {
|
|
|
24
25
|
subagentTracker;
|
|
25
26
|
// Retry manager reference
|
|
26
27
|
retryManager;
|
|
28
|
+
// Circuit breaker reference
|
|
29
|
+
circuitBreaker;
|
|
27
30
|
constructor(config, client, logger, metricsManager, subagentTracker) {
|
|
28
31
|
this.config = config;
|
|
29
32
|
this.client = client;
|
|
30
33
|
this.logger = logger;
|
|
31
|
-
this.modelSelector = new ModelSelector(config, client);
|
|
32
34
|
this.metricsManager = metricsManager;
|
|
33
35
|
this.subagentTracker = subagentTracker;
|
|
36
|
+
// Initialize circuit breaker if enabled
|
|
37
|
+
if (config.circuitBreaker?.enabled) {
|
|
38
|
+
this.circuitBreaker = new CircuitBreaker(config.circuitBreaker, logger, metricsManager, client);
|
|
39
|
+
}
|
|
40
|
+
this.modelSelector = new ModelSelector(config, client, this.circuitBreaker);
|
|
34
41
|
this.currentSessionModel = new Map();
|
|
35
42
|
this.modelRequestStartTimes = new Map();
|
|
36
43
|
this.retryState = new Map();
|
|
@@ -301,6 +308,11 @@ export class FallbackHandler {
|
|
|
301
308
|
// Non-rate-limit error - record model failure metric
|
|
302
309
|
const tracked = this.currentSessionModel.get(sessionID);
|
|
303
310
|
if (tracked) {
|
|
311
|
+
// Record failure to circuit breaker (isRateLimit = false)
|
|
312
|
+
if (this.circuitBreaker) {
|
|
313
|
+
const modelKey = getModelKey(tracked.providerID, tracked.modelID);
|
|
314
|
+
this.circuitBreaker.recordFailure(modelKey, false);
|
|
315
|
+
}
|
|
304
316
|
if (this.metricsManager) {
|
|
305
317
|
this.metricsManager.recordModelFailure(tracked.providerID, tracked.modelID);
|
|
306
318
|
// Check if this was a fallback attempt and record failure
|
|
@@ -326,6 +338,11 @@ export class FallbackHandler {
|
|
|
326
338
|
// Record fallback success metric
|
|
327
339
|
const tracked = this.currentSessionModel.get(sessionID);
|
|
328
340
|
if (tracked) {
|
|
341
|
+
// Record success to circuit breaker
|
|
342
|
+
if (this.circuitBreaker) {
|
|
343
|
+
const modelKey = getModelKey(tracked.providerID, tracked.modelID);
|
|
344
|
+
this.circuitBreaker.recordSuccess(modelKey);
|
|
345
|
+
}
|
|
329
346
|
if (this.metricsManager) {
|
|
330
347
|
this.metricsManager.recordFallbackSuccess(tracked.providerID, tracked.modelID, fallbackInfo.timestamp);
|
|
331
348
|
// Record model performance metric
|
|
@@ -374,6 +391,10 @@ export class FallbackHandler {
|
|
|
374
391
|
}
|
|
375
392
|
this.modelSelector.cleanupStaleEntries();
|
|
376
393
|
this.retryManager.cleanupStaleEntries(SESSION_ENTRY_TTL_MS);
|
|
394
|
+
// Clean up circuit breaker stale entries
|
|
395
|
+
if (this.circuitBreaker) {
|
|
396
|
+
this.circuitBreaker.cleanupStaleEntries();
|
|
397
|
+
}
|
|
377
398
|
}
|
|
378
399
|
/**
|
|
379
400
|
* Clean up all resources
|
|
@@ -385,5 +406,9 @@ export class FallbackHandler {
|
|
|
385
406
|
this.fallbackInProgress.clear();
|
|
386
407
|
this.fallbackMessages.clear();
|
|
387
408
|
this.retryManager.destroy();
|
|
409
|
+
// Destroy circuit breaker
|
|
410
|
+
if (this.circuitBreaker) {
|
|
411
|
+
this.circuitBreaker.destroy();
|
|
412
|
+
}
|
|
388
413
|
}
|
|
389
414
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Model selection logic based on fallback mode
|
|
3
3
|
*/
|
|
4
4
|
import type { FallbackModel, PluginConfig, OpenCodeClient } from '../types/index.js';
|
|
5
|
+
import type { CircuitBreaker } from '../circuitbreaker/index.js';
|
|
5
6
|
/**
|
|
6
7
|
* Model Selector class for handling model selection strategies
|
|
7
8
|
*/
|
|
@@ -9,7 +10,8 @@ export declare class ModelSelector {
|
|
|
9
10
|
private rateLimitedModels;
|
|
10
11
|
private config;
|
|
11
12
|
private client;
|
|
12
|
-
|
|
13
|
+
private circuitBreaker?;
|
|
14
|
+
constructor(config: PluginConfig, client: OpenCodeClient, circuitBreaker?: CircuitBreaker);
|
|
13
15
|
/**
|
|
14
16
|
* Check if a model is currently rate limited
|
|
15
17
|
*/
|
|
@@ -22,6 +24,10 @@ export declare class ModelSelector {
|
|
|
22
24
|
* Find the next available model that is not rate limited
|
|
23
25
|
*/
|
|
24
26
|
private findNextAvailableModel;
|
|
27
|
+
/**
|
|
28
|
+
* Check if a model is available (not rate limited and not blocked by circuit breaker)
|
|
29
|
+
*/
|
|
30
|
+
private isModelAvailable;
|
|
25
31
|
/**
|
|
26
32
|
* Apply the fallback mode logic
|
|
27
33
|
*/
|
|
@@ -10,9 +10,11 @@ export class ModelSelector {
|
|
|
10
10
|
rateLimitedModels;
|
|
11
11
|
config;
|
|
12
12
|
client;
|
|
13
|
-
|
|
13
|
+
circuitBreaker;
|
|
14
|
+
constructor(config, client, circuitBreaker) {
|
|
14
15
|
this.config = config;
|
|
15
16
|
this.client = client;
|
|
17
|
+
this.circuitBreaker = circuitBreaker;
|
|
16
18
|
this.rateLimitedModels = new Map();
|
|
17
19
|
}
|
|
18
20
|
/**
|
|
@@ -48,7 +50,7 @@ export class ModelSelector {
|
|
|
48
50
|
for (let i = searchStartIndex + 1; i < this.config.fallbackModels.length; i++) {
|
|
49
51
|
const model = this.config.fallbackModels[i];
|
|
50
52
|
const key = getModelKey(model.providerID, model.modelID);
|
|
51
|
-
if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID)) {
|
|
53
|
+
if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID) && this.isModelAvailable(model.providerID, model.modelID)) {
|
|
52
54
|
return model;
|
|
53
55
|
}
|
|
54
56
|
}
|
|
@@ -56,12 +58,23 @@ export class ModelSelector {
|
|
|
56
58
|
for (let i = 0; i <= searchStartIndex && i < this.config.fallbackModels.length; i++) {
|
|
57
59
|
const model = this.config.fallbackModels[i];
|
|
58
60
|
const key = getModelKey(model.providerID, model.modelID);
|
|
59
|
-
if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID)) {
|
|
61
|
+
if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID) && this.isModelAvailable(model.providerID, model.modelID)) {
|
|
60
62
|
return model;
|
|
61
63
|
}
|
|
62
64
|
}
|
|
63
65
|
return null;
|
|
64
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Check if a model is available (not rate limited and not blocked by circuit breaker)
|
|
69
|
+
*/
|
|
70
|
+
isModelAvailable(providerID, modelID) {
|
|
71
|
+
// Check circuit breaker if enabled
|
|
72
|
+
if (this.circuitBreaker && this.config.circuitBreaker?.enabled) {
|
|
73
|
+
const modelKey = getModelKey(providerID, modelID);
|
|
74
|
+
return this.circuitBreaker.canExecute(modelKey);
|
|
75
|
+
}
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
65
78
|
/**
|
|
66
79
|
* Apply the fallback mode logic
|
|
67
80
|
*/
|
|
@@ -79,7 +92,7 @@ export class ModelSelector {
|
|
|
79
92
|
const lastModel = this.config.fallbackModels[this.config.fallbackModels.length - 1];
|
|
80
93
|
if (lastModel) {
|
|
81
94
|
const isLastModelCurrent = currentProviderID === lastModel.providerID && currentModelID === lastModel.modelID;
|
|
82
|
-
if (!isLastModelCurrent && !this.isModelRateLimited(lastModel.providerID, lastModel.modelID)) {
|
|
95
|
+
if (!isLastModelCurrent && !this.isModelRateLimited(lastModel.providerID, lastModel.modelID) && this.isModelAvailable(lastModel.providerID, lastModel.modelID)) {
|
|
83
96
|
// Use the last model for one more try
|
|
84
97
|
safeShowToast(this.client, {
|
|
85
98
|
body: {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Metrics Manager - Handles metrics collection, aggregation, and reporting
|
|
3
3
|
*/
|
|
4
4
|
import type { Logger } from '../../logger.js';
|
|
5
|
-
import type { MetricsConfig, MetricsData, RateLimitMetrics, FallbackTargetMetrics, ModelPerformanceMetrics } from '../types/index.js';
|
|
5
|
+
import type { MetricsConfig, MetricsData, RateLimitMetrics, FallbackTargetMetrics, ModelPerformanceMetrics, CircuitBreakerStateType } from '../types/index.js';
|
|
6
6
|
/**
|
|
7
7
|
* Metrics Manager class for collecting and reporting metrics
|
|
8
8
|
*/
|
|
@@ -61,6 +61,15 @@ export declare class MetricsManager {
|
|
|
61
61
|
* Record a failed retry
|
|
62
62
|
*/
|
|
63
63
|
recordRetryFailure(): void;
|
|
64
|
+
/**
|
|
65
|
+
* Record a circuit breaker state transition
|
|
66
|
+
*/
|
|
67
|
+
recordCircuitBreakerStateTransition(modelKey: string, oldState: CircuitBreakerStateType, newState: CircuitBreakerStateType): void;
|
|
68
|
+
/**
|
|
69
|
+
* Helper method to update circuit breaker state counts
|
|
70
|
+
* @private
|
|
71
|
+
*/
|
|
72
|
+
private updateCircuitBreakerStateCounts;
|
|
64
73
|
/**
|
|
65
74
|
* Get a copy of the current metrics
|
|
66
75
|
*/
|
|
@@ -31,6 +31,18 @@ export class MetricsManager {
|
|
|
31
31
|
byModel: new Map(),
|
|
32
32
|
},
|
|
33
33
|
modelPerformance: new Map(),
|
|
34
|
+
circuitBreaker: {
|
|
35
|
+
total: {
|
|
36
|
+
stateTransitions: 0,
|
|
37
|
+
opens: 0,
|
|
38
|
+
closes: 0,
|
|
39
|
+
halfOpens: 0,
|
|
40
|
+
currentOpen: 0,
|
|
41
|
+
currentHalfOpen: 0,
|
|
42
|
+
currentClosed: 0,
|
|
43
|
+
},
|
|
44
|
+
byModel: new Map(),
|
|
45
|
+
},
|
|
34
46
|
startedAt: Date.now(),
|
|
35
47
|
generatedAt: Date.now(),
|
|
36
48
|
};
|
|
@@ -71,6 +83,18 @@ export class MetricsManager {
|
|
|
71
83
|
byModel: new Map(),
|
|
72
84
|
},
|
|
73
85
|
modelPerformance: new Map(),
|
|
86
|
+
circuitBreaker: {
|
|
87
|
+
total: {
|
|
88
|
+
stateTransitions: 0,
|
|
89
|
+
opens: 0,
|
|
90
|
+
closes: 0,
|
|
91
|
+
halfOpens: 0,
|
|
92
|
+
currentOpen: 0,
|
|
93
|
+
currentHalfOpen: 0,
|
|
94
|
+
currentClosed: 0,
|
|
95
|
+
},
|
|
96
|
+
byModel: new Map(),
|
|
97
|
+
},
|
|
74
98
|
startedAt: Date.now(),
|
|
75
99
|
generatedAt: Date.now(),
|
|
76
100
|
};
|
|
@@ -229,6 +253,62 @@ export class MetricsManager {
|
|
|
229
253
|
return;
|
|
230
254
|
this.metrics.retries.failed++;
|
|
231
255
|
}
|
|
256
|
+
/**
|
|
257
|
+
* Record a circuit breaker state transition
|
|
258
|
+
*/
|
|
259
|
+
recordCircuitBreakerStateTransition(modelKey, oldState, newState) {
|
|
260
|
+
if (!this.config.enabled)
|
|
261
|
+
return;
|
|
262
|
+
// Update total metrics
|
|
263
|
+
this.metrics.circuitBreaker.total.stateTransitions++;
|
|
264
|
+
this.updateCircuitBreakerStateCounts(this.metrics.circuitBreaker.total, oldState, newState);
|
|
265
|
+
// Update model-specific metrics
|
|
266
|
+
let modelMetrics = this.metrics.circuitBreaker.byModel.get(modelKey);
|
|
267
|
+
if (!modelMetrics) {
|
|
268
|
+
modelMetrics = {
|
|
269
|
+
stateTransitions: 0,
|
|
270
|
+
opens: 0,
|
|
271
|
+
closes: 0,
|
|
272
|
+
halfOpens: 0,
|
|
273
|
+
currentOpen: 0,
|
|
274
|
+
currentHalfOpen: 0,
|
|
275
|
+
currentClosed: 1, // Start with CLOSED
|
|
276
|
+
};
|
|
277
|
+
this.metrics.circuitBreaker.byModel.set(modelKey, modelMetrics);
|
|
278
|
+
}
|
|
279
|
+
modelMetrics.stateTransitions++;
|
|
280
|
+
this.updateCircuitBreakerStateCounts(modelMetrics, oldState, newState);
|
|
281
|
+
this.metrics.circuitBreaker.byModel.set(modelKey, modelMetrics);
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Helper method to update circuit breaker state counts
|
|
285
|
+
* @private
|
|
286
|
+
*/
|
|
287
|
+
updateCircuitBreakerStateCounts(metrics, oldState, newState) {
|
|
288
|
+
// Update state counts based on old state
|
|
289
|
+
if (oldState === 'OPEN') {
|
|
290
|
+
metrics.currentOpen--;
|
|
291
|
+
}
|
|
292
|
+
else if (oldState === 'HALF_OPEN') {
|
|
293
|
+
metrics.currentHalfOpen--;
|
|
294
|
+
}
|
|
295
|
+
else if (oldState === 'CLOSED') {
|
|
296
|
+
metrics.currentClosed--;
|
|
297
|
+
}
|
|
298
|
+
// Update state counts based on new state
|
|
299
|
+
if (newState === 'OPEN') {
|
|
300
|
+
metrics.opens++;
|
|
301
|
+
metrics.currentOpen++;
|
|
302
|
+
}
|
|
303
|
+
else if (newState === 'HALF_OPEN') {
|
|
304
|
+
metrics.halfOpens++;
|
|
305
|
+
metrics.currentHalfOpen++;
|
|
306
|
+
}
|
|
307
|
+
else if (newState === 'CLOSED') {
|
|
308
|
+
metrics.closes++;
|
|
309
|
+
metrics.currentClosed++;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
232
312
|
/**
|
|
233
313
|
* Get a copy of the current metrics
|
|
234
314
|
*/
|
|
@@ -266,6 +346,10 @@ export class MetricsManager {
|
|
|
266
346
|
byModel: Object.fromEntries(Array.from(metrics.retries.byModel.entries()).map(([k, v]) => [k, v])),
|
|
267
347
|
},
|
|
268
348
|
modelPerformance: Object.fromEntries(Array.from(metrics.modelPerformance.entries()).map(([k, v]) => [k, v])),
|
|
349
|
+
circuitBreaker: {
|
|
350
|
+
...metrics.circuitBreaker,
|
|
351
|
+
byModel: Object.fromEntries(Array.from(metrics.circuitBreaker.byModel.entries()).map(([k, v]) => [k, v])),
|
|
352
|
+
},
|
|
269
353
|
startedAt: metrics.startedAt,
|
|
270
354
|
generatedAt: metrics.generatedAt,
|
|
271
355
|
};
|
|
@@ -342,6 +426,31 @@ export class MetricsManager {
|
|
|
342
426
|
}
|
|
343
427
|
}
|
|
344
428
|
lines.push("");
|
|
429
|
+
// Circuit Breaker
|
|
430
|
+
lines.push("Circuit Breaker:");
|
|
431
|
+
lines.push("-".repeat(40));
|
|
432
|
+
lines.push(` State Transitions: ${this.metrics.circuitBreaker.total.stateTransitions}`);
|
|
433
|
+
lines.push(` Opens: ${this.metrics.circuitBreaker.total.opens}`);
|
|
434
|
+
lines.push(` Closes: ${this.metrics.circuitBreaker.total.closes}`);
|
|
435
|
+
lines.push(` Half Opens: ${this.metrics.circuitBreaker.total.halfOpens}`);
|
|
436
|
+
lines.push("");
|
|
437
|
+
lines.push(" Current State Distribution:");
|
|
438
|
+
lines.push(` CLOSED: ${this.metrics.circuitBreaker.total.currentClosed}`);
|
|
439
|
+
lines.push(` HALF_OPEN: ${this.metrics.circuitBreaker.total.currentHalfOpen}`);
|
|
440
|
+
lines.push(` OPEN: ${this.metrics.circuitBreaker.total.currentOpen}`);
|
|
441
|
+
if (this.metrics.circuitBreaker.byModel.size > 0) {
|
|
442
|
+
lines.push("");
|
|
443
|
+
lines.push(" By Model:");
|
|
444
|
+
for (const [model, data] of this.metrics.circuitBreaker.byModel.entries()) {
|
|
445
|
+
lines.push(` ${model}:`);
|
|
446
|
+
lines.push(` State Transitions: ${data.stateTransitions}`);
|
|
447
|
+
lines.push(` Opens: ${data.opens}`);
|
|
448
|
+
lines.push(` Closes: ${data.closes}`);
|
|
449
|
+
lines.push(` Half Opens: ${data.halfOpens}`);
|
|
450
|
+
lines.push(` Current State: ${data.currentOpen > 0 ? 'OPEN' : data.currentHalfOpen > 0 ? 'HALF_OPEN' : 'CLOSED'}`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
lines.push("");
|
|
345
454
|
// Model Performance
|
|
346
455
|
lines.push("Model Performance:");
|
|
347
456
|
lines.push("-".repeat(40));
|
|
@@ -428,6 +537,34 @@ export class MetricsManager {
|
|
|
428
537
|
].join(","));
|
|
429
538
|
}
|
|
430
539
|
lines.push("");
|
|
540
|
+
// Circuit Breaker Summary CSV
|
|
541
|
+
lines.push("=== CIRCUIT_BREAKER_SUMMARY ===");
|
|
542
|
+
lines.push(`state_transitions,opens,closes,half_opens,current_open,current_half_open,current_closed`);
|
|
543
|
+
lines.push([
|
|
544
|
+
this.metrics.circuitBreaker.total.stateTransitions,
|
|
545
|
+
this.metrics.circuitBreaker.total.opens,
|
|
546
|
+
this.metrics.circuitBreaker.total.closes,
|
|
547
|
+
this.metrics.circuitBreaker.total.halfOpens,
|
|
548
|
+
this.metrics.circuitBreaker.total.currentOpen,
|
|
549
|
+
this.metrics.circuitBreaker.total.currentHalfOpen,
|
|
550
|
+
this.metrics.circuitBreaker.total.currentClosed,
|
|
551
|
+
].join(","));
|
|
552
|
+
lines.push("");
|
|
553
|
+
// Circuit Breaker by Model CSV
|
|
554
|
+
lines.push("=== CIRCUIT_BREAKER_BY_MODEL ===");
|
|
555
|
+
lines.push("model,state_transitions,opens,closes,half_opens,current_state");
|
|
556
|
+
for (const [model, data] of this.metrics.circuitBreaker.byModel.entries()) {
|
|
557
|
+
const currentState = data.currentOpen > 0 ? 'OPEN' : data.currentHalfOpen > 0 ? 'HALF_OPEN' : 'CLOSED';
|
|
558
|
+
lines.push([
|
|
559
|
+
model,
|
|
560
|
+
data.stateTransitions,
|
|
561
|
+
data.opens,
|
|
562
|
+
data.closes,
|
|
563
|
+
data.halfOpens,
|
|
564
|
+
currentState,
|
|
565
|
+
].join(","));
|
|
566
|
+
}
|
|
567
|
+
lines.push("");
|
|
431
568
|
// Model Performance CSV
|
|
432
569
|
lines.push("=== MODEL_PERFORMANCE ===");
|
|
433
570
|
lines.push("model,requests,successes,failures,avg_response_time_ms,success_rate");
|
|
@@ -33,6 +33,31 @@ export interface RetryPolicy {
|
|
|
33
33
|
jitterFactor: number;
|
|
34
34
|
timeoutMs?: number;
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Circuit breaker state
|
|
38
|
+
*/
|
|
39
|
+
export type CircuitBreakerStateType = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
|
|
40
|
+
/**
|
|
41
|
+
* Circuit breaker configuration
|
|
42
|
+
*/
|
|
43
|
+
export interface CircuitBreakerConfig {
|
|
44
|
+
enabled: boolean;
|
|
45
|
+
failureThreshold: number;
|
|
46
|
+
recoveryTimeoutMs: number;
|
|
47
|
+
halfOpenMaxCalls: number;
|
|
48
|
+
successThreshold: number;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Circuit breaker state data
|
|
52
|
+
*/
|
|
53
|
+
export interface CircuitBreakerState {
|
|
54
|
+
state: CircuitBreakerStateType;
|
|
55
|
+
failureCount: number;
|
|
56
|
+
successCount: number;
|
|
57
|
+
lastFailureTime: number;
|
|
58
|
+
lastSuccessTime: number;
|
|
59
|
+
nextAttemptTime: number;
|
|
60
|
+
}
|
|
36
61
|
/**
|
|
37
62
|
* Metrics output configuration
|
|
38
63
|
*/
|
|
@@ -60,6 +85,7 @@ export interface PluginConfig {
|
|
|
60
85
|
maxSubagentDepth?: number;
|
|
61
86
|
enableSubagentFallback?: boolean;
|
|
62
87
|
retryPolicy?: RetryPolicy;
|
|
88
|
+
circuitBreaker?: CircuitBreakerConfig;
|
|
63
89
|
log?: LogConfig;
|
|
64
90
|
metrics?: MetricsConfig;
|
|
65
91
|
}
|
|
@@ -183,6 +209,18 @@ export interface ModelPerformanceMetrics {
|
|
|
183
209
|
failures: number;
|
|
184
210
|
averageResponseTime?: number;
|
|
185
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Circuit breaker metrics
|
|
214
|
+
*/
|
|
215
|
+
export interface CircuitBreakerMetrics {
|
|
216
|
+
stateTransitions: number;
|
|
217
|
+
opens: number;
|
|
218
|
+
closes: number;
|
|
219
|
+
halfOpens: number;
|
|
220
|
+
currentOpen: number;
|
|
221
|
+
currentHalfOpen: number;
|
|
222
|
+
currentClosed: number;
|
|
223
|
+
}
|
|
186
224
|
/**
|
|
187
225
|
* Retry metrics
|
|
188
226
|
*/
|
|
@@ -210,6 +248,10 @@ export interface MetricsData {
|
|
|
210
248
|
};
|
|
211
249
|
retries: RetryMetrics;
|
|
212
250
|
modelPerformance: Map<string, ModelPerformanceMetrics>;
|
|
251
|
+
circuitBreaker: {
|
|
252
|
+
total: CircuitBreakerMetrics;
|
|
253
|
+
byModel: Map<string, CircuitBreakerMetrics>;
|
|
254
|
+
};
|
|
213
255
|
startedAt: number;
|
|
214
256
|
generatedAt: number;
|
|
215
257
|
}
|
|
@@ -314,6 +356,10 @@ export declare const DEFAULT_FALLBACK_MODELS: FallbackModel[];
|
|
|
314
356
|
* Default retry policy
|
|
315
357
|
*/
|
|
316
358
|
export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
|
|
359
|
+
/**
|
|
360
|
+
* Default circuit breaker configuration
|
|
361
|
+
*/
|
|
362
|
+
export declare const DEFAULT_CIRCUIT_BREAKER_CONFIG: CircuitBreakerConfig;
|
|
317
363
|
/**
|
|
318
364
|
* Valid fallback modes
|
|
319
365
|
*/
|
package/dist/src/types/index.js
CHANGED
|
@@ -23,6 +23,16 @@ export const DEFAULT_RETRY_POLICY = {
|
|
|
23
23
|
jitterEnabled: false,
|
|
24
24
|
jitterFactor: 0.1,
|
|
25
25
|
};
|
|
26
|
+
/**
|
|
27
|
+
* Default circuit breaker configuration
|
|
28
|
+
*/
|
|
29
|
+
export const DEFAULT_CIRCUIT_BREAKER_CONFIG = {
|
|
30
|
+
enabled: false,
|
|
31
|
+
failureThreshold: 5,
|
|
32
|
+
recoveryTimeoutMs: 60000,
|
|
33
|
+
halfOpenMaxCalls: 1,
|
|
34
|
+
successThreshold: 2,
|
|
35
|
+
};
|
|
26
36
|
/**
|
|
27
37
|
* Valid fallback modes
|
|
28
38
|
*/
|
|
@@ -6,6 +6,13 @@ import type { PluginConfig } from '../types/index.js';
|
|
|
6
6
|
* Default plugin configuration
|
|
7
7
|
*/
|
|
8
8
|
export declare const DEFAULT_CONFIG: PluginConfig;
|
|
9
|
+
/**
|
|
10
|
+
* Result of config loading, includes which file was loaded
|
|
11
|
+
*/
|
|
12
|
+
export interface ConfigLoadResult {
|
|
13
|
+
config: PluginConfig;
|
|
14
|
+
source: string | null;
|
|
15
|
+
}
|
|
9
16
|
/**
|
|
10
17
|
* Validate configuration values
|
|
11
18
|
*/
|
|
@@ -13,4 +20,4 @@ export declare function validateConfig(config: Partial<PluginConfig>): PluginCon
|
|
|
13
20
|
/**
|
|
14
21
|
* Load and validate config from file paths
|
|
15
22
|
*/
|
|
16
|
-
export declare function loadConfig(directory: string):
|
|
23
|
+
export declare function loadConfig(directory: string, worktree?: string): ConfigLoadResult;
|
package/dist/src/utils/config.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { existsSync, readFileSync } from "fs";
|
|
5
5
|
import { join } from "path";
|
|
6
|
-
import { DEFAULT_FALLBACK_MODELS, VALID_FALLBACK_MODES, VALID_RESET_INTERVALS, DEFAULT_RETRY_POLICY, VALID_RETRY_STRATEGIES, } from '../types/index.js';
|
|
6
|
+
import { DEFAULT_FALLBACK_MODELS, VALID_FALLBACK_MODES, VALID_RESET_INTERVALS, DEFAULT_RETRY_POLICY, VALID_RETRY_STRATEGIES, DEFAULT_CIRCUIT_BREAKER_CONFIG, } from '../types/index.js';
|
|
7
7
|
/**
|
|
8
8
|
* Default plugin configuration
|
|
9
9
|
*/
|
|
@@ -13,6 +13,7 @@ export const DEFAULT_CONFIG = {
|
|
|
13
13
|
enabled: true,
|
|
14
14
|
fallbackMode: "cycle",
|
|
15
15
|
retryPolicy: DEFAULT_RETRY_POLICY,
|
|
16
|
+
circuitBreaker: DEFAULT_CIRCUIT_BREAKER_CONFIG,
|
|
16
17
|
log: {
|
|
17
18
|
level: "warn",
|
|
18
19
|
format: "simple",
|
|
@@ -37,13 +38,17 @@ export function validateConfig(config) {
|
|
|
37
38
|
return {
|
|
38
39
|
...DEFAULT_CONFIG,
|
|
39
40
|
...config,
|
|
40
|
-
fallbackModels: config.fallbackModels
|
|
41
|
+
fallbackModels: Array.isArray(config.fallbackModels) ? config.fallbackModels : DEFAULT_CONFIG.fallbackModels,
|
|
41
42
|
fallbackMode: mode && VALID_FALLBACK_MODES.includes(mode) ? mode : DEFAULT_CONFIG.fallbackMode,
|
|
42
43
|
retryPolicy: config.retryPolicy ? {
|
|
43
44
|
...DEFAULT_CONFIG.retryPolicy,
|
|
44
45
|
...config.retryPolicy,
|
|
45
46
|
strategy: strategy && VALID_RETRY_STRATEGIES.includes(strategy) ? strategy : DEFAULT_CONFIG.retryPolicy.strategy,
|
|
46
47
|
} : DEFAULT_CONFIG.retryPolicy,
|
|
48
|
+
circuitBreaker: config.circuitBreaker ? {
|
|
49
|
+
...DEFAULT_CONFIG.circuitBreaker,
|
|
50
|
+
...config.circuitBreaker,
|
|
51
|
+
} : DEFAULT_CONFIG.circuitBreaker,
|
|
47
52
|
log: config.log ? { ...DEFAULT_CONFIG.log, ...config.log } : DEFAULT_CONFIG.log,
|
|
48
53
|
metrics: config.metrics ? {
|
|
49
54
|
...DEFAULT_CONFIG.metrics,
|
|
@@ -59,20 +64,30 @@ export function validateConfig(config) {
|
|
|
59
64
|
/**
|
|
60
65
|
* Load and validate config from file paths
|
|
61
66
|
*/
|
|
62
|
-
export function loadConfig(directory) {
|
|
67
|
+
export function loadConfig(directory, worktree) {
|
|
63
68
|
const homedir = process.env.HOME || "";
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
69
|
+
const xdgConfigHome = process.env.XDG_CONFIG_HOME || join(homedir, ".config");
|
|
70
|
+
// Build search paths: worktree first, then directory, then home locations
|
|
71
|
+
const searchDirs = [];
|
|
72
|
+
if (worktree) {
|
|
73
|
+
searchDirs.push(worktree);
|
|
74
|
+
}
|
|
75
|
+
if (!worktree || worktree !== directory) {
|
|
76
|
+
searchDirs.push(directory);
|
|
77
|
+
}
|
|
78
|
+
const configPaths = [];
|
|
79
|
+
for (const dir of searchDirs) {
|
|
80
|
+
configPaths.push(join(dir, ".opencode", "rate-limit-fallback.json"));
|
|
81
|
+
configPaths.push(join(dir, "rate-limit-fallback.json"));
|
|
82
|
+
}
|
|
83
|
+
configPaths.push(join(homedir, ".opencode", "rate-limit-fallback.json"));
|
|
84
|
+
configPaths.push(join(xdgConfigHome, "opencode", "rate-limit-fallback.json"));
|
|
70
85
|
for (const configPath of configPaths) {
|
|
71
86
|
if (existsSync(configPath)) {
|
|
72
87
|
try {
|
|
73
88
|
const content = readFileSync(configPath, "utf-8");
|
|
74
89
|
const userConfig = JSON.parse(content);
|
|
75
|
-
return validateConfig(userConfig);
|
|
90
|
+
return { config: validateConfig(userConfig), source: configPath };
|
|
76
91
|
}
|
|
77
92
|
catch (error) {
|
|
78
93
|
// Log config errors to console immediately before logger is initialized
|
|
@@ -81,5 +96,5 @@ export function loadConfig(directory) {
|
|
|
81
96
|
}
|
|
82
97
|
}
|
|
83
98
|
}
|
|
84
|
-
return DEFAULT_CONFIG;
|
|
99
|
+
return { config: DEFAULT_CONFIG, source: null };
|
|
85
100
|
}
|
|
@@ -24,8 +24,8 @@ export function isRateLimitError(error) {
|
|
|
24
24
|
"too many requests",
|
|
25
25
|
"quota exceeded",
|
|
26
26
|
];
|
|
27
|
-
// Check for 429 in text (explicit HTTP status code)
|
|
28
|
-
if (responseBody
|
|
27
|
+
// Check for 429 in text (explicit HTTP status code, word-boundary to avoid false positives like "4291")
|
|
28
|
+
if (/\b429\b/.test(responseBody) || /\b429\b/.test(message)) {
|
|
29
29
|
return true;
|
|
30
30
|
}
|
|
31
31
|
// Check for strict rate limit keywords
|
package/package.json
CHANGED