neuro-cli 5.0.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/openrouter.d.ts +7 -1
- package/dist/api/openrouter.js +152 -37
- package/dist/commands/commands.js +11 -0
- package/dist/core/engine.d.ts +2 -0
- package/dist/core/engine.js +21 -1
- package/dist/core/fallback.d.ts +7 -1
- package/dist/core/fallback.js +68 -3
- package/dist/core/model-router.d.ts +6 -0
- package/dist/core/model-router.js +40 -1
- package/dist/core/rate-limiter.d.ts +162 -0
- package/dist/core/rate-limiter.js +589 -0
- package/package.json +1 -1
package/dist/api/openrouter.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Message, ToolCall, ToolDefinition, ToolResult } from '../core/types.js';
|
|
2
|
+
import { RateLimitManager } from '../core/rate-limiter.js';
|
|
2
3
|
export interface StreamCallbacks {
|
|
3
4
|
onToken?: (token: string) => void;
|
|
4
5
|
onToolCall?: (toolCall: ToolCall) => void;
|
|
@@ -24,11 +25,16 @@ export declare class OpenRouterClient {
|
|
|
24
25
|
private apiKey;
|
|
25
26
|
private baseUrl;
|
|
26
27
|
private totalUsage;
|
|
28
|
+
private rateLimiter;
|
|
27
29
|
constructor(apiKey: string, baseUrl?: string);
|
|
28
30
|
get usage(): TokenUsage;
|
|
29
31
|
resetUsage(): void;
|
|
30
32
|
/**
|
|
31
|
-
*
|
|
33
|
+
* Set the rate limiter instance for this client
|
|
34
|
+
*/
|
|
35
|
+
setRateLimiter(rateLimiter: RateLimitManager): void;
|
|
36
|
+
/**
|
|
37
|
+
* Main chat completion with streaming + rate limit awareness
|
|
32
38
|
*/
|
|
33
39
|
chat(request: ChatRequest, callbacks?: StreamCallbacks): Promise<{
|
|
34
40
|
content: string;
|
package/dist/api/openrouter.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// ============================================================
|
|
2
2
|
// NeuroCLI - OpenRouter API Client
|
|
3
3
|
// Streaming + Tool Use + Multi-model support
|
|
4
|
+
// Now with RateLimitManager integration
|
|
4
5
|
// ============================================================
|
|
5
6
|
// @ts-ignore
|
|
6
7
|
import { createParser } from 'eventsource-parser';
|
|
@@ -9,6 +10,7 @@ export class OpenRouterClient {
|
|
|
9
10
|
apiKey;
|
|
10
11
|
baseUrl;
|
|
11
12
|
totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
13
|
+
rateLimiter = null;
|
|
12
14
|
constructor(apiKey, baseUrl = 'https://openrouter.ai/api/v1') {
|
|
13
15
|
this.apiKey = apiKey;
|
|
14
16
|
this.baseUrl = baseUrl;
|
|
@@ -20,15 +22,42 @@ export class OpenRouterClient {
|
|
|
20
22
|
this.totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
21
23
|
}
|
|
22
24
|
/**
|
|
23
|
-
*
|
|
25
|
+
* Set the rate limiter instance for this client
|
|
26
|
+
*/
|
|
27
|
+
setRateLimiter(rateLimiter) {
|
|
28
|
+
this.rateLimiter = rateLimiter;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Main chat completion with streaming + rate limit awareness
|
|
24
32
|
*/
|
|
25
33
|
async chat(request, callbacks) {
|
|
26
34
|
const model = MODELS[request.model];
|
|
27
35
|
if (!model) {
|
|
28
36
|
throw new Error(`Unknown model: ${request.model}`);
|
|
29
37
|
}
|
|
38
|
+
// --- Rate limit pre-check ---
|
|
39
|
+
let activeModel = request.model;
|
|
40
|
+
if (this.rateLimiter) {
|
|
41
|
+
const check = await this.rateLimiter.acquire(request.model);
|
|
42
|
+
if (check.suggestedModel) {
|
|
43
|
+
// Switch to the suggested alternative model
|
|
44
|
+
activeModel = check.suggestedModel;
|
|
45
|
+
callbacks?.onThinking?.(`🔄 Rate limiter rotated to ${MODELS[activeModel]?.name || activeModel}`);
|
|
46
|
+
}
|
|
47
|
+
else if (!check.allowed) {
|
|
48
|
+
// Not allowed and no alternative - wait
|
|
49
|
+
if (check.waitMs > 0) {
|
|
50
|
+
callbacks?.onThinking?.(`⏳ Rate limited, waiting ${Math.ceil(check.waitMs / 1000)}s...`);
|
|
51
|
+
await new Promise(resolve => setTimeout(resolve, check.waitMs));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const activeModelConfig = MODELS[activeModel];
|
|
56
|
+
if (!activeModelConfig) {
|
|
57
|
+
throw new Error(`Unknown model after rate limit rotation: ${activeModel}`);
|
|
58
|
+
}
|
|
30
59
|
const body = {
|
|
31
|
-
model:
|
|
60
|
+
model: activeModel,
|
|
32
61
|
messages: request.messages.map(m => ({
|
|
33
62
|
role: m.role,
|
|
34
63
|
content: m.content,
|
|
@@ -38,9 +67,9 @@ export class OpenRouterClient {
|
|
|
38
67
|
})),
|
|
39
68
|
stream: request.stream ?? true,
|
|
40
69
|
temperature: request.temperature ?? 0.7,
|
|
41
|
-
max_tokens: request.maxTokens ??
|
|
70
|
+
max_tokens: request.maxTokens ?? activeModelConfig.maxOutput,
|
|
42
71
|
};
|
|
43
|
-
if (request.tools && request.tools.length > 0 &&
|
|
72
|
+
if (request.tools && request.tools.length > 0 && activeModelConfig.supportsTools) {
|
|
44
73
|
body.tools = request.tools.map(t => ({
|
|
45
74
|
type: 'function',
|
|
46
75
|
function: {
|
|
@@ -54,47 +83,133 @@ export class OpenRouterClient {
|
|
|
54
83
|
let lastError = null;
|
|
55
84
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
56
85
|
if (attempt > 0) {
|
|
57
|
-
//
|
|
58
|
-
|
|
59
|
-
|
|
86
|
+
// Smart backoff: use rate limiter cooldown if available, otherwise exponential
|
|
87
|
+
let waitTime;
|
|
88
|
+
if (this.rateLimiter) {
|
|
89
|
+
const state = this.rateLimiter.getStatus().find(s => s.model === activeModel);
|
|
90
|
+
waitTime = state && state.cooldown > 0 ? state.cooldown * 1000 : Math.min(5000 * Math.pow(2, attempt - 1), 30000);
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
waitTime = Math.min(5000 * Math.pow(3, attempt - 1), 30000);
|
|
94
|
+
}
|
|
95
|
+
callbacks?.onThinking?.(`⏳ Retrying in ${Math.ceil(waitTime / 1000)}s (attempt ${attempt + 1}/${maxRetries + 1})...`);
|
|
60
96
|
await new Promise(resolve => setTimeout(resolve, waitTime));
|
|
61
97
|
}
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
'
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if (response.
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
98
|
+
const startTime = Date.now();
|
|
99
|
+
try {
|
|
100
|
+
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
101
|
+
method: 'POST',
|
|
102
|
+
headers: {
|
|
103
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
104
|
+
'Content-Type': 'application/json',
|
|
105
|
+
'HTTP-Referer': 'https://neurocli.dev',
|
|
106
|
+
'X-Title': 'NeuroCLI',
|
|
107
|
+
},
|
|
108
|
+
body: JSON.stringify(body),
|
|
109
|
+
});
|
|
110
|
+
const responseTimeMs = Date.now() - startTime;
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
const errorBody = await response.text();
|
|
113
|
+
// Parse retry_after from 429 errors
|
|
114
|
+
if (response.status === 429) {
|
|
115
|
+
let retryAfterSeconds;
|
|
116
|
+
try {
|
|
117
|
+
const errorData = JSON.parse(errorBody);
|
|
118
|
+
retryAfterSeconds = errorData?.error?.metadata?.retry_after_seconds;
|
|
119
|
+
}
|
|
120
|
+
catch { }
|
|
121
|
+
// Record rate limit in manager
|
|
122
|
+
if (this.rateLimiter) {
|
|
123
|
+
this.rateLimiter.recordRateLimit(activeModel, retryAfterSeconds);
|
|
124
|
+
}
|
|
125
|
+
// Try model rotation before retrying
|
|
126
|
+
if (this.rateLimiter && attempt < maxRetries) {
|
|
127
|
+
const nextModel = this.rateLimiter.getNextRotationModel(activeModel);
|
|
128
|
+
if (nextModel && nextModel !== activeModel) {
|
|
129
|
+
const nextModelConfig = MODELS[nextModel];
|
|
130
|
+
if (nextModelConfig) {
|
|
131
|
+
callbacks?.onThinking?.(`🔄 Rotating ${MODELS[activeModel]?.name || activeModel} → ${nextModelConfig.name} due to rate limit`);
|
|
132
|
+
activeModel = nextModel;
|
|
133
|
+
body.model = nextModel;
|
|
134
|
+
body.max_tokens = request.maxTokens ?? nextModelConfig.maxOutput;
|
|
135
|
+
if (request.tools && request.tools.length > 0 && nextModelConfig.supportsTools) {
|
|
136
|
+
// tools already set
|
|
137
|
+
}
|
|
138
|
+
else if (request.tools && !nextModelConfig.supportsTools) {
|
|
139
|
+
delete body.tools;
|
|
140
|
+
}
|
|
141
|
+
// Don't count this as a retry attempt for the new model
|
|
142
|
+
attempt--;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (retryAfterSeconds && attempt < maxRetries) {
|
|
148
|
+
lastError = new Error(`Rate limited, retry after ${retryAfterSeconds}s`);
|
|
81
149
|
continue; // Retry
|
|
82
150
|
}
|
|
151
|
+
if (attempt < maxRetries) {
|
|
152
|
+
lastError = new Error(`Rate limited (429): ${errorBody}`);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
83
155
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
156
|
+
// Server errors (5xx) - record and potentially rotate
|
|
157
|
+
if (response.status >= 500) {
|
|
158
|
+
if (this.rateLimiter) {
|
|
159
|
+
this.rateLimiter.recordServerError(activeModel);
|
|
160
|
+
}
|
|
161
|
+
if (attempt < maxRetries) {
|
|
162
|
+
lastError = new Error(`Server error (${response.status}): ${errorBody}`);
|
|
163
|
+
// Try model rotation for server errors too
|
|
164
|
+
if (this.rateLimiter) {
|
|
165
|
+
const nextModel = this.rateLimiter.getNextRotationModel(activeModel);
|
|
166
|
+
if (nextModel && nextModel !== activeModel) {
|
|
167
|
+
const nextModelConfig = MODELS[nextModel];
|
|
168
|
+
if (nextModelConfig) {
|
|
169
|
+
callbacks?.onThinking?.(`🔄 Rotating away from ${MODELS[activeModel]?.name || activeModel} due to server error`);
|
|
170
|
+
activeModel = nextModel;
|
|
171
|
+
body.model = nextModel;
|
|
172
|
+
body.max_tokens = request.maxTokens ?? nextModelConfig.maxOutput;
|
|
173
|
+
if (request.tools && request.tools.length > 0 && nextModelConfig.supportsTools) {
|
|
174
|
+
// tools already set
|
|
175
|
+
}
|
|
176
|
+
else if (request.tools && !nextModelConfig.supportsTools) {
|
|
177
|
+
delete body.tools;
|
|
178
|
+
}
|
|
179
|
+
attempt--;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
89
186
|
}
|
|
187
|
+
throw new Error(`OpenRouter API error (${response.status}): ${errorBody}`);
|
|
188
|
+
}
|
|
189
|
+
// Success!
|
|
190
|
+
if (this.rateLimiter) {
|
|
191
|
+
this.rateLimiter.recordSuccess(activeModel, responseTimeMs);
|
|
192
|
+
}
|
|
193
|
+
if (request.stream && callbacks) {
|
|
194
|
+
const result = await this.handleStreamingResponse(response, activeModel, callbacks);
|
|
195
|
+
return result;
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
const result = await this.handleNonStreamingResponse(response, activeModel);
|
|
199
|
+
return result;
|
|
90
200
|
}
|
|
91
|
-
throw new Error(`OpenRouter API error (${response.status}): ${errorBody}`);
|
|
92
|
-
}
|
|
93
|
-
if (request.stream && callbacks) {
|
|
94
|
-
return this.handleStreamingResponse(response, request.model, callbacks);
|
|
95
201
|
}
|
|
96
|
-
|
|
97
|
-
|
|
202
|
+
catch (error) {
|
|
203
|
+
// Network-level errors (fetch failed)
|
|
204
|
+
if (error instanceof TypeError && error.message.includes('fetch')) {
|
|
205
|
+
if (this.rateLimiter) {
|
|
206
|
+
this.rateLimiter.recordServerError(activeModel);
|
|
207
|
+
}
|
|
208
|
+
lastError = error;
|
|
209
|
+
if (attempt < maxRetries)
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
throw error;
|
|
98
213
|
}
|
|
99
214
|
}
|
|
100
215
|
throw lastError || new Error('Max retries exceeded');
|
|
@@ -297,6 +297,17 @@ export class CommandSystem {
|
|
|
297
297
|
description: 'Browser automation (navigate/screenshot/click)',
|
|
298
298
|
prompt: 'Use the browser to navigate, take screenshots, click elements, or extract content from web pages. {args}',
|
|
299
299
|
},
|
|
300
|
+
// --- Rate Limit Commands ---
|
|
301
|
+
{
|
|
302
|
+
name: 'ratelimit',
|
|
303
|
+
description: 'Show rate limiter status for all models',
|
|
304
|
+
prompt: 'Show the current rate limiter status. Display health scores, cooldowns, 429 counts, and queue status for all tracked models. {args}',
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
name: 'ratelimit-reset',
|
|
308
|
+
description: 'Reset rate limiter state',
|
|
309
|
+
prompt: 'Reset all rate limiter cooldowns and health scores. Use this if models are stuck in cooldown. {args}',
|
|
310
|
+
},
|
|
300
311
|
];
|
|
301
312
|
for (const cmd of bundled) {
|
|
302
313
|
if (!this.commands.has(cmd.name)) {
|
package/dist/core/engine.d.ts
CHANGED
|
@@ -57,6 +57,7 @@ import { TerminalUX } from './terminal-ux.js';
|
|
|
57
57
|
import { MultiSessionManager } from './multi-session.js';
|
|
58
58
|
import { GitWorktreeManager } from './git-worktree.js';
|
|
59
59
|
import { AutoUpdater } from './updater.js';
|
|
60
|
+
import { RateLimitManager } from './rate-limiter.js';
|
|
60
61
|
export declare class NeuroEngine {
|
|
61
62
|
config: NeuroConfig;
|
|
62
63
|
client: OpenRouterClient;
|
|
@@ -117,6 +118,7 @@ export declare class NeuroEngine {
|
|
|
117
118
|
multiSession: MultiSessionManager;
|
|
118
119
|
gitWorktree: GitWorktreeManager;
|
|
119
120
|
updater: AutoUpdater;
|
|
121
|
+
rateLimiter: RateLimitManager;
|
|
120
122
|
private autoApproveSet;
|
|
121
123
|
private requireApprovalSet;
|
|
122
124
|
constructor(config: NeuroConfig);
|
package/dist/core/engine.js
CHANGED
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
// Spec-Driven Development, LLM Evaluator Hooks, MCP Apps,
|
|
16
16
|
// Multi-Model Orchestrator, Smart Monitor, Outcome Grading,
|
|
17
17
|
// Observability (OTLP), Auto-Compact, Terminal UX,
|
|
18
|
-
// Multi-Session, Git Worktree, Auto-Updater
|
|
18
|
+
// Multi-Session, Git Worktree, Auto-Updater,
|
|
19
|
+
// Rate Limit Manager
|
|
19
20
|
// ============================================================
|
|
20
21
|
import { join } from 'path';
|
|
21
22
|
import { homedir } from 'os';
|
|
@@ -79,6 +80,7 @@ import { TerminalUX } from './terminal-ux.js';
|
|
|
79
80
|
import { MultiSessionManager } from './multi-session.js';
|
|
80
81
|
import { GitWorktreeManager } from './git-worktree.js';
|
|
81
82
|
import { AutoUpdater } from './updater.js';
|
|
83
|
+
import { RateLimitManager } from './rate-limiter.js';
|
|
82
84
|
export class NeuroEngine {
|
|
83
85
|
config;
|
|
84
86
|
client;
|
|
@@ -143,11 +145,27 @@ export class NeuroEngine {
|
|
|
143
145
|
multiSession;
|
|
144
146
|
gitWorktree;
|
|
145
147
|
updater;
|
|
148
|
+
rateLimiter;
|
|
146
149
|
autoApproveSet;
|
|
147
150
|
requireApprovalSet;
|
|
148
151
|
constructor(config) {
|
|
149
152
|
this.config = config;
|
|
153
|
+
// Rate Limit Manager - initialized FIRST so it's available for all systems
|
|
154
|
+
this.rateLimiter = new RateLimitManager({
|
|
155
|
+
globalRpm: 20,
|
|
156
|
+
perModelRpm: 10,
|
|
157
|
+
baseCooldownMs: 10000,
|
|
158
|
+
maxCooldownMs: 120000,
|
|
159
|
+
backoffMultiplier: 2.0,
|
|
160
|
+
rotationPoolSize: 5,
|
|
161
|
+
enableQueue: true,
|
|
162
|
+
maxQueueSize: 50,
|
|
163
|
+
maxQueueWaitMs: 60000,
|
|
164
|
+
adaptiveLearning: true,
|
|
165
|
+
verbose: true,
|
|
166
|
+
});
|
|
150
167
|
this.client = new OpenRouterClient(config.apiKey, config.baseUrl);
|
|
168
|
+
this.client.setRateLimiter(this.rateLimiter);
|
|
151
169
|
this.registry = registerAllTools(new ToolRegistry());
|
|
152
170
|
this.contextManager = new ContextManager(config.defaultModel, config.context.maxTokens);
|
|
153
171
|
this.sessionManager = new SessionManager();
|
|
@@ -166,6 +184,7 @@ export class NeuroEngine {
|
|
|
166
184
|
return false;
|
|
167
185
|
});
|
|
168
186
|
this.fallback = new FallbackChain(this.client, config.fallbackChain);
|
|
187
|
+
this.fallback.setRateLimiter(this.rateLimiter);
|
|
169
188
|
// Sandbox system
|
|
170
189
|
this.sandbox = new Sandbox(config.sandbox);
|
|
171
190
|
// Plugin system
|
|
@@ -196,6 +215,7 @@ export class NeuroEngine {
|
|
|
196
215
|
categoryOverrides: {},
|
|
197
216
|
maxTokenBudget: config.context.maxTokens,
|
|
198
217
|
}, Object.fromEntries(Object.entries(MODELS).map(([id, m]) => [id, { name: m.name, contextWindow: m.contextWindow, maxOutput: m.maxOutput }])));
|
|
218
|
+
this.modelRouter.setRateLimiter(this.rateLimiter);
|
|
199
219
|
// Output styles
|
|
200
220
|
this.styleManager = new StyleManager(process.cwd());
|
|
201
221
|
// Extended thinking
|
package/dist/core/fallback.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { OpenRouterClient, TokenUsage } from '../api/openrouter.js';
|
|
2
2
|
import { Message } from './types.js';
|
|
3
|
+
import { RateLimitManager } from './rate-limiter.js';
|
|
3
4
|
export interface FallbackConfig {
|
|
4
5
|
models: string[];
|
|
5
6
|
maxRetries: number;
|
|
@@ -27,9 +28,14 @@ export interface FallbackResult {
|
|
|
27
28
|
export declare class FallbackChain {
|
|
28
29
|
private config;
|
|
29
30
|
private client;
|
|
31
|
+
private rateLimiter;
|
|
30
32
|
constructor(client: OpenRouterClient, config?: Partial<FallbackConfig>);
|
|
31
33
|
/**
|
|
32
|
-
*
|
|
34
|
+
* Set the rate limiter for intelligent model rotation
|
|
35
|
+
*/
|
|
36
|
+
setRateLimiter(rateLimiter: RateLimitManager): void;
|
|
37
|
+
/**
|
|
38
|
+
* Execute a chat request with fallback chain + rate limit awareness
|
|
33
39
|
*/
|
|
34
40
|
chatWithFallback(primaryModel: string, messages: Message[], options: {
|
|
35
41
|
tools?: unknown[];
|
package/dist/core/fallback.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// ============================================================
|
|
2
2
|
// NeuroCLI - Fallback Model Chain
|
|
3
3
|
// Automatic model fallback on failure
|
|
4
|
+
// Now with RateLimitManager integration
|
|
4
5
|
// ============================================================
|
|
5
6
|
import { MODELS } from '../api/models.js';
|
|
6
7
|
import chalk from 'chalk';
|
|
7
8
|
export class FallbackChain {
|
|
8
9
|
config;
|
|
9
10
|
client;
|
|
11
|
+
rateLimiter = null;
|
|
10
12
|
constructor(client, config) {
|
|
11
13
|
this.client = client;
|
|
12
14
|
this.config = {
|
|
@@ -18,13 +20,67 @@ export class FallbackChain {
|
|
|
18
20
|
};
|
|
19
21
|
}
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
23
|
+
* Set the rate limiter for intelligent model rotation
|
|
24
|
+
*/
|
|
25
|
+
setRateLimiter(rateLimiter) {
|
|
26
|
+
this.rateLimiter = rateLimiter;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Execute a chat request with fallback chain + rate limit awareness
|
|
22
30
|
*/
|
|
23
31
|
async chatWithFallback(primaryModel, messages, options, onModelSwitch) {
|
|
24
|
-
|
|
32
|
+
// Build the model chain: primary + configured fallbacks
|
|
33
|
+
let chain = [primaryModel, ...this.config.models.filter(m => m !== primaryModel)];
|
|
34
|
+
// If rate limiter is available, prioritize models with good health
|
|
35
|
+
if (this.rateLimiter) {
|
|
36
|
+
const availableModels = this.rateLimiter.getNextRotationModel(primaryModel);
|
|
37
|
+
// Insert rate-limiter-suggested models at the front of the chain
|
|
38
|
+
const healthyAlternatives = [];
|
|
39
|
+
const allStates = this.rateLimiter.getStatus();
|
|
40
|
+
for (const state of allStates) {
|
|
41
|
+
if (state.model !== primaryModel && state.health >= 50 && state.cooldown === 0) {
|
|
42
|
+
healthyAlternatives.push(state.model);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Remove duplicates and merge chains
|
|
46
|
+
const seen = new Set([primaryModel]);
|
|
47
|
+
const mergedChain = [primaryModel];
|
|
48
|
+
// Add healthy alternatives first (from rate limiter)
|
|
49
|
+
for (const m of healthyAlternatives) {
|
|
50
|
+
if (!seen.has(m)) {
|
|
51
|
+
seen.add(m);
|
|
52
|
+
mergedChain.push(m);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
// Then add original fallback chain
|
|
56
|
+
for (const m of chain) {
|
|
57
|
+
if (!seen.has(m)) {
|
|
58
|
+
seen.add(m);
|
|
59
|
+
mergedChain.push(m);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
chain = mergedChain;
|
|
63
|
+
}
|
|
25
64
|
const attempts = [];
|
|
26
65
|
let totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
27
66
|
for (const model of chain) {
|
|
67
|
+
// Skip models that are in cooldown (if rate limiter is available)
|
|
68
|
+
if (this.rateLimiter) {
|
|
69
|
+
const check = this.rateLimiter.checkRateLimit(model);
|
|
70
|
+
if (!check.allowed && check.waitMs > 30000) {
|
|
71
|
+
// If wait is more than 30s, skip this model entirely
|
|
72
|
+
console.log(chalk.gray(` ⏭️ Skipping ${MODELS[model]?.name || model} (cooldown: ${Math.ceil(check.waitMs / 1000)}s)`));
|
|
73
|
+
attempts.push({ model, success: false, error: `In cooldown (${Math.ceil(check.waitMs / 1000)}s)` });
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (!check.allowed && check.suggestedModel) {
|
|
77
|
+
// Use the suggested alternative instead
|
|
78
|
+
if (!chain.includes(check.suggestedModel)) {
|
|
79
|
+
chain.push(check.suggestedModel);
|
|
80
|
+
}
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
28
84
|
for (let retry = 0; retry <= this.config.maxRetries; retry++) {
|
|
29
85
|
try {
|
|
30
86
|
const modelInfo = MODELS[model];
|
|
@@ -62,7 +118,16 @@ export class FallbackChain {
|
|
|
62
118
|
errMsg.includes('503') ||
|
|
63
119
|
errMsg.includes('500'));
|
|
64
120
|
if (shouldFallback) {
|
|
65
|
-
|
|
121
|
+
// Record the rate limit or server error
|
|
122
|
+
if (this.rateLimiter) {
|
|
123
|
+
if (errMsg.includes('429') || errMsg.toLowerCase().includes('rate_limit')) {
|
|
124
|
+
this.rateLimiter.recordRateLimit(model);
|
|
125
|
+
}
|
|
126
|
+
else if (errMsg.includes('500') || errMsg.includes('503')) {
|
|
127
|
+
this.rateLimiter.recordServerError(model);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
console.log(chalk.yellow(` ⚠ ${MODELS[model]?.name || model} failed: ${errMsg.slice(0, 80)}`));
|
|
66
131
|
break; // Try next model in chain
|
|
67
132
|
}
|
|
68
133
|
// For other errors, retry on same model
|
|
@@ -29,9 +29,15 @@ export declare class ModelRouter {
|
|
|
29
29
|
private availableModels;
|
|
30
30
|
private currentEffort;
|
|
31
31
|
private forcedModel;
|
|
32
|
+
private rateLimiter;
|
|
32
33
|
constructor(config: RouterConfig, availableModels: Record<string, ModelInfo>);
|
|
34
|
+
/**
|
|
35
|
+
* Set the rate limiter for rate-aware routing decisions
|
|
36
|
+
*/
|
|
37
|
+
setRateLimiter(rateLimiter: import('./rate-limiter.js').RateLimitManager): void;
|
|
33
38
|
/**
|
|
34
39
|
* Route a prompt to the best available model.
|
|
40
|
+
* Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
|
|
35
41
|
*/
|
|
36
42
|
route(prompt: string, effort?: EffortLevel): RoutingDecision;
|
|
37
43
|
/**
|
|
@@ -124,6 +124,7 @@ export class ModelRouter {
|
|
|
124
124
|
availableModels;
|
|
125
125
|
currentEffort = 'medium';
|
|
126
126
|
forcedModel = null;
|
|
127
|
+
rateLimiter = null;
|
|
127
128
|
// -----------------------------------------------------------------------
|
|
128
129
|
// Construction
|
|
129
130
|
// -----------------------------------------------------------------------
|
|
@@ -135,15 +136,53 @@ export class ModelRouter {
|
|
|
135
136
|
// -----------------------------------------------------------------------
|
|
136
137
|
// Public API
|
|
137
138
|
// -----------------------------------------------------------------------
|
|
139
|
+
/**
|
|
140
|
+
* Set the rate limiter for rate-aware routing decisions
|
|
141
|
+
*/
|
|
142
|
+
setRateLimiter(rateLimiter) {
|
|
143
|
+
this.rateLimiter = rateLimiter;
|
|
144
|
+
}
|
|
138
145
|
/**
|
|
139
146
|
* Route a prompt to the best available model.
|
|
147
|
+
* Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
|
|
140
148
|
*/
|
|
141
149
|
route(prompt, effort) {
|
|
142
150
|
const effectiveEffort = effort ?? this.currentEffort;
|
|
143
151
|
const analysis = this.analyzePrompt(prompt);
|
|
144
152
|
const estimatedTokens = this.estimateTokens(prompt);
|
|
145
|
-
|
|
153
|
+
let model = this.selectModel(analysis.complexity, analysis.category, effectiveEffort);
|
|
146
154
|
const alternatives = this.getAlternatives(analysis.complexity, analysis.category);
|
|
155
|
+
// --- Rate-limit-aware routing ---
|
|
156
|
+
if (this.rateLimiter) {
|
|
157
|
+
const check = this.rateLimiter.checkRateLimit(model);
|
|
158
|
+
if (!check.allowed && check.suggestedModel) {
|
|
159
|
+
// Use the rate limiter's suggested alternative
|
|
160
|
+
const suggested = check.suggestedModel;
|
|
161
|
+
if (this.availableModels[suggested]) {
|
|
162
|
+
model = suggested;
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
// The suggested model isn't in our available models, try alternatives
|
|
166
|
+
for (const alt of alternatives) {
|
|
167
|
+
const altCheck = this.rateLimiter.checkRateLimit(alt);
|
|
168
|
+
if (altCheck.allowed) {
|
|
169
|
+
model = alt;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
else if (!check.allowed) {
|
|
176
|
+
// Model is rate limited and no suggested alternative, try alternatives
|
|
177
|
+
for (const alt of alternatives) {
|
|
178
|
+
const altCheck = this.rateLimiter.checkRateLimit(alt);
|
|
179
|
+
if (altCheck.allowed) {
|
|
180
|
+
model = alt;
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
147
186
|
const reasoning = this.buildReasoning(analysis, effectiveEffort, model);
|
|
148
187
|
return {
|
|
149
188
|
model,
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
export interface RateLimiterConfig {
|
|
2
|
+
/** Maximum requests per minute across all models (default: 20) */
|
|
3
|
+
globalRpm: number;
|
|
4
|
+
/** Maximum requests per minute per model (default: 10) */
|
|
5
|
+
perModelRpm: number;
|
|
6
|
+
/** Base cooldown after a 429 in ms (default: 10000) */
|
|
7
|
+
baseCooldownMs: number;
|
|
8
|
+
/** Maximum cooldown in ms (default: 120000) */
|
|
9
|
+
maxCooldownMs: number;
|
|
10
|
+
/** Multiplier for consecutive 429s (default: 2.0) */
|
|
11
|
+
backoffMultiplier: number;
|
|
12
|
+
/** Number of free model rotation slots (default: 5) */
|
|
13
|
+
rotationPoolSize: number;
|
|
14
|
+
/** Enable request queuing during rate limits (default: true) */
|
|
15
|
+
enableQueue: boolean;
|
|
16
|
+
/** Maximum queue size (default: 50) */
|
|
17
|
+
maxQueueSize: number;
|
|
18
|
+
/** Maximum time a request can wait in queue in ms (default: 60000) */
|
|
19
|
+
maxQueueWaitMs: number;
|
|
20
|
+
/** Enable adaptive learning from rate limit patterns (default: true) */
|
|
21
|
+
adaptiveLearning: boolean;
|
|
22
|
+
/** Log rate limit events to console (default: true) */
|
|
23
|
+
verbose: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface ModelRateState {
|
|
26
|
+
/** Model ID */
|
|
27
|
+
modelId: string;
|
|
28
|
+
/** Current cooldown end time (0 = not cooling down) */
|
|
29
|
+
cooldownUntil: number;
|
|
30
|
+
/** Number of consecutive 429s */
|
|
31
|
+
consecutive429s: number;
|
|
32
|
+
/** Total 429s received */
|
|
33
|
+
total429s: number;
|
|
34
|
+
/** Total successful requests */
|
|
35
|
+
totalSuccesses: number;
|
|
36
|
+
/** Requests in current minute window */
|
|
37
|
+
requestsThisMinute: number;
|
|
38
|
+
/** Minute window start time */
|
|
39
|
+
minuteWindowStart: number;
|
|
40
|
+
/** Last request timestamp */
|
|
41
|
+
lastRequestAt: number;
|
|
42
|
+
/** Average response time in ms */
|
|
43
|
+
avgResponseMs: number;
|
|
44
|
+
/** Whether model is currently in rotation pool */
|
|
45
|
+
inRotationPool: boolean;
|
|
46
|
+
/** Health score 0-100 (100 = healthy) */
|
|
47
|
+
healthScore: number;
|
|
48
|
+
}
|
|
49
|
+
export interface RateLimitCheck {
|
|
50
|
+
/** Whether the request can proceed */
|
|
51
|
+
allowed: boolean;
|
|
52
|
+
/** Wait time in ms if not allowed (0 if allowed) */
|
|
53
|
+
waitMs: number;
|
|
54
|
+
/** Suggested alternative model (null if current is fine) */
|
|
55
|
+
suggestedModel: string | null;
|
|
56
|
+
/** Reason for the decision */
|
|
57
|
+
reason: string;
|
|
58
|
+
}
|
|
59
|
+
export interface QueuedRequest {
|
|
60
|
+
id: string;
|
|
61
|
+
model: string;
|
|
62
|
+
enqueuedAt: number;
|
|
63
|
+
resolve: (check: RateLimitCheck) => void;
|
|
64
|
+
reject: (error: Error) => void;
|
|
65
|
+
}
|
|
66
|
+
export declare class RateLimitManager {
|
|
67
|
+
private config;
|
|
68
|
+
private modelStates;
|
|
69
|
+
private globalRequestTimestamps;
|
|
70
|
+
private requestQueue;
|
|
71
|
+
private queueProcessTimer;
|
|
72
|
+
private rotationPool;
|
|
73
|
+
private currentRotationIndex;
|
|
74
|
+
private requestIdCounter;
|
|
75
|
+
private rateLimitPatterns;
|
|
76
|
+
constructor(config?: Partial<RateLimiterConfig>);
|
|
77
|
+
/**
|
|
78
|
+
* Check if a request can be made to a specific model.
|
|
79
|
+
* Returns a RateLimitCheck with the decision and optional alternative model.
|
|
80
|
+
*/
|
|
81
|
+
checkRateLimit(modelId: string): RateLimitCheck;
|
|
82
|
+
/**
|
|
83
|
+
* Acquire permission to make a request. Resolves when the request can proceed.
|
|
84
|
+
* If queuing is enabled, waits in queue. Otherwise, rejects if rate limited.
|
|
85
|
+
*/
|
|
86
|
+
acquire(modelId: string): Promise<RateLimitCheck>;
|
|
87
|
+
/**
|
|
88
|
+
* Record a successful request completion
|
|
89
|
+
*/
|
|
90
|
+
recordSuccess(modelId: string, responseTimeMs: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* Record a rate limit (429) response
|
|
93
|
+
*/
|
|
94
|
+
recordRateLimit(modelId: string, retryAfterSeconds?: number): void;
|
|
95
|
+
/**
|
|
96
|
+
* Record a server error (5xx) - treat as potential overload
|
|
97
|
+
*/
|
|
98
|
+
recordServerError(modelId: string): void;
|
|
99
|
+
/**
|
|
100
|
+
* Get the next available model from the rotation pool.
|
|
101
|
+
* Skips models that are in cooldown or have low health.
|
|
102
|
+
*/
|
|
103
|
+
getNextRotationModel(excludeModel?: string): string | null;
|
|
104
|
+
/**
|
|
105
|
+
* Get the best available model for a given requested model.
|
|
106
|
+
* Tries the requested model first, then falls back through the rotation pool.
|
|
107
|
+
*/
|
|
108
|
+
getBestAvailableModel(requestedModel: string): string;
|
|
109
|
+
/**
|
|
110
|
+
* Get rate limit status for all tracked models
|
|
111
|
+
*/
|
|
112
|
+
getStatus(): Array<{
|
|
113
|
+
model: string;
|
|
114
|
+
name: string;
|
|
115
|
+
health: number;
|
|
116
|
+
cooldown: number;
|
|
117
|
+
consecutive429s: number;
|
|
118
|
+
total429s: number;
|
|
119
|
+
totalSuccesses: number;
|
|
120
|
+
rpm: number;
|
|
121
|
+
avgResponseMs: number;
|
|
122
|
+
}>;
|
|
123
|
+
/**
|
|
124
|
+
* Get queue status
|
|
125
|
+
*/
|
|
126
|
+
getQueueStatus(): {
|
|
127
|
+
size: number;
|
|
128
|
+
oldestWaitMs: number;
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Reset all rate limit state (useful for testing)
|
|
132
|
+
*/
|
|
133
|
+
reset(): void;
|
|
134
|
+
/**
|
|
135
|
+
* Reset a specific model's rate limit state
|
|
136
|
+
*/
|
|
137
|
+
resetModel(modelId: string): void;
|
|
138
|
+
/**
|
|
139
|
+
* Update configuration at runtime
|
|
140
|
+
*/
|
|
141
|
+
updateConfig(updates: Partial<RateLimiterConfig>): void;
|
|
142
|
+
/**
|
|
143
|
+
* Print a formatted status table
|
|
144
|
+
*/
|
|
145
|
+
printStatus(): void;
|
|
146
|
+
private getOrCreateState;
|
|
147
|
+
private recordRequestStart;
|
|
148
|
+
private cleanupGlobalTimestamps;
|
|
149
|
+
private cleanupModelTimestamps;
|
|
150
|
+
private findAvailableModel;
|
|
151
|
+
private initializeRotationPool;
|
|
152
|
+
private enqueueRequest;
|
|
153
|
+
private startQueueProcessor;
|
|
154
|
+
private processQueue;
|
|
155
|
+
private recordRateLimitPattern;
|
|
156
|
+
private getHealthBar;
|
|
157
|
+
/**
|
|
158
|
+
* Cleanup - call when shutting down
|
|
159
|
+
*/
|
|
160
|
+
dispose(): void;
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=rate-limiter.d.ts.map
|
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
// ============================================================
|
|
2
|
+
// NeuroCLI - Rate Limit Manager
|
|
3
|
+
// Intelligent rate limiting with token bucket, per-model
|
|
4
|
+
// cooldown tracking, model rotation, and adaptive backoff
|
|
5
|
+
// ============================================================
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import { MODELS, getFreeModelsWithTools } from '../api/models.js';
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
// RateLimitManager
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
export class RateLimitManager {
|
|
12
|
+
config;
|
|
13
|
+
modelStates = new Map();
|
|
14
|
+
globalRequestTimestamps = [];
|
|
15
|
+
requestQueue = [];
|
|
16
|
+
queueProcessTimer = null;
|
|
17
|
+
rotationPool = [];
|
|
18
|
+
currentRotationIndex = 0;
|
|
19
|
+
requestIdCounter = 0;
|
|
20
|
+
// Adaptive learning data
|
|
21
|
+
rateLimitPatterns = new Map();
|
|
22
|
+
constructor(config) {
|
|
23
|
+
this.config = {
|
|
24
|
+
globalRpm: 20,
|
|
25
|
+
perModelRpm: 10,
|
|
26
|
+
baseCooldownMs: 10000,
|
|
27
|
+
maxCooldownMs: 120000,
|
|
28
|
+
backoffMultiplier: 2.0,
|
|
29
|
+
rotationPoolSize: 5,
|
|
30
|
+
enableQueue: true,
|
|
31
|
+
maxQueueSize: 50,
|
|
32
|
+
maxQueueWaitMs: 60000,
|
|
33
|
+
adaptiveLearning: true,
|
|
34
|
+
verbose: true,
|
|
35
|
+
...config,
|
|
36
|
+
};
|
|
37
|
+
// Initialize rotation pool with free models that support tools
|
|
38
|
+
this.initializeRotationPool();
|
|
39
|
+
// Start queue processor
|
|
40
|
+
this.startQueueProcessor();
|
|
41
|
+
}
|
|
42
|
+
// =========================================================================
|
|
43
|
+
// Pre-Request Checking
|
|
44
|
+
// =========================================================================
|
|
45
|
+
/**
|
|
46
|
+
* Check if a request can be made to a specific model.
|
|
47
|
+
* Returns a RateLimitCheck with the decision and optional alternative model.
|
|
48
|
+
*/
|
|
49
|
+
checkRateLimit(modelId) {
|
|
50
|
+
const now = Date.now();
|
|
51
|
+
// 1. Check global RPM
|
|
52
|
+
this.cleanupGlobalTimestamps(now);
|
|
53
|
+
if (this.globalRequestTimestamps.length >= this.config.globalRpm) {
|
|
54
|
+
const oldestInWindow = this.globalRequestTimestamps[0];
|
|
55
|
+
const waitMs = Math.max(0, 60000 - (now - oldestInWindow));
|
|
56
|
+
const alternative = this.findAvailableModel(modelId);
|
|
57
|
+
if (alternative) {
|
|
58
|
+
return {
|
|
59
|
+
allowed: false,
|
|
60
|
+
waitMs: 0,
|
|
61
|
+
suggestedModel: alternative,
|
|
62
|
+
reason: `Global RPM limit reached (${this.config.globalRpm}/min). Suggesting alternative: ${MODELS[alternative]?.name || alternative}`,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
allowed: false,
|
|
67
|
+
waitMs,
|
|
68
|
+
suggestedModel: null,
|
|
69
|
+
reason: `Global RPM limit reached (${this.config.globalRpm}/min). Wait ${Math.ceil(waitMs / 1000)}s or try a different model.`,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
// 2. Check per-model cooldown
|
|
73
|
+
const state = this.getOrCreateState(modelId);
|
|
74
|
+
if (state.cooldownUntil > now) {
|
|
75
|
+
const waitMs = state.cooldownUntil - now;
|
|
76
|
+
const alternative = this.findAvailableModel(modelId);
|
|
77
|
+
if (alternative) {
|
|
78
|
+
return {
|
|
79
|
+
allowed: false,
|
|
80
|
+
waitMs: 0,
|
|
81
|
+
suggestedModel: alternative,
|
|
82
|
+
reason: `${MODELS[modelId]?.name || modelId} is in cooldown (${Math.ceil(waitMs / 1000)}s left). Rotating to ${MODELS[alternative]?.name || alternative}`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
allowed: false,
|
|
87
|
+
waitMs,
|
|
88
|
+
suggestedModel: null,
|
|
89
|
+
reason: `${MODELS[modelId]?.name || modelId} is in cooldown. Wait ${Math.ceil(waitMs / 1000)}s.`,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
// 3. Check per-model RPM
|
|
93
|
+
this.cleanupModelTimestamps(state, now);
|
|
94
|
+
if (state.requestsThisMinute >= this.config.perModelRpm) {
|
|
95
|
+
const alternative = this.findAvailableModel(modelId);
|
|
96
|
+
if (alternative) {
|
|
97
|
+
return {
|
|
98
|
+
allowed: false,
|
|
99
|
+
waitMs: 0,
|
|
100
|
+
suggestedModel: alternative,
|
|
101
|
+
reason: `${MODELS[modelId]?.name || modelId} RPM limit (${this.config.perModelRpm}/min). Rotating to ${MODELS[alternative]?.name || alternative}`,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
const minuteElapsed = now - state.minuteWindowStart;
|
|
105
|
+
const waitMs = Math.max(0, 60000 - minuteElapsed);
|
|
106
|
+
return {
|
|
107
|
+
allowed: false,
|
|
108
|
+
waitMs,
|
|
109
|
+
suggestedModel: null,
|
|
110
|
+
reason: `${MODELS[modelId]?.name || modelId} RPM limit reached. Wait ${Math.ceil(waitMs / 1000)}s.`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
// 4. Check health score - if model is unhealthy, suggest alternative
|
|
114
|
+
if (state.healthScore < 30 && state.consecutive429s > 3) {
|
|
115
|
+
const alternative = this.findAvailableModel(modelId);
|
|
116
|
+
if (alternative) {
|
|
117
|
+
return {
|
|
118
|
+
allowed: true,
|
|
119
|
+
waitMs: 0,
|
|
120
|
+
suggestedModel: alternative,
|
|
121
|
+
reason: `${MODELS[modelId]?.name || modelId} health is low (${state.healthScore}/100). Suggesting ${MODELS[alternative]?.name || alternative} for better reliability.`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// All checks passed
|
|
126
|
+
return {
|
|
127
|
+
allowed: true,
|
|
128
|
+
waitMs: 0,
|
|
129
|
+
suggestedModel: null,
|
|
130
|
+
reason: 'OK',
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Acquire permission to make a request. Resolves when the request can proceed.
|
|
135
|
+
* If queuing is enabled, waits in queue. Otherwise, rejects if rate limited.
|
|
136
|
+
*/
|
|
137
|
+
async acquire(modelId) {
|
|
138
|
+
const check = this.checkRateLimit(modelId);
|
|
139
|
+
if (check.allowed) {
|
|
140
|
+
// If there's a suggested model due to low health, we still allow but suggest
|
|
141
|
+
this.recordRequestStart(modelId);
|
|
142
|
+
return check;
|
|
143
|
+
}
|
|
144
|
+
// If there's an alternative model suggested, switch to it
|
|
145
|
+
if (check.suggestedModel) {
|
|
146
|
+
const altCheck = this.checkRateLimit(check.suggestedModel);
|
|
147
|
+
if (altCheck.allowed) {
|
|
148
|
+
this.recordRequestStart(check.suggestedModel);
|
|
149
|
+
if (this.config.verbose) {
|
|
150
|
+
console.log(chalk.cyan(` 🔄 Rate limiter: switching ${MODELS[modelId]?.name || modelId} → ${MODELS[check.suggestedModel]?.name || check.suggestedModel}`));
|
|
151
|
+
}
|
|
152
|
+
return { ...altCheck, suggestedModel: check.suggestedModel, reason: check.reason };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// If queuing is enabled, queue the request
|
|
156
|
+
if (this.config.enableQueue && check.waitMs > 0) {
|
|
157
|
+
return this.enqueueRequest(modelId);
|
|
158
|
+
}
|
|
159
|
+
// Can't proceed, return the check with wait info
|
|
160
|
+
if (this.config.verbose) {
|
|
161
|
+
console.log(chalk.yellow(` ⏳ Rate limited: ${check.reason}`));
|
|
162
|
+
}
|
|
163
|
+
return check;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Record a successful request completion
|
|
167
|
+
*/
|
|
168
|
+
recordSuccess(modelId, responseTimeMs) {
|
|
169
|
+
const state = this.getOrCreateState(modelId);
|
|
170
|
+
state.consecutive429s = 0;
|
|
171
|
+
state.totalSuccesses++;
|
|
172
|
+
state.lastRequestAt = Date.now();
|
|
173
|
+
// Update average response time (exponential moving average)
|
|
174
|
+
if (state.avgResponseMs === 0) {
|
|
175
|
+
state.avgResponseMs = responseTimeMs;
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
state.avgResponseMs = state.avgResponseMs * 0.8 + responseTimeMs * 0.2;
|
|
179
|
+
}
|
|
180
|
+
// Recover health score
|
|
181
|
+
state.healthScore = Math.min(100, state.healthScore + 5);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Record a rate limit (429) response
|
|
185
|
+
*/
|
|
186
|
+
recordRateLimit(modelId, retryAfterSeconds) {
|
|
187
|
+
const state = this.getOrCreateState(modelId);
|
|
188
|
+
const now = Date.now();
|
|
189
|
+
state.consecutive429s++;
|
|
190
|
+
state.total429s++;
|
|
191
|
+
state.lastRequestAt = now;
|
|
192
|
+
// Calculate cooldown duration
|
|
193
|
+
let cooldownMs;
|
|
194
|
+
if (retryAfterSeconds && retryAfterSeconds > 0) {
|
|
195
|
+
// Respect server-provided retry-after
|
|
196
|
+
cooldownMs = retryAfterSeconds * 1000;
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
// Exponential backoff based on consecutive 429s
|
|
200
|
+
cooldownMs = Math.min(this.config.baseCooldownMs * Math.pow(this.config.backoffMultiplier, state.consecutive429s - 1), this.config.maxCooldownMs);
|
|
201
|
+
}
|
|
202
|
+
state.cooldownUntil = now + cooldownMs;
|
|
203
|
+
// Decrease health score
|
|
204
|
+
state.healthScore = Math.max(0, state.healthScore - (10 + state.consecutive429s * 5));
|
|
205
|
+
// Adaptive learning: record the pattern
|
|
206
|
+
if (this.config.adaptiveLearning) {
|
|
207
|
+
this.recordRateLimitPattern(modelId, cooldownMs);
|
|
208
|
+
}
|
|
209
|
+
if (this.config.verbose) {
|
|
210
|
+
console.log(chalk.yellow(` 🚫 Rate limited on ${MODELS[modelId]?.name || modelId}: ` +
|
|
211
|
+
`cooldown ${Math.ceil(cooldownMs / 1000)}s ` +
|
|
212
|
+
`(consecutive: ${state.consecutive429s}, health: ${state.healthScore}/100)`));
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Record a server error (5xx) - treat as potential overload
|
|
217
|
+
*/
|
|
218
|
+
recordServerError(modelId) {
|
|
219
|
+
const state = this.getOrCreateState(modelId);
|
|
220
|
+
const now = Date.now();
|
|
221
|
+
// Apply a shorter cooldown for server errors
|
|
222
|
+
state.cooldownUntil = now + this.config.baseCooldownMs / 2;
|
|
223
|
+
state.healthScore = Math.max(0, state.healthScore - 15);
|
|
224
|
+
if (this.config.verbose) {
|
|
225
|
+
console.log(chalk.yellow(` ⚠️ Server error on ${MODELS[modelId]?.name || modelId}: ` +
|
|
226
|
+
`cooldown ${Math.ceil(this.config.baseCooldownMs / 2000)}s, health: ${state.healthScore}/100`));
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
// =========================================================================
|
|
230
|
+
// Model Rotation
|
|
231
|
+
// =========================================================================
|
|
232
|
+
/**
|
|
233
|
+
* Get the next available model from the rotation pool.
|
|
234
|
+
* Skips models that are in cooldown or have low health.
|
|
235
|
+
*/
|
|
236
|
+
getNextRotationModel(excludeModel) {
|
|
237
|
+
const now = Date.now();
|
|
238
|
+
const poolSize = this.rotationPool.length;
|
|
239
|
+
if (poolSize === 0)
|
|
240
|
+
return null;
|
|
241
|
+
// Try each model in the pool, starting from current index
|
|
242
|
+
for (let i = 0; i < poolSize; i++) {
|
|
243
|
+
const idx = (this.currentRotationIndex + i) % poolSize;
|
|
244
|
+
const model = this.rotationPool[idx];
|
|
245
|
+
if (model === excludeModel)
|
|
246
|
+
continue;
|
|
247
|
+
const state = this.getOrCreateState(model);
|
|
248
|
+
// Skip if in cooldown
|
|
249
|
+
if (state.cooldownUntil > now)
|
|
250
|
+
continue;
|
|
251
|
+
// Skip if health is too low
|
|
252
|
+
if (state.healthScore < 20)
|
|
253
|
+
continue;
|
|
254
|
+
// Move rotation index forward
|
|
255
|
+
this.currentRotationIndex = (idx + 1) % poolSize;
|
|
256
|
+
return model;
|
|
257
|
+
}
|
|
258
|
+
// All models in pool are rate limited or unhealthy
|
|
259
|
+
// Find the one with the shortest wait time
|
|
260
|
+
let bestModel = null;
|
|
261
|
+
let shortestWait = Infinity;
|
|
262
|
+
for (const model of this.rotationPool) {
|
|
263
|
+
if (model === excludeModel)
|
|
264
|
+
continue;
|
|
265
|
+
const state = this.getOrCreateState(model);
|
|
266
|
+
const waitMs = Math.max(0, state.cooldownUntil - now);
|
|
267
|
+
if (waitMs < shortestWait) {
|
|
268
|
+
shortestWait = waitMs;
|
|
269
|
+
bestModel = model;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return bestModel;
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Get the best available model for a given requested model.
|
|
276
|
+
* Tries the requested model first, then falls back through the rotation pool.
|
|
277
|
+
*/
|
|
278
|
+
getBestAvailableModel(requestedModel) {
|
|
279
|
+
// Check if requested model is available
|
|
280
|
+
const check = this.checkRateLimit(requestedModel);
|
|
281
|
+
if (check.allowed)
|
|
282
|
+
return requestedModel;
|
|
283
|
+
// Try rotation pool
|
|
284
|
+
const alternative = this.findAvailableModel(requestedModel);
|
|
285
|
+
if (alternative)
|
|
286
|
+
return alternative;
|
|
287
|
+
// Last resort: return requested model (will wait)
|
|
288
|
+
return requestedModel;
|
|
289
|
+
}
|
|
290
|
+
// =========================================================================
|
|
291
|
+
// Status & Diagnostics
|
|
292
|
+
// =========================================================================
|
|
293
|
+
/**
|
|
294
|
+
* Get rate limit status for all tracked models
|
|
295
|
+
*/
|
|
296
|
+
getStatus() {
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
const result = [];
|
|
299
|
+
for (const [modelId, state] of this.modelStates) {
|
|
300
|
+
this.cleanupModelTimestamps(state, now);
|
|
301
|
+
result.push({
|
|
302
|
+
model: modelId,
|
|
303
|
+
name: MODELS[modelId]?.name || modelId,
|
|
304
|
+
health: state.healthScore,
|
|
305
|
+
cooldown: Math.max(0, Math.ceil((state.cooldownUntil - now) / 1000)),
|
|
306
|
+
consecutive429s: state.consecutive429s,
|
|
307
|
+
total429s: state.total429s,
|
|
308
|
+
totalSuccesses: state.totalSuccesses,
|
|
309
|
+
rpm: state.requestsThisMinute,
|
|
310
|
+
avgResponseMs: Math.round(state.avgResponseMs),
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
return result.sort((a, b) => b.health - a.health);
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Get queue status
|
|
317
|
+
*/
|
|
318
|
+
getQueueStatus() {
|
|
319
|
+
if (this.requestQueue.length === 0)
|
|
320
|
+
return { size: 0, oldestWaitMs: 0 };
|
|
321
|
+
return {
|
|
322
|
+
size: this.requestQueue.length,
|
|
323
|
+
oldestWaitMs: Date.now() - this.requestQueue[0].enqueuedAt,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Reset all rate limit state (useful for testing)
|
|
328
|
+
*/
|
|
329
|
+
reset() {
|
|
330
|
+
for (const state of this.modelStates.values()) {
|
|
331
|
+
state.cooldownUntil = 0;
|
|
332
|
+
state.consecutive429s = 0;
|
|
333
|
+
state.healthScore = 100;
|
|
334
|
+
state.requestsThisMinute = 0;
|
|
335
|
+
}
|
|
336
|
+
this.globalRequestTimestamps = [];
|
|
337
|
+
this.requestQueue.forEach(r => r.reject(new Error('Rate limiter reset')));
|
|
338
|
+
this.requestQueue = [];
|
|
339
|
+
if (this.config.verbose) {
|
|
340
|
+
console.log(chalk.gray(' ↻ Rate limiter state reset'));
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Reset a specific model's rate limit state
|
|
345
|
+
*/
|
|
346
|
+
resetModel(modelId) {
|
|
347
|
+
const state = this.modelStates.get(modelId);
|
|
348
|
+
if (state) {
|
|
349
|
+
state.cooldownUntil = 0;
|
|
350
|
+
state.consecutive429s = 0;
|
|
351
|
+
state.healthScore = 100;
|
|
352
|
+
state.requestsThisMinute = 0;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Update configuration at runtime
|
|
357
|
+
*/
|
|
358
|
+
updateConfig(updates) {
|
|
359
|
+
Object.assign(this.config, updates);
|
|
360
|
+
if (updates.rotationPoolSize !== undefined) {
|
|
361
|
+
this.initializeRotationPool();
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Print a formatted status table
|
|
366
|
+
*/
|
|
367
|
+
printStatus() {
|
|
368
|
+
const status = this.getStatus();
|
|
369
|
+
const now = Date.now();
|
|
370
|
+
console.log(chalk.bold('\n📊 Rate Limiter Status'));
|
|
371
|
+
console.log(chalk.gray('─'.repeat(80)));
|
|
372
|
+
for (const s of status) {
|
|
373
|
+
const healthBar = this.getHealthBar(s.health);
|
|
374
|
+
const cooldownStr = s.cooldown > 0 ? chalk.red(`${s.cooldown}s`) : chalk.green('✓');
|
|
375
|
+
const rpmStr = `${s.rpm}/${this.config.perModelRpm}`;
|
|
376
|
+
console.log(` ${s.name.padEnd(35)} ${healthBar} ${cooldownStr.padEnd(8)} ` +
|
|
377
|
+
`429s: ${chalk.yellow(String(s.total429s).padEnd(4))} ` +
|
|
378
|
+
`OK: ${chalk.green(String(s.totalSuccesses).padEnd(4))} ` +
|
|
379
|
+
`RPM: ${rpmStr.padEnd(6)} ` +
|
|
380
|
+
`avg: ${s.avgResponseMs}ms`);
|
|
381
|
+
}
|
|
382
|
+
const queueStatus = this.getQueueStatus();
|
|
383
|
+
if (queueStatus.size > 0) {
|
|
384
|
+
console.log(chalk.yellow(`\n Queue: ${queueStatus.size} requests waiting (oldest: ${Math.ceil(queueStatus.oldestWaitMs / 1000)}s)`));
|
|
385
|
+
}
|
|
386
|
+
console.log(chalk.gray('─'.repeat(80)));
|
|
387
|
+
}
|
|
388
|
+
// =========================================================================
|
|
389
|
+
// Private Methods
|
|
390
|
+
// =========================================================================
|
|
391
|
+
getOrCreateState(modelId) {
|
|
392
|
+
if (!this.modelStates.has(modelId)) {
|
|
393
|
+
this.modelStates.set(modelId, {
|
|
394
|
+
modelId,
|
|
395
|
+
cooldownUntil: 0,
|
|
396
|
+
consecutive429s: 0,
|
|
397
|
+
total429s: 0,
|
|
398
|
+
totalSuccesses: 0,
|
|
399
|
+
requestsThisMinute: 0,
|
|
400
|
+
minuteWindowStart: Date.now(),
|
|
401
|
+
lastRequestAt: 0,
|
|
402
|
+
avgResponseMs: 0,
|
|
403
|
+
inRotationPool: this.rotationPool.includes(modelId),
|
|
404
|
+
healthScore: 100,
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
return this.modelStates.get(modelId);
|
|
408
|
+
}
|
|
409
|
+
recordRequestStart(modelId) {
|
|
410
|
+
const now = Date.now();
|
|
411
|
+
const state = this.getOrCreateState(modelId);
|
|
412
|
+
// Update per-model minute window
|
|
413
|
+
this.cleanupModelTimestamps(state, now);
|
|
414
|
+
state.requestsThisMinute++;
|
|
415
|
+
state.lastRequestAt = now;
|
|
416
|
+
// Update global timestamps
|
|
417
|
+
this.cleanupGlobalTimestamps(now);
|
|
418
|
+
this.globalRequestTimestamps.push(now);
|
|
419
|
+
}
|
|
420
|
+
cleanupGlobalTimestamps(now) {
|
|
421
|
+
const oneMinuteAgo = now - 60000;
|
|
422
|
+
while (this.globalRequestTimestamps.length > 0 && this.globalRequestTimestamps[0] < oneMinuteAgo) {
|
|
423
|
+
this.globalRequestTimestamps.shift();
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
cleanupModelTimestamps(state, now) {
|
|
427
|
+
if (now - state.minuteWindowStart >= 60000) {
|
|
428
|
+
state.requestsThisMinute = 0;
|
|
429
|
+
state.minuteWindowStart = now;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
findAvailableModel(excludeModel) {
|
|
433
|
+
const now = Date.now();
|
|
434
|
+
// First, try the rotation pool
|
|
435
|
+
for (const model of this.rotationPool) {
|
|
436
|
+
if (model === excludeModel)
|
|
437
|
+
continue;
|
|
438
|
+
const state = this.getOrCreateState(model);
|
|
439
|
+
if (state.cooldownUntil <= now && state.healthScore >= 20) {
|
|
440
|
+
this.cleanupModelTimestamps(state, now);
|
|
441
|
+
if (state.requestsThisMinute < this.config.perModelRpm) {
|
|
442
|
+
return model;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
// Then try any tracked model that's available
|
|
447
|
+
for (const [modelId, state] of this.modelStates) {
|
|
448
|
+
if (modelId === excludeModel)
|
|
449
|
+
continue;
|
|
450
|
+
if (state.cooldownUntil <= now && state.healthScore >= 20) {
|
|
451
|
+
this.cleanupModelTimestamps(state, now);
|
|
452
|
+
if (state.requestsThisMinute < this.config.perModelRpm) {
|
|
453
|
+
return modelId;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
return null;
|
|
458
|
+
}
|
|
459
|
+
initializeRotationPool() {
|
|
460
|
+
const freeModels = getFreeModelsWithTools();
|
|
461
|
+
// Sort by context window (larger is better for coding)
|
|
462
|
+
const sorted = freeModels.sort((a, b) => b.contextWindow - a.contextWindow);
|
|
463
|
+
// Take top N models for rotation
|
|
464
|
+
this.rotationPool = sorted
|
|
465
|
+
.slice(0, Math.min(this.config.rotationPoolSize, sorted.length))
|
|
466
|
+
.map(m => m.id);
|
|
467
|
+
// Mark all as in rotation pool
|
|
468
|
+
for (const modelId of this.rotationPool) {
|
|
469
|
+
const state = this.getOrCreateState(modelId);
|
|
470
|
+
state.inRotationPool = true;
|
|
471
|
+
}
|
|
472
|
+
if (this.config.verbose) {
|
|
473
|
+
console.log(chalk.gray(` 🔄 Rate limiter rotation pool: ${this.rotationPool.map(m => MODELS[m]?.name || m).join(', ')}`));
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
enqueueRequest(modelId) {
|
|
477
|
+
return new Promise((resolve, reject) => {
|
|
478
|
+
if (this.requestQueue.length >= this.config.maxQueueSize) {
|
|
479
|
+
reject(new Error(`Rate limiter queue full (${this.config.maxQueueSize}). Try again later.`));
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const id = `req_${++this.requestIdCounter}`;
|
|
483
|
+
const queued = {
|
|
484
|
+
id,
|
|
485
|
+
model: modelId,
|
|
486
|
+
enqueuedAt: Date.now(),
|
|
487
|
+
resolve,
|
|
488
|
+
reject,
|
|
489
|
+
};
|
|
490
|
+
this.requestQueue.push(queued);
|
|
491
|
+
if (this.config.verbose) {
|
|
492
|
+
console.log(chalk.gray(` ⏳ Request queued for ${MODELS[modelId]?.name || modelId} (queue: ${this.requestQueue.length})`));
|
|
493
|
+
}
|
|
494
|
+
// Set a timeout to reject if waiting too long
|
|
495
|
+
setTimeout(() => {
|
|
496
|
+
const idx = this.requestQueue.findIndex(r => r.id === id);
|
|
497
|
+
if (idx !== -1) {
|
|
498
|
+
this.requestQueue.splice(idx, 1);
|
|
499
|
+
reject(new Error(`Request timed out waiting in rate limiter queue (${this.config.maxQueueWaitMs / 1000}s)`));
|
|
500
|
+
}
|
|
501
|
+
}, this.config.maxQueueWaitMs);
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
startQueueProcessor() {
|
|
505
|
+
// Process queue every 2 seconds
|
|
506
|
+
this.queueProcessTimer = setInterval(() => {
|
|
507
|
+
this.processQueue();
|
|
508
|
+
}, 2000);
|
|
509
|
+
}
|
|
510
|
+
processQueue() {
|
|
511
|
+
if (this.requestQueue.length === 0)
|
|
512
|
+
return;
|
|
513
|
+
const now = Date.now();
|
|
514
|
+
const toProcess = [];
|
|
515
|
+
// Find requests that can now proceed
|
|
516
|
+
for (let i = this.requestQueue.length - 1; i >= 0; i--) {
|
|
517
|
+
const req = this.requestQueue[i];
|
|
518
|
+
// Check if request has been waiting too long
|
|
519
|
+
if (now - req.enqueuedAt > this.config.maxQueueWaitMs) {
|
|
520
|
+
this.requestQueue.splice(i, 1);
|
|
521
|
+
req.reject(new Error('Request timed out in rate limiter queue'));
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
524
|
+
const check = this.checkRateLimit(req.model);
|
|
525
|
+
if (check.allowed) {
|
|
526
|
+
toProcess.push(req);
|
|
527
|
+
this.requestQueue.splice(i, 1);
|
|
528
|
+
}
|
|
529
|
+
else if (check.suggestedModel) {
|
|
530
|
+
// Try alternative model
|
|
531
|
+
const altCheck = this.checkRateLimit(check.suggestedModel);
|
|
532
|
+
if (altCheck.allowed) {
|
|
533
|
+
this.recordRequestStart(check.suggestedModel);
|
|
534
|
+
req.resolve({ ...altCheck, suggestedModel: check.suggestedModel, reason: check.reason });
|
|
535
|
+
this.requestQueue.splice(i, 1);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
// Process approved requests
|
|
540
|
+
for (const req of toProcess) {
|
|
541
|
+
this.recordRequestStart(req.model);
|
|
542
|
+
req.resolve({ allowed: true, waitMs: 0, suggestedModel: null, reason: 'Queued request approved' });
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
recordRateLimitPattern(modelId, cooldownMs) {
|
|
546
|
+
const hour = new Date().getHours();
|
|
547
|
+
let patterns = this.rateLimitPatterns.get(modelId);
|
|
548
|
+
if (!patterns) {
|
|
549
|
+
patterns = [];
|
|
550
|
+
this.rateLimitPatterns.set(modelId, patterns);
|
|
551
|
+
}
|
|
552
|
+
let existing = patterns.find(p => p.hourOfDay === hour);
|
|
553
|
+
if (!existing) {
|
|
554
|
+
existing = { hourOfDay: hour, avgCooldownMs: cooldownMs, count: 1 };
|
|
555
|
+
patterns.push(existing);
|
|
556
|
+
}
|
|
557
|
+
else {
|
|
558
|
+
existing.avgCooldownMs = (existing.avgCooldownMs * existing.count + cooldownMs) / (existing.count + 1);
|
|
559
|
+
existing.count++;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
getHealthBar(health) {
|
|
563
|
+
const filled = Math.round(health / 10);
|
|
564
|
+
const empty = 10 - filled;
|
|
565
|
+
let color;
|
|
566
|
+
if (health >= 70)
|
|
567
|
+
color = chalk.green;
|
|
568
|
+
else if (health >= 40)
|
|
569
|
+
color = chalk.yellow;
|
|
570
|
+
else
|
|
571
|
+
color = chalk.red;
|
|
572
|
+
return color('█'.repeat(filled) + '░'.repeat(empty));
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Cleanup - call when shutting down
|
|
576
|
+
*/
|
|
577
|
+
dispose() {
|
|
578
|
+
if (this.queueProcessTimer) {
|
|
579
|
+
clearInterval(this.queueProcessTimer);
|
|
580
|
+
this.queueProcessTimer = null;
|
|
581
|
+
}
|
|
582
|
+
// Reject all queued requests
|
|
583
|
+
for (const req of this.requestQueue) {
|
|
584
|
+
req.reject(new Error('Rate limiter shutting down'));
|
|
585
|
+
}
|
|
586
|
+
this.requestQueue = [];
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
//# sourceMappingURL=rate-limiter.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "neuro-cli",
|
|
3
|
-
"version": "5.0.
|
|
3
|
+
"version": "5.0.1",
|
|
4
4
|
"description": "Advanced AI-powered terminal coding assistant with multi-agent orchestration, sub-agent spawning, ACP protocol, OS-level sandboxing, spec-driven development, smart monitoring, multi-model routing, MCP Apps, auto-updater, and 23+ free models",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|