neuro-cli 5.0.0 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { Message, ToolCall, ToolDefinition, ToolResult } from '../core/types.js';
2
+ import { RateLimitManager } from '../core/rate-limiter.js';
2
3
  export interface StreamCallbacks {
3
4
  onToken?: (token: string) => void;
4
5
  onToolCall?: (toolCall: ToolCall) => void;
@@ -24,11 +25,16 @@ export declare class OpenRouterClient {
24
25
  private apiKey;
25
26
  private baseUrl;
26
27
  private totalUsage;
28
+ private rateLimiter;
27
29
  constructor(apiKey: string, baseUrl?: string);
28
30
  get usage(): TokenUsage;
29
31
  resetUsage(): void;
30
32
  /**
31
- * Main chat completion with streaming
33
+ * Set the rate limiter instance for this client
34
+ */
35
+ setRateLimiter(rateLimiter: RateLimitManager): void;
36
+ /**
37
+ * Main chat completion with streaming + rate limit awareness
32
38
  */
33
39
  chat(request: ChatRequest, callbacks?: StreamCallbacks): Promise<{
34
40
  content: string;
@@ -1,6 +1,7 @@
1
1
  // ============================================================
2
2
  // NeuroCLI - OpenRouter API Client
3
3
  // Streaming + Tool Use + Multi-model support
4
+ // Now with RateLimitManager integration
4
5
  // ============================================================
5
6
  // @ts-ignore
6
7
  import { createParser } from 'eventsource-parser';
@@ -9,6 +10,7 @@ export class OpenRouterClient {
9
10
  apiKey;
10
11
  baseUrl;
11
12
  totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
13
+ rateLimiter = null;
12
14
  constructor(apiKey, baseUrl = 'https://openrouter.ai/api/v1') {
13
15
  this.apiKey = apiKey;
14
16
  this.baseUrl = baseUrl;
@@ -20,15 +22,42 @@ export class OpenRouterClient {
20
22
  this.totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
21
23
  }
22
24
  /**
23
- * Main chat completion with streaming
25
+ * Set the rate limiter instance for this client
26
+ */
27
+ setRateLimiter(rateLimiter) {
28
+ this.rateLimiter = rateLimiter;
29
+ }
30
+ /**
31
+ * Main chat completion with streaming + rate limit awareness
24
32
  */
25
33
  async chat(request, callbacks) {
26
34
  const model = MODELS[request.model];
27
35
  if (!model) {
28
36
  throw new Error(`Unknown model: ${request.model}`);
29
37
  }
38
+ // --- Rate limit pre-check ---
39
+ let activeModel = request.model;
40
+ if (this.rateLimiter) {
41
+ const check = await this.rateLimiter.acquire(request.model);
42
+ if (check.suggestedModel) {
43
+ // Switch to the suggested alternative model
44
+ activeModel = check.suggestedModel;
45
+ callbacks?.onThinking?.(`🔄 Rate limiter rotated to ${MODELS[activeModel]?.name || activeModel}`);
46
+ }
47
+ else if (!check.allowed) {
48
+ // Not allowed and no alternative - wait
49
+ if (check.waitMs > 0) {
50
+ callbacks?.onThinking?.(`⏳ Rate limited, waiting ${Math.ceil(check.waitMs / 1000)}s...`);
51
+ await new Promise(resolve => setTimeout(resolve, check.waitMs));
52
+ }
53
+ }
54
+ }
55
+ const activeModelConfig = MODELS[activeModel];
56
+ if (!activeModelConfig) {
57
+ throw new Error(`Unknown model after rate limit rotation: ${activeModel}`);
58
+ }
30
59
  const body = {
31
- model: request.model,
60
+ model: activeModel,
32
61
  messages: request.messages.map(m => ({
33
62
  role: m.role,
34
63
  content: m.content,
@@ -38,9 +67,9 @@ export class OpenRouterClient {
38
67
  })),
39
68
  stream: request.stream ?? true,
40
69
  temperature: request.temperature ?? 0.7,
41
- max_tokens: request.maxTokens ?? model.maxOutput,
70
+ max_tokens: request.maxTokens ?? activeModelConfig.maxOutput,
42
71
  };
43
- if (request.tools && request.tools.length > 0 && model.supportsTools) {
72
+ if (request.tools && request.tools.length > 0 && activeModelConfig.supportsTools) {
44
73
  body.tools = request.tools.map(t => ({
45
74
  type: 'function',
46
75
  function: {
@@ -54,47 +83,133 @@ export class OpenRouterClient {
54
83
  let lastError = null;
55
84
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
56
85
  if (attempt > 0) {
57
- // Exponential backoff: 5s, 15s, 30s
58
- const waitTime = Math.min(5000 * Math.pow(3, attempt - 1), 30000);
59
- callbacks?.onThinking?.(`⏳ Rate limited, retrying in ${waitTime / 1000}s (attempt ${attempt + 1}/${maxRetries + 1})...`);
86
+ // Smart backoff: use rate limiter cooldown if available, otherwise exponential
87
+ let waitTime;
88
+ if (this.rateLimiter) {
89
+ const state = this.rateLimiter.getStatus().find(s => s.model === activeModel);
90
+ waitTime = state && state.cooldown > 0 ? state.cooldown * 1000 : Math.min(5000 * Math.pow(2, attempt - 1), 30000);
91
+ }
92
+ else {
93
+ waitTime = Math.min(5000 * Math.pow(3, attempt - 1), 30000);
94
+ }
95
+ callbacks?.onThinking?.(`⏳ Retrying in ${Math.ceil(waitTime / 1000)}s (attempt ${attempt + 1}/${maxRetries + 1})...`);
60
96
  await new Promise(resolve => setTimeout(resolve, waitTime));
61
97
  }
62
- const response = await fetch(`${this.baseUrl}/chat/completions`, {
63
- method: 'POST',
64
- headers: {
65
- 'Authorization': `Bearer ${this.apiKey}`,
66
- 'Content-Type': 'application/json',
67
- 'HTTP-Referer': 'https://neurocli.dev',
68
- 'X-Title': 'NeuroCLI',
69
- },
70
- body: JSON.stringify(body),
71
- });
72
- if (!response.ok) {
73
- const errorBody = await response.text();
74
- // Parse retry_after from 429 errors
75
- if (response.status === 429) {
76
- try {
77
- const errorData = JSON.parse(errorBody);
78
- const retryAfter = errorData?.error?.metadata?.retry_after_seconds;
79
- if (retryAfter && attempt < maxRetries) {
80
- lastError = new Error(`Rate limited, retry after ${retryAfter}s`);
98
+ const startTime = Date.now();
99
+ try {
100
+ const response = await fetch(`${this.baseUrl}/chat/completions`, {
101
+ method: 'POST',
102
+ headers: {
103
+ 'Authorization': `Bearer ${this.apiKey}`,
104
+ 'Content-Type': 'application/json',
105
+ 'HTTP-Referer': 'https://neurocli.dev',
106
+ 'X-Title': 'NeuroCLI',
107
+ },
108
+ body: JSON.stringify(body),
109
+ });
110
+ const responseTimeMs = Date.now() - startTime;
111
+ if (!response.ok) {
112
+ const errorBody = await response.text();
113
+ // Parse retry_after from 429 errors
114
+ if (response.status === 429) {
115
+ let retryAfterSeconds;
116
+ try {
117
+ const errorData = JSON.parse(errorBody);
118
+ retryAfterSeconds = errorData?.error?.metadata?.retry_after_seconds;
119
+ }
120
+ catch { }
121
+ // Record rate limit in manager
122
+ if (this.rateLimiter) {
123
+ this.rateLimiter.recordRateLimit(activeModel, retryAfterSeconds);
124
+ }
125
+ // Try model rotation before retrying
126
+ if (this.rateLimiter && attempt < maxRetries) {
127
+ const nextModel = this.rateLimiter.getNextRotationModel(activeModel);
128
+ if (nextModel && nextModel !== activeModel) {
129
+ const nextModelConfig = MODELS[nextModel];
130
+ if (nextModelConfig) {
131
+ callbacks?.onThinking?.(`🔄 Rotating ${MODELS[activeModel]?.name || activeModel} → ${nextModelConfig.name} due to rate limit`);
132
+ activeModel = nextModel;
133
+ body.model = nextModel;
134
+ body.max_tokens = request.maxTokens ?? nextModelConfig.maxOutput;
135
+ if (request.tools && request.tools.length > 0 && nextModelConfig.supportsTools) {
136
+ // tools already set
137
+ }
138
+ else if (request.tools && !nextModelConfig.supportsTools) {
139
+ delete body.tools;
140
+ }
141
+ // Don't count this as a retry attempt for the new model
142
+ attempt--;
143
+ continue;
144
+ }
145
+ }
146
+ }
147
+ if (retryAfterSeconds && attempt < maxRetries) {
148
+ lastError = new Error(`Rate limited, retry after ${retryAfterSeconds}s`);
81
149
  continue; // Retry
82
150
  }
151
+ if (attempt < maxRetries) {
152
+ lastError = new Error(`Rate limited (429): ${errorBody}`);
153
+ continue;
154
+ }
83
155
  }
84
- catch { }
85
- // If we can't parse retry_after or max retries reached, throw
86
- if (attempt < maxRetries) {
87
- lastError = new Error(`Rate limited (429): ${errorBody}`);
88
- continue;
156
+ // Server errors (5xx) - record and potentially rotate
157
+ if (response.status >= 500) {
158
+ if (this.rateLimiter) {
159
+ this.rateLimiter.recordServerError(activeModel);
160
+ }
161
+ if (attempt < maxRetries) {
162
+ lastError = new Error(`Server error (${response.status}): ${errorBody}`);
163
+ // Try model rotation for server errors too
164
+ if (this.rateLimiter) {
165
+ const nextModel = this.rateLimiter.getNextRotationModel(activeModel);
166
+ if (nextModel && nextModel !== activeModel) {
167
+ const nextModelConfig = MODELS[nextModel];
168
+ if (nextModelConfig) {
169
+ callbacks?.onThinking?.(`🔄 Rotating away from ${MODELS[activeModel]?.name || activeModel} due to server error`);
170
+ activeModel = nextModel;
171
+ body.model = nextModel;
172
+ body.max_tokens = request.maxTokens ?? nextModelConfig.maxOutput;
173
+ if (request.tools && request.tools.length > 0 && nextModelConfig.supportsTools) {
174
+ // tools already set
175
+ }
176
+ else if (request.tools && !nextModelConfig.supportsTools) {
177
+ delete body.tools;
178
+ }
179
+ attempt--;
180
+ continue;
181
+ }
182
+ }
183
+ }
184
+ continue;
185
+ }
89
186
  }
187
+ throw new Error(`OpenRouter API error (${response.status}): ${errorBody}`);
188
+ }
189
+ // Success!
190
+ if (this.rateLimiter) {
191
+ this.rateLimiter.recordSuccess(activeModel, responseTimeMs);
192
+ }
193
+ if (request.stream && callbacks) {
194
+ const result = await this.handleStreamingResponse(response, activeModel, callbacks);
195
+ return result;
196
+ }
197
+ else {
198
+ const result = await this.handleNonStreamingResponse(response, activeModel);
199
+ return result;
90
200
  }
91
- throw new Error(`OpenRouter API error (${response.status}): ${errorBody}`);
92
- }
93
- if (request.stream && callbacks) {
94
- return this.handleStreamingResponse(response, request.model, callbacks);
95
201
  }
96
- else {
97
- return this.handleNonStreamingResponse(response, request.model);
202
+ catch (error) {
203
+ // Network-level errors (fetch failed)
204
+ if (error instanceof TypeError && error.message.includes('fetch')) {
205
+ if (this.rateLimiter) {
206
+ this.rateLimiter.recordServerError(activeModel);
207
+ }
208
+ lastError = error;
209
+ if (attempt < maxRetries)
210
+ continue;
211
+ }
212
+ throw error;
98
213
  }
99
214
  }
100
215
  throw lastError || new Error('Max retries exceeded');
@@ -297,6 +297,17 @@ export class CommandSystem {
297
297
  description: 'Browser automation (navigate/screenshot/click)',
298
298
  prompt: 'Use the browser to navigate, take screenshots, click elements, or extract content from web pages. {args}',
299
299
  },
300
+ // --- Rate Limit Commands ---
301
+ {
302
+ name: 'ratelimit',
303
+ description: 'Show rate limiter status for all models',
304
+ prompt: 'Show the current rate limiter status. Display health scores, cooldowns, 429 counts, and queue status for all tracked models. {args}',
305
+ },
306
+ {
307
+ name: 'ratelimit-reset',
308
+ description: 'Reset rate limiter state',
309
+ prompt: 'Reset all rate limiter cooldowns and health scores. Use this if models are stuck in cooldown. {args}',
310
+ },
300
311
  ];
301
312
  for (const cmd of bundled) {
302
313
  if (!this.commands.has(cmd.name)) {
@@ -57,6 +57,7 @@ import { TerminalUX } from './terminal-ux.js';
57
57
  import { MultiSessionManager } from './multi-session.js';
58
58
  import { GitWorktreeManager } from './git-worktree.js';
59
59
  import { AutoUpdater } from './updater.js';
60
+ import { RateLimitManager } from './rate-limiter.js';
60
61
  export declare class NeuroEngine {
61
62
  config: NeuroConfig;
62
63
  client: OpenRouterClient;
@@ -117,6 +118,7 @@ export declare class NeuroEngine {
117
118
  multiSession: MultiSessionManager;
118
119
  gitWorktree: GitWorktreeManager;
119
120
  updater: AutoUpdater;
121
+ rateLimiter: RateLimitManager;
120
122
  private autoApproveSet;
121
123
  private requireApprovalSet;
122
124
  constructor(config: NeuroConfig);
@@ -15,7 +15,8 @@
15
15
  // Spec-Driven Development, LLM Evaluator Hooks, MCP Apps,
16
16
  // Multi-Model Orchestrator, Smart Monitor, Outcome Grading,
17
17
  // Observability (OTLP), Auto-Compact, Terminal UX,
18
- // Multi-Session, Git Worktree, Auto-Updater
18
+ // Multi-Session, Git Worktree, Auto-Updater,
19
+ // Rate Limit Manager
19
20
  // ============================================================
20
21
  import { join } from 'path';
21
22
  import { homedir } from 'os';
@@ -79,6 +80,7 @@ import { TerminalUX } from './terminal-ux.js';
79
80
  import { MultiSessionManager } from './multi-session.js';
80
81
  import { GitWorktreeManager } from './git-worktree.js';
81
82
  import { AutoUpdater } from './updater.js';
83
+ import { RateLimitManager } from './rate-limiter.js';
82
84
  export class NeuroEngine {
83
85
  config;
84
86
  client;
@@ -143,11 +145,27 @@ export class NeuroEngine {
143
145
  multiSession;
144
146
  gitWorktree;
145
147
  updater;
148
+ rateLimiter;
146
149
  autoApproveSet;
147
150
  requireApprovalSet;
148
151
  constructor(config) {
149
152
  this.config = config;
153
+ // Rate Limit Manager - initialized FIRST so it's available for all systems
154
+ this.rateLimiter = new RateLimitManager({
155
+ globalRpm: 20,
156
+ perModelRpm: 10,
157
+ baseCooldownMs: 10000,
158
+ maxCooldownMs: 120000,
159
+ backoffMultiplier: 2.0,
160
+ rotationPoolSize: 5,
161
+ enableQueue: true,
162
+ maxQueueSize: 50,
163
+ maxQueueWaitMs: 60000,
164
+ adaptiveLearning: true,
165
+ verbose: true,
166
+ });
150
167
  this.client = new OpenRouterClient(config.apiKey, config.baseUrl);
168
+ this.client.setRateLimiter(this.rateLimiter);
151
169
  this.registry = registerAllTools(new ToolRegistry());
152
170
  this.contextManager = new ContextManager(config.defaultModel, config.context.maxTokens);
153
171
  this.sessionManager = new SessionManager();
@@ -166,6 +184,7 @@ export class NeuroEngine {
166
184
  return false;
167
185
  });
168
186
  this.fallback = new FallbackChain(this.client, config.fallbackChain);
187
+ this.fallback.setRateLimiter(this.rateLimiter);
169
188
  // Sandbox system
170
189
  this.sandbox = new Sandbox(config.sandbox);
171
190
  // Plugin system
@@ -196,6 +215,7 @@ export class NeuroEngine {
196
215
  categoryOverrides: {},
197
216
  maxTokenBudget: config.context.maxTokens,
198
217
  }, Object.fromEntries(Object.entries(MODELS).map(([id, m]) => [id, { name: m.name, contextWindow: m.contextWindow, maxOutput: m.maxOutput }])));
218
+ this.modelRouter.setRateLimiter(this.rateLimiter);
199
219
  // Output styles
200
220
  this.styleManager = new StyleManager(process.cwd());
201
221
  // Extended thinking
@@ -1,5 +1,6 @@
1
1
  import { OpenRouterClient, TokenUsage } from '../api/openrouter.js';
2
2
  import { Message } from './types.js';
3
+ import { RateLimitManager } from './rate-limiter.js';
3
4
  export interface FallbackConfig {
4
5
  models: string[];
5
6
  maxRetries: number;
@@ -27,9 +28,14 @@ export interface FallbackResult {
27
28
  export declare class FallbackChain {
28
29
  private config;
29
30
  private client;
31
+ private rateLimiter;
30
32
  constructor(client: OpenRouterClient, config?: Partial<FallbackConfig>);
31
33
  /**
32
- * Execute a chat request with fallback chain
34
+ * Set the rate limiter for intelligent model rotation
35
+ */
36
+ setRateLimiter(rateLimiter: RateLimitManager): void;
37
+ /**
38
+ * Execute a chat request with fallback chain + rate limit awareness
33
39
  */
34
40
  chatWithFallback(primaryModel: string, messages: Message[], options: {
35
41
  tools?: unknown[];
@@ -1,12 +1,14 @@
1
1
  // ============================================================
2
2
  // NeuroCLI - Fallback Model Chain
3
3
  // Automatic model fallback on failure
4
+ // Now with RateLimitManager integration
4
5
  // ============================================================
5
6
  import { MODELS } from '../api/models.js';
6
7
  import chalk from 'chalk';
7
8
  export class FallbackChain {
8
9
  config;
9
10
  client;
11
+ rateLimiter = null;
10
12
  constructor(client, config) {
11
13
  this.client = client;
12
14
  this.config = {
@@ -18,13 +20,67 @@ export class FallbackChain {
18
20
  };
19
21
  }
20
22
  /**
21
- * Execute a chat request with fallback chain
23
+ * Set the rate limiter for intelligent model rotation
24
+ */
25
+ setRateLimiter(rateLimiter) {
26
+ this.rateLimiter = rateLimiter;
27
+ }
28
+ /**
29
+ * Execute a chat request with fallback chain + rate limit awareness
22
30
  */
23
31
  async chatWithFallback(primaryModel, messages, options, onModelSwitch) {
24
- const chain = [primaryModel, ...this.config.models.filter(m => m !== primaryModel)];
32
+ // Build the model chain: primary + configured fallbacks
33
+ let chain = [primaryModel, ...this.config.models.filter(m => m !== primaryModel)];
34
+ // If rate limiter is available, prioritize models with good health
35
+ if (this.rateLimiter) {
36
+ const availableModels = this.rateLimiter.getNextRotationModel(primaryModel);
37
+ // Insert rate-limiter-suggested models at the front of the chain
38
+ const healthyAlternatives = [];
39
+ const allStates = this.rateLimiter.getStatus();
40
+ for (const state of allStates) {
41
+ if (state.model !== primaryModel && state.health >= 50 && state.cooldown === 0) {
42
+ healthyAlternatives.push(state.model);
43
+ }
44
+ }
45
+ // Remove duplicates and merge chains
46
+ const seen = new Set([primaryModel]);
47
+ const mergedChain = [primaryModel];
48
+ // Add healthy alternatives first (from rate limiter)
49
+ for (const m of healthyAlternatives) {
50
+ if (!seen.has(m)) {
51
+ seen.add(m);
52
+ mergedChain.push(m);
53
+ }
54
+ }
55
+ // Then add original fallback chain
56
+ for (const m of chain) {
57
+ if (!seen.has(m)) {
58
+ seen.add(m);
59
+ mergedChain.push(m);
60
+ }
61
+ }
62
+ chain = mergedChain;
63
+ }
25
64
  const attempts = [];
26
65
  let totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
27
66
  for (const model of chain) {
67
+ // Skip models that are in cooldown (if rate limiter is available)
68
+ if (this.rateLimiter) {
69
+ const check = this.rateLimiter.checkRateLimit(model);
70
+ if (!check.allowed && check.waitMs > 30000) {
71
+ // If wait is more than 30s, skip this model entirely
72
+ console.log(chalk.gray(` ⏭️ Skipping ${MODELS[model]?.name || model} (cooldown: ${Math.ceil(check.waitMs / 1000)}s)`));
73
+ attempts.push({ model, success: false, error: `In cooldown (${Math.ceil(check.waitMs / 1000)}s)` });
74
+ continue;
75
+ }
76
+ if (!check.allowed && check.suggestedModel) {
77
+ // Use the suggested alternative instead
78
+ if (!chain.includes(check.suggestedModel)) {
79
+ chain.push(check.suggestedModel);
80
+ }
81
+ continue;
82
+ }
83
+ }
28
84
  for (let retry = 0; retry <= this.config.maxRetries; retry++) {
29
85
  try {
30
86
  const modelInfo = MODELS[model];
@@ -62,7 +118,16 @@ export class FallbackChain {
62
118
  errMsg.includes('503') ||
63
119
  errMsg.includes('500'));
64
120
  if (shouldFallback) {
65
- console.log(chalk.yellow(` ⚠ ${model} failed: ${errMsg.slice(0, 80)}`));
121
+ // Record the rate limit or server error
122
+ if (this.rateLimiter) {
123
+ if (errMsg.includes('429') || errMsg.toLowerCase().includes('rate_limit')) {
124
+ this.rateLimiter.recordRateLimit(model);
125
+ }
126
+ else if (errMsg.includes('500') || errMsg.includes('503')) {
127
+ this.rateLimiter.recordServerError(model);
128
+ }
129
+ }
130
+ console.log(chalk.yellow(` ⚠ ${MODELS[model]?.name || model} failed: ${errMsg.slice(0, 80)}`));
66
131
  break; // Try next model in chain
67
132
  }
68
133
  // For other errors, retry on same model
@@ -29,9 +29,15 @@ export declare class ModelRouter {
29
29
  private availableModels;
30
30
  private currentEffort;
31
31
  private forcedModel;
32
+ private rateLimiter;
32
33
  constructor(config: RouterConfig, availableModels: Record<string, ModelInfo>);
34
+ /**
35
+ * Set the rate limiter for rate-aware routing decisions
36
+ */
37
+ setRateLimiter(rateLimiter: import('./rate-limiter.js').RateLimitManager): void;
33
38
  /**
34
39
  * Route a prompt to the best available model.
40
+ * Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
35
41
  */
36
42
  route(prompt: string, effort?: EffortLevel): RoutingDecision;
37
43
  /**
@@ -124,6 +124,7 @@ export class ModelRouter {
124
124
  availableModels;
125
125
  currentEffort = 'medium';
126
126
  forcedModel = null;
127
+ rateLimiter = null;
127
128
  // -----------------------------------------------------------------------
128
129
  // Construction
129
130
  // -----------------------------------------------------------------------
@@ -135,15 +136,53 @@ export class ModelRouter {
135
136
  // -----------------------------------------------------------------------
136
137
  // Public API
137
138
  // -----------------------------------------------------------------------
139
+ /**
140
+ * Set the rate limiter for rate-aware routing decisions
141
+ */
142
+ setRateLimiter(rateLimiter) {
143
+ this.rateLimiter = rateLimiter;
144
+ }
138
145
  /**
139
146
  * Route a prompt to the best available model.
147
+ * Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
140
148
  */
141
149
  route(prompt, effort) {
142
150
  const effectiveEffort = effort ?? this.currentEffort;
143
151
  const analysis = this.analyzePrompt(prompt);
144
152
  const estimatedTokens = this.estimateTokens(prompt);
145
- const model = this.selectModel(analysis.complexity, analysis.category, effectiveEffort);
153
+ let model = this.selectModel(analysis.complexity, analysis.category, effectiveEffort);
146
154
  const alternatives = this.getAlternatives(analysis.complexity, analysis.category);
155
+ // --- Rate-limit-aware routing ---
156
+ if (this.rateLimiter) {
157
+ const check = this.rateLimiter.checkRateLimit(model);
158
+ if (!check.allowed && check.suggestedModel) {
159
+ // Use the rate limiter's suggested alternative
160
+ const suggested = check.suggestedModel;
161
+ if (this.availableModels[suggested]) {
162
+ model = suggested;
163
+ }
164
+ else {
165
+ // The suggested model isn't in our available models, try alternatives
166
+ for (const alt of alternatives) {
167
+ const altCheck = this.rateLimiter.checkRateLimit(alt);
168
+ if (altCheck.allowed) {
169
+ model = alt;
170
+ break;
171
+ }
172
+ }
173
+ }
174
+ }
175
+ else if (!check.allowed) {
176
+ // Model is rate limited and no suggested alternative, try alternatives
177
+ for (const alt of alternatives) {
178
+ const altCheck = this.rateLimiter.checkRateLimit(alt);
179
+ if (altCheck.allowed) {
180
+ model = alt;
181
+ break;
182
+ }
183
+ }
184
+ }
185
+ }
147
186
  const reasoning = this.buildReasoning(analysis, effectiveEffort, model);
148
187
  return {
149
188
  model,
@@ -0,0 +1,162 @@
1
+ export interface RateLimiterConfig {
2
+ /** Maximum requests per minute across all models (default: 20) */
3
+ globalRpm: number;
4
+ /** Maximum requests per minute per model (default: 10) */
5
+ perModelRpm: number;
6
+ /** Base cooldown after a 429 in ms (default: 10000) */
7
+ baseCooldownMs: number;
8
+ /** Maximum cooldown in ms (default: 120000) */
9
+ maxCooldownMs: number;
10
+ /** Multiplier for consecutive 429s (default: 2.0) */
11
+ backoffMultiplier: number;
12
+ /** Number of free model rotation slots (default: 5) */
13
+ rotationPoolSize: number;
14
+ /** Enable request queuing during rate limits (default: true) */
15
+ enableQueue: boolean;
16
+ /** Maximum queue size (default: 50) */
17
+ maxQueueSize: number;
18
+ /** Maximum time a request can wait in queue in ms (default: 60000) */
19
+ maxQueueWaitMs: number;
20
+ /** Enable adaptive learning from rate limit patterns (default: true) */
21
+ adaptiveLearning: boolean;
22
+ /** Log rate limit events to console (default: true) */
23
+ verbose: boolean;
24
+ }
25
+ export interface ModelRateState {
26
+ /** Model ID */
27
+ modelId: string;
28
+ /** Current cooldown end time (0 = not cooling down) */
29
+ cooldownUntil: number;
30
+ /** Number of consecutive 429s */
31
+ consecutive429s: number;
32
+ /** Total 429s received */
33
+ total429s: number;
34
+ /** Total successful requests */
35
+ totalSuccesses: number;
36
+ /** Requests in current minute window */
37
+ requestsThisMinute: number;
38
+ /** Minute window start time */
39
+ minuteWindowStart: number;
40
+ /** Last request timestamp */
41
+ lastRequestAt: number;
42
+ /** Average response time in ms */
43
+ avgResponseMs: number;
44
+ /** Whether model is currently in rotation pool */
45
+ inRotationPool: boolean;
46
+ /** Health score 0-100 (100 = healthy) */
47
+ healthScore: number;
48
+ }
49
+ export interface RateLimitCheck {
50
+ /** Whether the request can proceed */
51
+ allowed: boolean;
52
+ /** Wait time in ms if not allowed (0 if allowed) */
53
+ waitMs: number;
54
+ /** Suggested alternative model (null if current is fine) */
55
+ suggestedModel: string | null;
56
+ /** Reason for the decision */
57
+ reason: string;
58
+ }
59
+ export interface QueuedRequest {
60
+ id: string;
61
+ model: string;
62
+ enqueuedAt: number;
63
+ resolve: (check: RateLimitCheck) => void;
64
+ reject: (error: Error) => void;
65
+ }
66
+ export declare class RateLimitManager {
67
+ private config;
68
+ private modelStates;
69
+ private globalRequestTimestamps;
70
+ private requestQueue;
71
+ private queueProcessTimer;
72
+ private rotationPool;
73
+ private currentRotationIndex;
74
+ private requestIdCounter;
75
+ private rateLimitPatterns;
76
+ constructor(config?: Partial<RateLimiterConfig>);
77
+ /**
78
+ * Check if a request can be made to a specific model.
79
+ * Returns a RateLimitCheck with the decision and optional alternative model.
80
+ */
81
+ checkRateLimit(modelId: string): RateLimitCheck;
82
+ /**
83
+ * Acquire permission to make a request. Resolves when the request can proceed.
84
+ * If queuing is enabled, waits in queue. Otherwise, rejects if rate limited.
85
+ */
86
+ acquire(modelId: string): Promise<RateLimitCheck>;
87
+ /**
88
+ * Record a successful request completion
89
+ */
90
+ recordSuccess(modelId: string, responseTimeMs: number): void;
91
+ /**
92
+ * Record a rate limit (429) response
93
+ */
94
+ recordRateLimit(modelId: string, retryAfterSeconds?: number): void;
95
+ /**
96
+ * Record a server error (5xx) - treat as potential overload
97
+ */
98
+ recordServerError(modelId: string): void;
99
+ /**
100
+ * Get the next available model from the rotation pool.
101
+ * Skips models that are in cooldown or have low health.
102
+ */
103
+ getNextRotationModel(excludeModel?: string): string | null;
104
+ /**
105
+ * Get the best available model for a given requested model.
106
+ * Tries the requested model first, then falls back through the rotation pool.
107
+ */
108
+ getBestAvailableModel(requestedModel: string): string;
109
+ /**
110
+ * Get rate limit status for all tracked models
111
+ */
112
+ getStatus(): Array<{
113
+ model: string;
114
+ name: string;
115
+ health: number;
116
+ cooldown: number;
117
+ consecutive429s: number;
118
+ total429s: number;
119
+ totalSuccesses: number;
120
+ rpm: number;
121
+ avgResponseMs: number;
122
+ }>;
123
+ /**
124
+ * Get queue status
125
+ */
126
+ getQueueStatus(): {
127
+ size: number;
128
+ oldestWaitMs: number;
129
+ };
130
+ /**
131
+ * Reset all rate limit state (useful for testing)
132
+ */
133
+ reset(): void;
134
+ /**
135
+ * Reset a specific model's rate limit state
136
+ */
137
+ resetModel(modelId: string): void;
138
+ /**
139
+ * Update configuration at runtime
140
+ */
141
+ updateConfig(updates: Partial<RateLimiterConfig>): void;
142
+ /**
143
+ * Print a formatted status table
144
+ */
145
+ printStatus(): void;
146
+ private getOrCreateState;
147
+ private recordRequestStart;
148
+ private cleanupGlobalTimestamps;
149
+ private cleanupModelTimestamps;
150
+ private findAvailableModel;
151
+ private initializeRotationPool;
152
+ private enqueueRequest;
153
+ private startQueueProcessor;
154
+ private processQueue;
155
+ private recordRateLimitPattern;
156
+ private getHealthBar;
157
+ /**
158
+ * Cleanup - call when shutting down
159
+ */
160
+ dispose(): void;
161
+ }
162
+ //# sourceMappingURL=rate-limiter.d.ts.map
@@ -0,0 +1,589 @@
1
+ // ============================================================
2
+ // NeuroCLI - Rate Limit Manager
3
+ // Intelligent rate limiting with token bucket, per-model
4
+ // cooldown tracking, model rotation, and adaptive backoff
5
+ // ============================================================
6
+ import chalk from 'chalk';
7
+ import { MODELS, getFreeModelsWithTools } from '../api/models.js';
8
+ // ---------------------------------------------------------------------------
9
+ // RateLimitManager
10
+ // ---------------------------------------------------------------------------
11
+ export class RateLimitManager {
12
+ config;
13
+ modelStates = new Map();
14
+ globalRequestTimestamps = [];
15
+ requestQueue = [];
16
+ queueProcessTimer = null;
17
+ rotationPool = [];
18
+ currentRotationIndex = 0;
19
+ requestIdCounter = 0;
20
+ // Adaptive learning data
21
+ rateLimitPatterns = new Map();
22
+ constructor(config) {
23
+ this.config = {
24
+ globalRpm: 20,
25
+ perModelRpm: 10,
26
+ baseCooldownMs: 10000,
27
+ maxCooldownMs: 120000,
28
+ backoffMultiplier: 2.0,
29
+ rotationPoolSize: 5,
30
+ enableQueue: true,
31
+ maxQueueSize: 50,
32
+ maxQueueWaitMs: 60000,
33
+ adaptiveLearning: true,
34
+ verbose: true,
35
+ ...config,
36
+ };
37
+ // Initialize rotation pool with free models that support tools
38
+ this.initializeRotationPool();
39
+ // Start queue processor
40
+ this.startQueueProcessor();
41
+ }
42
+ // =========================================================================
43
+ // Pre-Request Checking
44
+ // =========================================================================
45
+ /**
46
+ * Check if a request can be made to a specific model.
47
+ * Returns a RateLimitCheck with the decision and optional alternative model.
48
+ */
49
+ checkRateLimit(modelId) {
50
+ const now = Date.now();
51
+ // 1. Check global RPM
52
+ this.cleanupGlobalTimestamps(now);
53
+ if (this.globalRequestTimestamps.length >= this.config.globalRpm) {
54
+ const oldestInWindow = this.globalRequestTimestamps[0];
55
+ const waitMs = Math.max(0, 60000 - (now - oldestInWindow));
56
+ const alternative = this.findAvailableModel(modelId);
57
+ if (alternative) {
58
+ return {
59
+ allowed: false,
60
+ waitMs: 0,
61
+ suggestedModel: alternative,
62
+ reason: `Global RPM limit reached (${this.config.globalRpm}/min). Suggesting alternative: ${MODELS[alternative]?.name || alternative}`,
63
+ };
64
+ }
65
+ return {
66
+ allowed: false,
67
+ waitMs,
68
+ suggestedModel: null,
69
+ reason: `Global RPM limit reached (${this.config.globalRpm}/min). Wait ${Math.ceil(waitMs / 1000)}s or try a different model.`,
70
+ };
71
+ }
72
+ // 2. Check per-model cooldown
73
+ const state = this.getOrCreateState(modelId);
74
+ if (state.cooldownUntil > now) {
75
+ const waitMs = state.cooldownUntil - now;
76
+ const alternative = this.findAvailableModel(modelId);
77
+ if (alternative) {
78
+ return {
79
+ allowed: false,
80
+ waitMs: 0,
81
+ suggestedModel: alternative,
82
+ reason: `${MODELS[modelId]?.name || modelId} is in cooldown (${Math.ceil(waitMs / 1000)}s left). Rotating to ${MODELS[alternative]?.name || alternative}`,
83
+ };
84
+ }
85
+ return {
86
+ allowed: false,
87
+ waitMs,
88
+ suggestedModel: null,
89
+ reason: `${MODELS[modelId]?.name || modelId} is in cooldown. Wait ${Math.ceil(waitMs / 1000)}s.`,
90
+ };
91
+ }
92
+ // 3. Check per-model RPM
93
+ this.cleanupModelTimestamps(state, now);
94
+ if (state.requestsThisMinute >= this.config.perModelRpm) {
95
+ const alternative = this.findAvailableModel(modelId);
96
+ if (alternative) {
97
+ return {
98
+ allowed: false,
99
+ waitMs: 0,
100
+ suggestedModel: alternative,
101
+ reason: `${MODELS[modelId]?.name || modelId} RPM limit (${this.config.perModelRpm}/min). Rotating to ${MODELS[alternative]?.name || alternative}`,
102
+ };
103
+ }
104
+ const minuteElapsed = now - state.minuteWindowStart;
105
+ const waitMs = Math.max(0, 60000 - minuteElapsed);
106
+ return {
107
+ allowed: false,
108
+ waitMs,
109
+ suggestedModel: null,
110
+ reason: `${MODELS[modelId]?.name || modelId} RPM limit reached. Wait ${Math.ceil(waitMs / 1000)}s.`,
111
+ };
112
+ }
113
+ // 4. Check health score - if model is unhealthy, suggest alternative
114
+ if (state.healthScore < 30 && state.consecutive429s > 3) {
115
+ const alternative = this.findAvailableModel(modelId);
116
+ if (alternative) {
117
+ return {
118
+ allowed: true,
119
+ waitMs: 0,
120
+ suggestedModel: alternative,
121
+ reason: `${MODELS[modelId]?.name || modelId} health is low (${state.healthScore}/100). Suggesting ${MODELS[alternative]?.name || alternative} for better reliability.`,
122
+ };
123
+ }
124
+ }
125
+ // All checks passed
126
+ return {
127
+ allowed: true,
128
+ waitMs: 0,
129
+ suggestedModel: null,
130
+ reason: 'OK',
131
+ };
132
+ }
133
+ /**
134
+ * Acquire permission to make a request. Resolves when the request can proceed.
135
+ * If queuing is enabled, waits in queue. Otherwise, rejects if rate limited.
136
+ */
137
+ async acquire(modelId) {
138
+ const check = this.checkRateLimit(modelId);
139
+ if (check.allowed) {
140
+ // If there's a suggested model due to low health, we still allow but suggest
141
+ this.recordRequestStart(modelId);
142
+ return check;
143
+ }
144
+ // If there's an alternative model suggested, switch to it
145
+ if (check.suggestedModel) {
146
+ const altCheck = this.checkRateLimit(check.suggestedModel);
147
+ if (altCheck.allowed) {
148
+ this.recordRequestStart(check.suggestedModel);
149
+ if (this.config.verbose) {
150
+ console.log(chalk.cyan(` 🔄 Rate limiter: switching ${MODELS[modelId]?.name || modelId} → ${MODELS[check.suggestedModel]?.name || check.suggestedModel}`));
151
+ }
152
+ return { ...altCheck, suggestedModel: check.suggestedModel, reason: check.reason };
153
+ }
154
+ }
155
+ // If queuing is enabled, queue the request
156
+ if (this.config.enableQueue && check.waitMs > 0) {
157
+ return this.enqueueRequest(modelId);
158
+ }
159
+ // Can't proceed, return the check with wait info
160
+ if (this.config.verbose) {
161
+ console.log(chalk.yellow(` ⏳ Rate limited: ${check.reason}`));
162
+ }
163
+ return check;
164
+ }
165
+ /**
166
+ * Record a successful request completion
167
+ */
168
+ recordSuccess(modelId, responseTimeMs) {
169
+ const state = this.getOrCreateState(modelId);
170
+ state.consecutive429s = 0;
171
+ state.totalSuccesses++;
172
+ state.lastRequestAt = Date.now();
173
+ // Update average response time (exponential moving average)
174
+ if (state.avgResponseMs === 0) {
175
+ state.avgResponseMs = responseTimeMs;
176
+ }
177
+ else {
178
+ state.avgResponseMs = state.avgResponseMs * 0.8 + responseTimeMs * 0.2;
179
+ }
180
+ // Recover health score
181
+ state.healthScore = Math.min(100, state.healthScore + 5);
182
+ }
183
+ /**
184
+ * Record a rate limit (429) response
185
+ */
186
+ recordRateLimit(modelId, retryAfterSeconds) {
187
+ const state = this.getOrCreateState(modelId);
188
+ const now = Date.now();
189
+ state.consecutive429s++;
190
+ state.total429s++;
191
+ state.lastRequestAt = now;
192
+ // Calculate cooldown duration
193
+ let cooldownMs;
194
+ if (retryAfterSeconds && retryAfterSeconds > 0) {
195
+ // Respect server-provided retry-after
196
+ cooldownMs = retryAfterSeconds * 1000;
197
+ }
198
+ else {
199
+ // Exponential backoff based on consecutive 429s
200
+ cooldownMs = Math.min(this.config.baseCooldownMs * Math.pow(this.config.backoffMultiplier, state.consecutive429s - 1), this.config.maxCooldownMs);
201
+ }
202
+ state.cooldownUntil = now + cooldownMs;
203
+ // Decrease health score
204
+ state.healthScore = Math.max(0, state.healthScore - (10 + state.consecutive429s * 5));
205
+ // Adaptive learning: record the pattern
206
+ if (this.config.adaptiveLearning) {
207
+ this.recordRateLimitPattern(modelId, cooldownMs);
208
+ }
209
+ if (this.config.verbose) {
210
+ console.log(chalk.yellow(` 🚫 Rate limited on ${MODELS[modelId]?.name || modelId}: ` +
211
+ `cooldown ${Math.ceil(cooldownMs / 1000)}s ` +
212
+ `(consecutive: ${state.consecutive429s}, health: ${state.healthScore}/100)`));
213
+ }
214
+ }
215
+ /**
216
+ * Record a server error (5xx) - treat as potential overload
217
+ */
218
+ recordServerError(modelId) {
219
+ const state = this.getOrCreateState(modelId);
220
+ const now = Date.now();
221
+ // Apply a shorter cooldown for server errors
222
+ state.cooldownUntil = now + this.config.baseCooldownMs / 2;
223
+ state.healthScore = Math.max(0, state.healthScore - 15);
224
+ if (this.config.verbose) {
225
+ console.log(chalk.yellow(` ⚠️ Server error on ${MODELS[modelId]?.name || modelId}: ` +
226
+ `cooldown ${Math.ceil(this.config.baseCooldownMs / 2000)}s, health: ${state.healthScore}/100`));
227
+ }
228
+ }
229
+ // =========================================================================
230
+ // Model Rotation
231
+ // =========================================================================
232
+ /**
233
+ * Get the next available model from the rotation pool.
234
+ * Skips models that are in cooldown or have low health.
235
+ */
236
+ getNextRotationModel(excludeModel) {
237
+ const now = Date.now();
238
+ const poolSize = this.rotationPool.length;
239
+ if (poolSize === 0)
240
+ return null;
241
+ // Try each model in the pool, starting from current index
242
+ for (let i = 0; i < poolSize; i++) {
243
+ const idx = (this.currentRotationIndex + i) % poolSize;
244
+ const model = this.rotationPool[idx];
245
+ if (model === excludeModel)
246
+ continue;
247
+ const state = this.getOrCreateState(model);
248
+ // Skip if in cooldown
249
+ if (state.cooldownUntil > now)
250
+ continue;
251
+ // Skip if health is too low
252
+ if (state.healthScore < 20)
253
+ continue;
254
+ // Move rotation index forward
255
+ this.currentRotationIndex = (idx + 1) % poolSize;
256
+ return model;
257
+ }
258
+ // All models in pool are rate limited or unhealthy
259
+ // Find the one with the shortest wait time
260
+ let bestModel = null;
261
+ let shortestWait = Infinity;
262
+ for (const model of this.rotationPool) {
263
+ if (model === excludeModel)
264
+ continue;
265
+ const state = this.getOrCreateState(model);
266
+ const waitMs = Math.max(0, state.cooldownUntil - now);
267
+ if (waitMs < shortestWait) {
268
+ shortestWait = waitMs;
269
+ bestModel = model;
270
+ }
271
+ }
272
+ return bestModel;
273
+ }
274
+ /**
275
+ * Get the best available model for a given requested model.
276
+ * Tries the requested model first, then falls back through the rotation pool.
277
+ */
278
+ getBestAvailableModel(requestedModel) {
279
+ // Check if requested model is available
280
+ const check = this.checkRateLimit(requestedModel);
281
+ if (check.allowed)
282
+ return requestedModel;
283
+ // Try rotation pool
284
+ const alternative = this.findAvailableModel(requestedModel);
285
+ if (alternative)
286
+ return alternative;
287
+ // Last resort: return requested model (will wait)
288
+ return requestedModel;
289
+ }
290
+ // =========================================================================
291
+ // Status & Diagnostics
292
+ // =========================================================================
293
+ /**
294
+ * Get rate limit status for all tracked models
295
+ */
296
+ getStatus() {
297
+ const now = Date.now();
298
+ const result = [];
299
+ for (const [modelId, state] of this.modelStates) {
300
+ this.cleanupModelTimestamps(state, now);
301
+ result.push({
302
+ model: modelId,
303
+ name: MODELS[modelId]?.name || modelId,
304
+ health: state.healthScore,
305
+ cooldown: Math.max(0, Math.ceil((state.cooldownUntil - now) / 1000)),
306
+ consecutive429s: state.consecutive429s,
307
+ total429s: state.total429s,
308
+ totalSuccesses: state.totalSuccesses,
309
+ rpm: state.requestsThisMinute,
310
+ avgResponseMs: Math.round(state.avgResponseMs),
311
+ });
312
+ }
313
+ return result.sort((a, b) => b.health - a.health);
314
+ }
315
+ /**
316
+ * Get queue status
317
+ */
318
+ getQueueStatus() {
319
+ if (this.requestQueue.length === 0)
320
+ return { size: 0, oldestWaitMs: 0 };
321
+ return {
322
+ size: this.requestQueue.length,
323
+ oldestWaitMs: Date.now() - this.requestQueue[0].enqueuedAt,
324
+ };
325
+ }
326
+ /**
327
+ * Reset all rate limit state (useful for testing)
328
+ */
329
+ reset() {
330
+ for (const state of this.modelStates.values()) {
331
+ state.cooldownUntil = 0;
332
+ state.consecutive429s = 0;
333
+ state.healthScore = 100;
334
+ state.requestsThisMinute = 0;
335
+ }
336
+ this.globalRequestTimestamps = [];
337
+ this.requestQueue.forEach(r => r.reject(new Error('Rate limiter reset')));
338
+ this.requestQueue = [];
339
+ if (this.config.verbose) {
340
+ console.log(chalk.gray(' ↻ Rate limiter state reset'));
341
+ }
342
+ }
343
+ /**
344
+ * Reset a specific model's rate limit state
345
+ */
346
+ resetModel(modelId) {
347
+ const state = this.modelStates.get(modelId);
348
+ if (state) {
349
+ state.cooldownUntil = 0;
350
+ state.consecutive429s = 0;
351
+ state.healthScore = 100;
352
+ state.requestsThisMinute = 0;
353
+ }
354
+ }
355
+ /**
356
+ * Update configuration at runtime
357
+ */
358
+ updateConfig(updates) {
359
+ Object.assign(this.config, updates);
360
+ if (updates.rotationPoolSize !== undefined) {
361
+ this.initializeRotationPool();
362
+ }
363
+ }
364
+ /**
365
+ * Print a formatted status table
366
+ */
367
+ printStatus() {
368
+ const status = this.getStatus();
369
+ const now = Date.now();
370
+ console.log(chalk.bold('\n📊 Rate Limiter Status'));
371
+ console.log(chalk.gray('─'.repeat(80)));
372
+ for (const s of status) {
373
+ const healthBar = this.getHealthBar(s.health);
374
+ const cooldownStr = s.cooldown > 0 ? chalk.red(`${s.cooldown}s`) : chalk.green('✓');
375
+ const rpmStr = `${s.rpm}/${this.config.perModelRpm}`;
376
+ console.log(` ${s.name.padEnd(35)} ${healthBar} ${cooldownStr.padEnd(8)} ` +
377
+ `429s: ${chalk.yellow(String(s.total429s).padEnd(4))} ` +
378
+ `OK: ${chalk.green(String(s.totalSuccesses).padEnd(4))} ` +
379
+ `RPM: ${rpmStr.padEnd(6)} ` +
380
+ `avg: ${s.avgResponseMs}ms`);
381
+ }
382
+ const queueStatus = this.getQueueStatus();
383
+ if (queueStatus.size > 0) {
384
+ console.log(chalk.yellow(`\n Queue: ${queueStatus.size} requests waiting (oldest: ${Math.ceil(queueStatus.oldestWaitMs / 1000)}s)`));
385
+ }
386
+ console.log(chalk.gray('─'.repeat(80)));
387
+ }
388
+ // =========================================================================
389
+ // Private Methods
390
+ // =========================================================================
391
+ getOrCreateState(modelId) {
392
+ if (!this.modelStates.has(modelId)) {
393
+ this.modelStates.set(modelId, {
394
+ modelId,
395
+ cooldownUntil: 0,
396
+ consecutive429s: 0,
397
+ total429s: 0,
398
+ totalSuccesses: 0,
399
+ requestsThisMinute: 0,
400
+ minuteWindowStart: Date.now(),
401
+ lastRequestAt: 0,
402
+ avgResponseMs: 0,
403
+ inRotationPool: this.rotationPool.includes(modelId),
404
+ healthScore: 100,
405
+ });
406
+ }
407
+ return this.modelStates.get(modelId);
408
+ }
409
+ recordRequestStart(modelId) {
410
+ const now = Date.now();
411
+ const state = this.getOrCreateState(modelId);
412
+ // Update per-model minute window
413
+ this.cleanupModelTimestamps(state, now);
414
+ state.requestsThisMinute++;
415
+ state.lastRequestAt = now;
416
+ // Update global timestamps
417
+ this.cleanupGlobalTimestamps(now);
418
+ this.globalRequestTimestamps.push(now);
419
+ }
420
+ cleanupGlobalTimestamps(now) {
421
+ const oneMinuteAgo = now - 60000;
422
+ while (this.globalRequestTimestamps.length > 0 && this.globalRequestTimestamps[0] < oneMinuteAgo) {
423
+ this.globalRequestTimestamps.shift();
424
+ }
425
+ }
426
+ cleanupModelTimestamps(state, now) {
427
+ if (now - state.minuteWindowStart >= 60000) {
428
+ state.requestsThisMinute = 0;
429
+ state.minuteWindowStart = now;
430
+ }
431
+ }
432
+ findAvailableModel(excludeModel) {
433
+ const now = Date.now();
434
+ // First, try the rotation pool
435
+ for (const model of this.rotationPool) {
436
+ if (model === excludeModel)
437
+ continue;
438
+ const state = this.getOrCreateState(model);
439
+ if (state.cooldownUntil <= now && state.healthScore >= 20) {
440
+ this.cleanupModelTimestamps(state, now);
441
+ if (state.requestsThisMinute < this.config.perModelRpm) {
442
+ return model;
443
+ }
444
+ }
445
+ }
446
+ // Then try any tracked model that's available
447
+ for (const [modelId, state] of this.modelStates) {
448
+ if (modelId === excludeModel)
449
+ continue;
450
+ if (state.cooldownUntil <= now && state.healthScore >= 20) {
451
+ this.cleanupModelTimestamps(state, now);
452
+ if (state.requestsThisMinute < this.config.perModelRpm) {
453
+ return modelId;
454
+ }
455
+ }
456
+ }
457
+ return null;
458
+ }
459
+ initializeRotationPool() {
460
+ const freeModels = getFreeModelsWithTools();
461
+ // Sort by context window (larger is better for coding)
462
+ const sorted = freeModels.sort((a, b) => b.contextWindow - a.contextWindow);
463
+ // Take top N models for rotation
464
+ this.rotationPool = sorted
465
+ .slice(0, Math.min(this.config.rotationPoolSize, sorted.length))
466
+ .map(m => m.id);
467
+ // Mark all as in rotation pool
468
+ for (const modelId of this.rotationPool) {
469
+ const state = this.getOrCreateState(modelId);
470
+ state.inRotationPool = true;
471
+ }
472
+ if (this.config.verbose) {
473
+ console.log(chalk.gray(` 🔄 Rate limiter rotation pool: ${this.rotationPool.map(m => MODELS[m]?.name || m).join(', ')}`));
474
+ }
475
+ }
476
+ enqueueRequest(modelId) {
477
+ return new Promise((resolve, reject) => {
478
+ if (this.requestQueue.length >= this.config.maxQueueSize) {
479
+ reject(new Error(`Rate limiter queue full (${this.config.maxQueueSize}). Try again later.`));
480
+ return;
481
+ }
482
+ const id = `req_${++this.requestIdCounter}`;
483
+ const queued = {
484
+ id,
485
+ model: modelId,
486
+ enqueuedAt: Date.now(),
487
+ resolve,
488
+ reject,
489
+ };
490
+ this.requestQueue.push(queued);
491
+ if (this.config.verbose) {
492
+ console.log(chalk.gray(` ⏳ Request queued for ${MODELS[modelId]?.name || modelId} (queue: ${this.requestQueue.length})`));
493
+ }
494
+ // Set a timeout to reject if waiting too long
495
+ setTimeout(() => {
496
+ const idx = this.requestQueue.findIndex(r => r.id === id);
497
+ if (idx !== -1) {
498
+ this.requestQueue.splice(idx, 1);
499
+ reject(new Error(`Request timed out waiting in rate limiter queue (${this.config.maxQueueWaitMs / 1000}s)`));
500
+ }
501
+ }, this.config.maxQueueWaitMs);
502
+ });
503
+ }
504
+ startQueueProcessor() {
505
+ // Process queue every 2 seconds
506
+ this.queueProcessTimer = setInterval(() => {
507
+ this.processQueue();
508
+ }, 2000);
509
+ }
510
+ processQueue() {
511
+ if (this.requestQueue.length === 0)
512
+ return;
513
+ const now = Date.now();
514
+ const toProcess = [];
515
+ // Find requests that can now proceed
516
+ for (let i = this.requestQueue.length - 1; i >= 0; i--) {
517
+ const req = this.requestQueue[i];
518
+ // Check if request has been waiting too long
519
+ if (now - req.enqueuedAt > this.config.maxQueueWaitMs) {
520
+ this.requestQueue.splice(i, 1);
521
+ req.reject(new Error('Request timed out in rate limiter queue'));
522
+ continue;
523
+ }
524
+ const check = this.checkRateLimit(req.model);
525
+ if (check.allowed) {
526
+ toProcess.push(req);
527
+ this.requestQueue.splice(i, 1);
528
+ }
529
+ else if (check.suggestedModel) {
530
+ // Try alternative model
531
+ const altCheck = this.checkRateLimit(check.suggestedModel);
532
+ if (altCheck.allowed) {
533
+ this.recordRequestStart(check.suggestedModel);
534
+ req.resolve({ ...altCheck, suggestedModel: check.suggestedModel, reason: check.reason });
535
+ this.requestQueue.splice(i, 1);
536
+ }
537
+ }
538
+ }
539
+ // Process approved requests
540
+ for (const req of toProcess) {
541
+ this.recordRequestStart(req.model);
542
+ req.resolve({ allowed: true, waitMs: 0, suggestedModel: null, reason: 'Queued request approved' });
543
+ }
544
+ }
545
+ recordRateLimitPattern(modelId, cooldownMs) {
546
+ const hour = new Date().getHours();
547
+ let patterns = this.rateLimitPatterns.get(modelId);
548
+ if (!patterns) {
549
+ patterns = [];
550
+ this.rateLimitPatterns.set(modelId, patterns);
551
+ }
552
+ let existing = patterns.find(p => p.hourOfDay === hour);
553
+ if (!existing) {
554
+ existing = { hourOfDay: hour, avgCooldownMs: cooldownMs, count: 1 };
555
+ patterns.push(existing);
556
+ }
557
+ else {
558
+ existing.avgCooldownMs = (existing.avgCooldownMs * existing.count + cooldownMs) / (existing.count + 1);
559
+ existing.count++;
560
+ }
561
+ }
562
+ getHealthBar(health) {
563
+ const filled = Math.round(health / 10);
564
+ const empty = 10 - filled;
565
+ let color;
566
+ if (health >= 70)
567
+ color = chalk.green;
568
+ else if (health >= 40)
569
+ color = chalk.yellow;
570
+ else
571
+ color = chalk.red;
572
+ return color('█'.repeat(filled) + '░'.repeat(empty));
573
+ }
574
+ /**
575
+ * Cleanup - call when shutting down
576
+ */
577
+ dispose() {
578
+ if (this.queueProcessTimer) {
579
+ clearInterval(this.queueProcessTimer);
580
+ this.queueProcessTimer = null;
581
+ }
582
+ // Reject all queued requests
583
+ for (const req of this.requestQueue) {
584
+ req.reject(new Error('Rate limiter shutting down'));
585
+ }
586
+ this.requestQueue = [];
587
+ }
588
+ }
589
+ //# sourceMappingURL=rate-limiter.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "neuro-cli",
3
- "version": "5.0.0",
3
+ "version": "5.0.1",
4
4
  "description": "Advanced AI-powered terminal coding assistant with multi-agent orchestration, sub-agent spawning, ACP protocol, OS-level sandboxing, spec-driven development, smart monitoring, multi-model routing, MCP Apps, auto-updater, and 23+ free models",
5
5
  "main": "dist/index.js",
6
6
  "bin": {