neuro-cli 4.3.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/base.d.ts +21 -2
- package/dist/agents/base.js +174 -31
- package/dist/agents/orchestrator.d.ts +3 -1
- package/dist/agents/orchestrator.js +32 -16
- package/dist/api/openrouter.d.ts +7 -1
- package/dist/api/openrouter.js +152 -37
- package/dist/commands/commands.js +11 -0
- package/dist/config/config.js +17 -17
- package/dist/core/engine.d.ts +8 -3
- package/dist/core/engine.js +79 -57
- package/dist/core/fallback.d.ts +7 -1
- package/dist/core/fallback.js +68 -3
- package/dist/core/model-router.d.ts +6 -0
- package/dist/core/model-router.js +43 -4
- package/dist/core/rate-limiter.d.ts +162 -0
- package/dist/core/rate-limiter.js +589 -0
- package/dist/index.js +2 -2
- package/dist/ui/renderer.d.ts +1 -1
- package/dist/ui/renderer.js +3 -2
- package/package.json +1 -1
package/dist/core/engine.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// ============================================================
|
|
2
|
-
// NeuroCLI - NeuroEngine
|
|
2
|
+
// NeuroCLI - NeuroEngine v5.0 (Claude Code-style Agentic Loop)
|
|
3
3
|
// The main engine that ties everything together
|
|
4
4
|
// Now with: Sandbox, Plugin SDK, Enhanced MCP, Enhanced Approval,
|
|
5
5
|
// Model Router, Prompt Cache, Undo/Redo, Output Styles,
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
// Spec-Driven Development, LLM Evaluator Hooks, MCP Apps,
|
|
16
16
|
// Multi-Model Orchestrator, Smart Monitor, Outcome Grading,
|
|
17
17
|
// Observability (OTLP), Auto-Compact, Terminal UX,
|
|
18
|
-
// Multi-Session, Git Worktree, Auto-Updater
|
|
18
|
+
// Multi-Session, Git Worktree, Auto-Updater,
|
|
19
|
+
// Rate Limit Manager
|
|
19
20
|
// ============================================================
|
|
20
21
|
import { join } from 'path';
|
|
21
22
|
import { homedir } from 'os';
|
|
@@ -79,6 +80,7 @@ import { TerminalUX } from './terminal-ux.js';
|
|
|
79
80
|
import { MultiSessionManager } from './multi-session.js';
|
|
80
81
|
import { GitWorktreeManager } from './git-worktree.js';
|
|
81
82
|
import { AutoUpdater } from './updater.js';
|
|
83
|
+
import { RateLimitManager } from './rate-limiter.js';
|
|
82
84
|
export class NeuroEngine {
|
|
83
85
|
config;
|
|
84
86
|
client;
|
|
@@ -143,11 +145,27 @@ export class NeuroEngine {
|
|
|
143
145
|
multiSession;
|
|
144
146
|
gitWorktree;
|
|
145
147
|
updater;
|
|
148
|
+
rateLimiter;
|
|
146
149
|
autoApproveSet;
|
|
147
150
|
requireApprovalSet;
|
|
148
151
|
constructor(config) {
|
|
149
152
|
this.config = config;
|
|
153
|
+
// Rate Limit Manager - initialized FIRST so it's available for all systems
|
|
154
|
+
this.rateLimiter = new RateLimitManager({
|
|
155
|
+
globalRpm: 20,
|
|
156
|
+
perModelRpm: 10,
|
|
157
|
+
baseCooldownMs: 10000,
|
|
158
|
+
maxCooldownMs: 120000,
|
|
159
|
+
backoffMultiplier: 2.0,
|
|
160
|
+
rotationPoolSize: 5,
|
|
161
|
+
enableQueue: true,
|
|
162
|
+
maxQueueSize: 50,
|
|
163
|
+
maxQueueWaitMs: 60000,
|
|
164
|
+
adaptiveLearning: true,
|
|
165
|
+
verbose: true,
|
|
166
|
+
});
|
|
150
167
|
this.client = new OpenRouterClient(config.apiKey, config.baseUrl);
|
|
168
|
+
this.client.setRateLimiter(this.rateLimiter);
|
|
151
169
|
this.registry = registerAllTools(new ToolRegistry());
|
|
152
170
|
this.contextManager = new ContextManager(config.defaultModel, config.context.maxTokens);
|
|
153
171
|
this.sessionManager = new SessionManager();
|
|
@@ -166,6 +184,7 @@ export class NeuroEngine {
|
|
|
166
184
|
return false;
|
|
167
185
|
});
|
|
168
186
|
this.fallback = new FallbackChain(this.client, config.fallbackChain);
|
|
187
|
+
this.fallback.setRateLimiter(this.rateLimiter);
|
|
169
188
|
// Sandbox system
|
|
170
189
|
this.sandbox = new Sandbox(config.sandbox);
|
|
171
190
|
// Plugin system
|
|
@@ -196,6 +215,7 @@ export class NeuroEngine {
|
|
|
196
215
|
categoryOverrides: {},
|
|
197
216
|
maxTokenBudget: config.context.maxTokens,
|
|
198
217
|
}, Object.fromEntries(Object.entries(MODELS).map(([id, m]) => [id, { name: m.name, contextWindow: m.contextWindow, maxOutput: m.maxOutput }])));
|
|
218
|
+
this.modelRouter.setRateLimiter(this.rateLimiter);
|
|
199
219
|
// Output styles
|
|
200
220
|
this.styleManager = new StyleManager(process.cwd());
|
|
201
221
|
// Extended thinking
|
|
@@ -483,7 +503,7 @@ export class NeuroEngine {
|
|
|
483
503
|
this.gitWorktree = new GitWorktreeManager(process.cwd());
|
|
484
504
|
// Auto-Updater
|
|
485
505
|
this.updater = new AutoUpdater({
|
|
486
|
-
currentVersion: '
|
|
506
|
+
currentVersion: '5.0.0',
|
|
487
507
|
autoCheck: true,
|
|
488
508
|
autoUpdate: false,
|
|
489
509
|
});
|
|
@@ -523,6 +543,7 @@ Key responsibilities:
|
|
|
523
543
|
- Manage dependencies between sub-tasks
|
|
524
544
|
- Synthesize results from multiple agents
|
|
525
545
|
- Handle errors and re-plan if needed
|
|
546
|
+
- IMPORTANT: Keep working until the ENTIRE task is COMPLETE
|
|
526
547
|
|
|
527
548
|
Always consider the strengths of each agent when delegating:
|
|
528
549
|
- Planner: For task decomposition and architecture decisions
|
|
@@ -537,7 +558,7 @@ Always consider the strengths of each agent when delegating:
|
|
|
537
558
|
temperature: 0.7,
|
|
538
559
|
maxTokens: 4096,
|
|
539
560
|
tools: [],
|
|
540
|
-
maxIterations:
|
|
561
|
+
maxIterations: 0, // 0 = no limit, run until done
|
|
541
562
|
};
|
|
542
563
|
this.orchestrator = new Orchestrator(orchestratorConfig, this.client, this.registry, process.cwd(), this.sessionManager.getCurrent()?.id || 'default');
|
|
543
564
|
// Register all agents with orchestrator
|
|
@@ -634,7 +655,9 @@ Always consider the strengths of each agent when delegating:
|
|
|
634
655
|
}
|
|
635
656
|
}
|
|
636
657
|
/**
|
|
637
|
-
* Process a user message
|
|
658
|
+
* Process a user message — Claude Code style
|
|
659
|
+
* Every task is sent to the best agent which runs until completion.
|
|
660
|
+
* No arbitrary early termination — the agent keeps working until done.
|
|
638
661
|
*/
|
|
639
662
|
async processMessage(message, mode = 'auto', targetAgent) {
|
|
640
663
|
// Check spending limit via spending monitor
|
|
@@ -687,7 +710,7 @@ Always consider the strengths of each agent when delegating:
|
|
|
687
710
|
const skillAdditions = this.skillSystem.getSystemPromptAdditions();
|
|
688
711
|
const styleAddition = this.styleManager.getSystemPromptAddition();
|
|
689
712
|
const thinkingAddition = this.extendedThinking.getSystemPromptAddition();
|
|
690
|
-
// Build UI callbacks
|
|
713
|
+
// Build UI callbacks — Claude Code style
|
|
691
714
|
const callbacks = {
|
|
692
715
|
onThinking: (thinking) => this.ui.thinking(thinking),
|
|
693
716
|
onToken: (token) => this.ui.streamingToken(token),
|
|
@@ -702,28 +725,32 @@ Always consider the strengths of each agent when delegating:
|
|
|
702
725
|
return;
|
|
703
726
|
}
|
|
704
727
|
this.ui.toolCall(name, args);
|
|
705
|
-
// Record in undo/redo for file modification tools
|
|
706
|
-
if (['write_file', 'edit_file', 'apply_diff', 'delete_file'].includes(name)) {
|
|
707
|
-
// The undo/redo push will happen in the tool execution result handler
|
|
708
|
-
}
|
|
709
728
|
},
|
|
710
729
|
onToolResult: (name, result, isError) => this.ui.toolResult(name, result, isError),
|
|
711
730
|
onApprovalNeeded: async (name, args, risk) => {
|
|
712
731
|
return this.handleApproval(name, args, risk);
|
|
713
732
|
},
|
|
714
|
-
onIteration: (i,
|
|
715
|
-
|
|
733
|
+
onIteration: (i, _max) => {
|
|
734
|
+
// Claude Code style: show iteration count without a ceiling
|
|
735
|
+
this.ui.agentActivity(this.config.defaultModel, 'working', `step ${i}`);
|
|
736
|
+
},
|
|
737
|
+
onCycleStart: (cycle, summary) => {
|
|
738
|
+
this.ui.agentActivity(this.config.defaultModel, 'working', `cycle ${cycle}: ${summary}`);
|
|
739
|
+
},
|
|
740
|
+
onTaskComplete: (reason) => {
|
|
741
|
+
this.ui.agentActivity(this.config.defaultModel, 'done', reason);
|
|
716
742
|
},
|
|
717
743
|
};
|
|
718
744
|
let result;
|
|
719
745
|
const activeModel = routeDecision?.model || this.config.defaultModel;
|
|
746
|
+
// === Claude Code-style routing: always send to the best agent and let it run until done ===
|
|
720
747
|
if (mode === 'direct' && targetAgent) {
|
|
748
|
+
// Direct mode: use specified agent
|
|
721
749
|
const agent = this.agents.get(targetAgent);
|
|
722
750
|
if (!agent) {
|
|
723
751
|
this.ui.error(`Agent not found: ${targetAgent}`);
|
|
724
752
|
return { content: '', usage: { inputTokens: 0, outputTokens: 0, cost: 0 } };
|
|
725
753
|
}
|
|
726
|
-
// Override agent model if defaultModel differs from agent's model
|
|
727
754
|
const originalModel = agent.configModel;
|
|
728
755
|
if (this.config.defaultModel !== originalModel) {
|
|
729
756
|
agent.configModel = this.config.defaultModel;
|
|
@@ -735,10 +762,10 @@ Always consider the strengths of each agent when delegating:
|
|
|
735
762
|
finally {
|
|
736
763
|
this.ui.endStreaming();
|
|
737
764
|
}
|
|
738
|
-
// Restore original model
|
|
739
765
|
agent.configModel = originalModel;
|
|
740
766
|
}
|
|
741
767
|
else if (mode === 'agent') {
|
|
768
|
+
// Orchestration mode: multi-agent with re-planning
|
|
742
769
|
const orchestrateResult = await this.orchestrator.orchestrate(message, callbacks);
|
|
743
770
|
result = {
|
|
744
771
|
content: orchestrateResult.content,
|
|
@@ -749,58 +776,52 @@ Always consider the strengths of each agent when delegating:
|
|
|
749
776
|
};
|
|
750
777
|
}
|
|
751
778
|
else {
|
|
752
|
-
|
|
779
|
+
// Auto mode — Claude Code style: route to best agent, let it run until done
|
|
753
780
|
const category = routeDecision?.category || this.modelRouter.getCategory(message);
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
781
|
+
const targetAgentName = this.selectAgentForCategory(category);
|
|
782
|
+
const agent = this.agents.get(targetAgentName);
|
|
783
|
+
if (agent) {
|
|
784
|
+
// Override agent model with routed model if different
|
|
785
|
+
const originalModel = agent.configModel;
|
|
786
|
+
if (activeModel !== originalModel) {
|
|
787
|
+
agent.configModel = activeModel;
|
|
788
|
+
}
|
|
789
|
+
this.ui.info(`Using ${targetAgentName} agent (${category} task) — will run until complete`);
|
|
790
|
+
this.ui.startStreaming();
|
|
791
|
+
try {
|
|
792
|
+
result = await agent.run(message, callbacks);
|
|
793
|
+
}
|
|
794
|
+
finally {
|
|
795
|
+
this.ui.endStreaming();
|
|
796
|
+
}
|
|
797
|
+
// Restore original model
|
|
798
|
+
agent.configModel = originalModel;
|
|
799
|
+
}
|
|
800
|
+
else {
|
|
801
|
+
// Fallback: try Coder, then orchestrator
|
|
802
|
+
const coderAgent = this.agents.get('Coder');
|
|
803
|
+
if (coderAgent) {
|
|
804
|
+
this.ui.info(`Using Coder agent — will run until complete`);
|
|
760
805
|
this.ui.startStreaming();
|
|
761
806
|
try {
|
|
762
|
-
result = await
|
|
807
|
+
result = await coderAgent.run(message, callbacks);
|
|
763
808
|
}
|
|
764
809
|
finally {
|
|
765
810
|
this.ui.endStreaming();
|
|
766
811
|
}
|
|
767
812
|
}
|
|
768
813
|
else {
|
|
769
|
-
|
|
770
|
-
const
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
else {
|
|
781
|
-
this.ui.warning('No agents initialized, using orchestration mode');
|
|
782
|
-
const orchestrateResult = await this.orchestrator.orchestrate(message, callbacks);
|
|
783
|
-
result = {
|
|
784
|
-
content: orchestrateResult.content,
|
|
785
|
-
toolCallsMade: 0,
|
|
786
|
-
iterations: orchestrateResult.execution.iterations,
|
|
787
|
-
usage: orchestrateResult.totalUsage,
|
|
788
|
-
execution: orchestrateResult.execution,
|
|
789
|
-
};
|
|
790
|
-
}
|
|
814
|
+
this.ui.warning('No agents initialized, using orchestration mode');
|
|
815
|
+
const orchestrateResult = await this.orchestrator.orchestrate(message, callbacks);
|
|
816
|
+
result = {
|
|
817
|
+
content: orchestrateResult.content,
|
|
818
|
+
toolCallsMade: 0,
|
|
819
|
+
iterations: orchestrateResult.execution.iterations,
|
|
820
|
+
usage: orchestrateResult.totalUsage,
|
|
821
|
+
execution: orchestrateResult.execution,
|
|
822
|
+
};
|
|
791
823
|
}
|
|
792
824
|
}
|
|
793
|
-
else {
|
|
794
|
-
this.ui.thinking('Analyzing task complexity... Using multi-agent orchestration');
|
|
795
|
-
const orchestrateResult = await this.orchestrator.orchestrate(message, callbacks);
|
|
796
|
-
result = {
|
|
797
|
-
content: orchestrateResult.content,
|
|
798
|
-
toolCallsMade: 0,
|
|
799
|
-
iterations: orchestrateResult.execution.iterations,
|
|
800
|
-
usage: orchestrateResult.totalUsage,
|
|
801
|
-
execution: orchestrateResult.execution,
|
|
802
|
-
};
|
|
803
|
-
}
|
|
804
825
|
}
|
|
805
826
|
// Parse extended thinking blocks from response
|
|
806
827
|
const thinkingResult = this.extendedThinking.parseResponse(result.content);
|
|
@@ -914,8 +935,9 @@ Always consider the strengths of each agent when delegating:
|
|
|
914
935
|
return result.approved;
|
|
915
936
|
}
|
|
916
937
|
/**
|
|
917
|
-
* Assess task complexity
|
|
918
|
-
*
|
|
938
|
+
* Assess task complexity — delegates to ModelRouter
|
|
939
|
+
* Note: complexity no longer determines execution mode (all tasks run until complete)
|
|
940
|
+
* but is still used for model selection
|
|
919
941
|
*/
|
|
920
942
|
assessComplexity(message) {
|
|
921
943
|
const decision = this.modelRouter.route(message);
|
package/dist/core/fallback.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { OpenRouterClient, TokenUsage } from '../api/openrouter.js';
|
|
2
2
|
import { Message } from './types.js';
|
|
3
|
+
import { RateLimitManager } from './rate-limiter.js';
|
|
3
4
|
export interface FallbackConfig {
|
|
4
5
|
models: string[];
|
|
5
6
|
maxRetries: number;
|
|
@@ -27,9 +28,14 @@ export interface FallbackResult {
|
|
|
27
28
|
export declare class FallbackChain {
|
|
28
29
|
private config;
|
|
29
30
|
private client;
|
|
31
|
+
private rateLimiter;
|
|
30
32
|
constructor(client: OpenRouterClient, config?: Partial<FallbackConfig>);
|
|
31
33
|
/**
|
|
32
|
-
*
|
|
34
|
+
* Set the rate limiter for intelligent model rotation
|
|
35
|
+
*/
|
|
36
|
+
setRateLimiter(rateLimiter: RateLimitManager): void;
|
|
37
|
+
/**
|
|
38
|
+
* Execute a chat request with fallback chain + rate limit awareness
|
|
33
39
|
*/
|
|
34
40
|
chatWithFallback(primaryModel: string, messages: Message[], options: {
|
|
35
41
|
tools?: unknown[];
|
package/dist/core/fallback.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// ============================================================
|
|
2
2
|
// NeuroCLI - Fallback Model Chain
|
|
3
3
|
// Automatic model fallback on failure
|
|
4
|
+
// Now with RateLimitManager integration
|
|
4
5
|
// ============================================================
|
|
5
6
|
import { MODELS } from '../api/models.js';
|
|
6
7
|
import chalk from 'chalk';
|
|
7
8
|
export class FallbackChain {
|
|
8
9
|
config;
|
|
9
10
|
client;
|
|
11
|
+
rateLimiter = null;
|
|
10
12
|
constructor(client, config) {
|
|
11
13
|
this.client = client;
|
|
12
14
|
this.config = {
|
|
@@ -18,13 +20,67 @@ export class FallbackChain {
|
|
|
18
20
|
};
|
|
19
21
|
}
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
23
|
+
* Set the rate limiter for intelligent model rotation
|
|
24
|
+
*/
|
|
25
|
+
setRateLimiter(rateLimiter) {
|
|
26
|
+
this.rateLimiter = rateLimiter;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Execute a chat request with fallback chain + rate limit awareness
|
|
22
30
|
*/
|
|
23
31
|
async chatWithFallback(primaryModel, messages, options, onModelSwitch) {
|
|
24
|
-
|
|
32
|
+
// Build the model chain: primary + configured fallbacks
|
|
33
|
+
let chain = [primaryModel, ...this.config.models.filter(m => m !== primaryModel)];
|
|
34
|
+
// If rate limiter is available, prioritize models with good health
|
|
35
|
+
if (this.rateLimiter) {
|
|
36
|
+
const availableModels = this.rateLimiter.getNextRotationModel(primaryModel);
|
|
37
|
+
// Insert rate-limiter-suggested models at the front of the chain
|
|
38
|
+
const healthyAlternatives = [];
|
|
39
|
+
const allStates = this.rateLimiter.getStatus();
|
|
40
|
+
for (const state of allStates) {
|
|
41
|
+
if (state.model !== primaryModel && state.health >= 50 && state.cooldown === 0) {
|
|
42
|
+
healthyAlternatives.push(state.model);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Remove duplicates and merge chains
|
|
46
|
+
const seen = new Set([primaryModel]);
|
|
47
|
+
const mergedChain = [primaryModel];
|
|
48
|
+
// Add healthy alternatives first (from rate limiter)
|
|
49
|
+
for (const m of healthyAlternatives) {
|
|
50
|
+
if (!seen.has(m)) {
|
|
51
|
+
seen.add(m);
|
|
52
|
+
mergedChain.push(m);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
// Then add original fallback chain
|
|
56
|
+
for (const m of chain) {
|
|
57
|
+
if (!seen.has(m)) {
|
|
58
|
+
seen.add(m);
|
|
59
|
+
mergedChain.push(m);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
chain = mergedChain;
|
|
63
|
+
}
|
|
25
64
|
const attempts = [];
|
|
26
65
|
let totalUsage = { inputTokens: 0, outputTokens: 0, cost: 0 };
|
|
27
66
|
for (const model of chain) {
|
|
67
|
+
// Skip models that are in cooldown (if rate limiter is available)
|
|
68
|
+
if (this.rateLimiter) {
|
|
69
|
+
const check = this.rateLimiter.checkRateLimit(model);
|
|
70
|
+
if (!check.allowed && check.waitMs > 30000) {
|
|
71
|
+
// If wait is more than 30s, skip this model entirely
|
|
72
|
+
console.log(chalk.gray(` ⏭️ Skipping ${MODELS[model]?.name || model} (cooldown: ${Math.ceil(check.waitMs / 1000)}s)`));
|
|
73
|
+
attempts.push({ model, success: false, error: `In cooldown (${Math.ceil(check.waitMs / 1000)}s)` });
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (!check.allowed && check.suggestedModel) {
|
|
77
|
+
// Use the suggested alternative instead
|
|
78
|
+
if (!chain.includes(check.suggestedModel)) {
|
|
79
|
+
chain.push(check.suggestedModel);
|
|
80
|
+
}
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
28
84
|
for (let retry = 0; retry <= this.config.maxRetries; retry++) {
|
|
29
85
|
try {
|
|
30
86
|
const modelInfo = MODELS[model];
|
|
@@ -62,7 +118,16 @@ export class FallbackChain {
|
|
|
62
118
|
errMsg.includes('503') ||
|
|
63
119
|
errMsg.includes('500'));
|
|
64
120
|
if (shouldFallback) {
|
|
65
|
-
|
|
121
|
+
// Record the rate limit or server error
|
|
122
|
+
if (this.rateLimiter) {
|
|
123
|
+
if (errMsg.includes('429') || errMsg.toLowerCase().includes('rate_limit')) {
|
|
124
|
+
this.rateLimiter.recordRateLimit(model);
|
|
125
|
+
}
|
|
126
|
+
else if (errMsg.includes('500') || errMsg.includes('503')) {
|
|
127
|
+
this.rateLimiter.recordServerError(model);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
console.log(chalk.yellow(` ⚠ ${MODELS[model]?.name || model} failed: ${errMsg.slice(0, 80)}`));
|
|
66
131
|
break; // Try next model in chain
|
|
67
132
|
}
|
|
68
133
|
// For other errors, retry on same model
|
|
@@ -29,9 +29,15 @@ export declare class ModelRouter {
|
|
|
29
29
|
private availableModels;
|
|
30
30
|
private currentEffort;
|
|
31
31
|
private forcedModel;
|
|
32
|
+
private rateLimiter;
|
|
32
33
|
constructor(config: RouterConfig, availableModels: Record<string, ModelInfo>);
|
|
34
|
+
/**
|
|
35
|
+
* Set the rate limiter for rate-aware routing decisions
|
|
36
|
+
*/
|
|
37
|
+
setRateLimiter(rateLimiter: import('./rate-limiter.js').RateLimitManager): void;
|
|
33
38
|
/**
|
|
34
39
|
* Route a prompt to the best available model.
|
|
40
|
+
* Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
|
|
35
41
|
*/
|
|
36
42
|
route(prompt: string, effort?: EffortLevel): RoutingDecision;
|
|
37
43
|
/**
|
|
@@ -72,7 +72,7 @@ const COMPLEXITY_SIGNALS = {
|
|
|
72
72
|
/\b(hello|hi|thanks|yes|no|ok|sure|done|correct|right|please)\b/i,
|
|
73
73
|
],
|
|
74
74
|
lengthThreshold: 80,
|
|
75
|
-
maxSteps:
|
|
75
|
+
maxSteps: 0, // 0 = no limit (Claude Code style)
|
|
76
76
|
},
|
|
77
77
|
moderate: {
|
|
78
78
|
patterns: [
|
|
@@ -83,7 +83,7 @@ const COMPLEXITY_SIGNALS = {
|
|
|
83
83
|
/\b(refactor|clean|simplify|optimize|restructure)\b/i,
|
|
84
84
|
],
|
|
85
85
|
lengthThreshold: 300,
|
|
86
|
-
maxSteps:
|
|
86
|
+
maxSteps: 0, // 0 = no limit (Claude Code style)
|
|
87
87
|
},
|
|
88
88
|
complex: {
|
|
89
89
|
patterns: [
|
|
@@ -97,7 +97,7 @@ const COMPLEXITY_SIGNALS = {
|
|
|
97
97
|
/\b(security\s+(audit|review|hardening)|performance\s+(optim|tuning|profiling)|scalab(ility|le))\b/i,
|
|
98
98
|
],
|
|
99
99
|
lengthThreshold: 500,
|
|
100
|
-
maxSteps:
|
|
100
|
+
maxSteps: 0, // 0 = no limit (Claude Code style)
|
|
101
101
|
},
|
|
102
102
|
};
|
|
103
103
|
// ---------------------------------------------------------------------------
|
|
@@ -124,6 +124,7 @@ export class ModelRouter {
|
|
|
124
124
|
availableModels;
|
|
125
125
|
currentEffort = 'medium';
|
|
126
126
|
forcedModel = null;
|
|
127
|
+
rateLimiter = null;
|
|
127
128
|
// -----------------------------------------------------------------------
|
|
128
129
|
// Construction
|
|
129
130
|
// -----------------------------------------------------------------------
|
|
@@ -135,15 +136,53 @@ export class ModelRouter {
|
|
|
135
136
|
// -----------------------------------------------------------------------
|
|
136
137
|
// Public API
|
|
137
138
|
// -----------------------------------------------------------------------
|
|
139
|
+
/**
|
|
140
|
+
* Set the rate limiter for rate-aware routing decisions
|
|
141
|
+
*/
|
|
142
|
+
setRateLimiter(rateLimiter) {
|
|
143
|
+
this.rateLimiter = rateLimiter;
|
|
144
|
+
}
|
|
138
145
|
/**
|
|
139
146
|
* Route a prompt to the best available model.
|
|
147
|
+
* Now rate-limit-aware: avoids models in cooldown and rotates to healthy alternatives.
|
|
140
148
|
*/
|
|
141
149
|
route(prompt, effort) {
|
|
142
150
|
const effectiveEffort = effort ?? this.currentEffort;
|
|
143
151
|
const analysis = this.analyzePrompt(prompt);
|
|
144
152
|
const estimatedTokens = this.estimateTokens(prompt);
|
|
145
|
-
|
|
153
|
+
let model = this.selectModel(analysis.complexity, analysis.category, effectiveEffort);
|
|
146
154
|
const alternatives = this.getAlternatives(analysis.complexity, analysis.category);
|
|
155
|
+
// --- Rate-limit-aware routing ---
|
|
156
|
+
if (this.rateLimiter) {
|
|
157
|
+
const check = this.rateLimiter.checkRateLimit(model);
|
|
158
|
+
if (!check.allowed && check.suggestedModel) {
|
|
159
|
+
// Use the rate limiter's suggested alternative
|
|
160
|
+
const suggested = check.suggestedModel;
|
|
161
|
+
if (this.availableModels[suggested]) {
|
|
162
|
+
model = suggested;
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
// The suggested model isn't in our available models, try alternatives
|
|
166
|
+
for (const alt of alternatives) {
|
|
167
|
+
const altCheck = this.rateLimiter.checkRateLimit(alt);
|
|
168
|
+
if (altCheck.allowed) {
|
|
169
|
+
model = alt;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
else if (!check.allowed) {
|
|
176
|
+
// Model is rate limited and no suggested alternative, try alternatives
|
|
177
|
+
for (const alt of alternatives) {
|
|
178
|
+
const altCheck = this.rateLimiter.checkRateLimit(alt);
|
|
179
|
+
if (altCheck.allowed) {
|
|
180
|
+
model = alt;
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
147
186
|
const reasoning = this.buildReasoning(analysis, effectiveEffort, model);
|
|
148
187
|
return {
|
|
149
188
|
model,
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
export interface RateLimiterConfig {
|
|
2
|
+
/** Maximum requests per minute across all models (default: 20) */
|
|
3
|
+
globalRpm: number;
|
|
4
|
+
/** Maximum requests per minute per model (default: 10) */
|
|
5
|
+
perModelRpm: number;
|
|
6
|
+
/** Base cooldown after a 429 in ms (default: 10000) */
|
|
7
|
+
baseCooldownMs: number;
|
|
8
|
+
/** Maximum cooldown in ms (default: 120000) */
|
|
9
|
+
maxCooldownMs: number;
|
|
10
|
+
/** Multiplier for consecutive 429s (default: 2.0) */
|
|
11
|
+
backoffMultiplier: number;
|
|
12
|
+
/** Number of free model rotation slots (default: 5) */
|
|
13
|
+
rotationPoolSize: number;
|
|
14
|
+
/** Enable request queuing during rate limits (default: true) */
|
|
15
|
+
enableQueue: boolean;
|
|
16
|
+
/** Maximum queue size (default: 50) */
|
|
17
|
+
maxQueueSize: number;
|
|
18
|
+
/** Maximum time a request can wait in queue in ms (default: 60000) */
|
|
19
|
+
maxQueueWaitMs: number;
|
|
20
|
+
/** Enable adaptive learning from rate limit patterns (default: true) */
|
|
21
|
+
adaptiveLearning: boolean;
|
|
22
|
+
/** Log rate limit events to console (default: true) */
|
|
23
|
+
verbose: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface ModelRateState {
|
|
26
|
+
/** Model ID */
|
|
27
|
+
modelId: string;
|
|
28
|
+
/** Current cooldown end time (0 = not cooling down) */
|
|
29
|
+
cooldownUntil: number;
|
|
30
|
+
/** Number of consecutive 429s */
|
|
31
|
+
consecutive429s: number;
|
|
32
|
+
/** Total 429s received */
|
|
33
|
+
total429s: number;
|
|
34
|
+
/** Total successful requests */
|
|
35
|
+
totalSuccesses: number;
|
|
36
|
+
/** Requests in current minute window */
|
|
37
|
+
requestsThisMinute: number;
|
|
38
|
+
/** Minute window start time */
|
|
39
|
+
minuteWindowStart: number;
|
|
40
|
+
/** Last request timestamp */
|
|
41
|
+
lastRequestAt: number;
|
|
42
|
+
/** Average response time in ms */
|
|
43
|
+
avgResponseMs: number;
|
|
44
|
+
/** Whether model is currently in rotation pool */
|
|
45
|
+
inRotationPool: boolean;
|
|
46
|
+
/** Health score 0-100 (100 = healthy) */
|
|
47
|
+
healthScore: number;
|
|
48
|
+
}
|
|
49
|
+
export interface RateLimitCheck {
|
|
50
|
+
/** Whether the request can proceed */
|
|
51
|
+
allowed: boolean;
|
|
52
|
+
/** Wait time in ms if not allowed (0 if allowed) */
|
|
53
|
+
waitMs: number;
|
|
54
|
+
/** Suggested alternative model (null if current is fine) */
|
|
55
|
+
suggestedModel: string | null;
|
|
56
|
+
/** Reason for the decision */
|
|
57
|
+
reason: string;
|
|
58
|
+
}
|
|
59
|
+
export interface QueuedRequest {
|
|
60
|
+
id: string;
|
|
61
|
+
model: string;
|
|
62
|
+
enqueuedAt: number;
|
|
63
|
+
resolve: (check: RateLimitCheck) => void;
|
|
64
|
+
reject: (error: Error) => void;
|
|
65
|
+
}
|
|
66
|
+
export declare class RateLimitManager {
|
|
67
|
+
private config;
|
|
68
|
+
private modelStates;
|
|
69
|
+
private globalRequestTimestamps;
|
|
70
|
+
private requestQueue;
|
|
71
|
+
private queueProcessTimer;
|
|
72
|
+
private rotationPool;
|
|
73
|
+
private currentRotationIndex;
|
|
74
|
+
private requestIdCounter;
|
|
75
|
+
private rateLimitPatterns;
|
|
76
|
+
constructor(config?: Partial<RateLimiterConfig>);
|
|
77
|
+
/**
|
|
78
|
+
* Check if a request can be made to a specific model.
|
|
79
|
+
* Returns a RateLimitCheck with the decision and optional alternative model.
|
|
80
|
+
*/
|
|
81
|
+
checkRateLimit(modelId: string): RateLimitCheck;
|
|
82
|
+
/**
|
|
83
|
+
* Acquire permission to make a request. Resolves when the request can proceed.
|
|
84
|
+
* If queuing is enabled, waits in queue. Otherwise, rejects if rate limited.
|
|
85
|
+
*/
|
|
86
|
+
acquire(modelId: string): Promise<RateLimitCheck>;
|
|
87
|
+
/**
|
|
88
|
+
* Record a successful request completion
|
|
89
|
+
*/
|
|
90
|
+
recordSuccess(modelId: string, responseTimeMs: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* Record a rate limit (429) response
|
|
93
|
+
*/
|
|
94
|
+
recordRateLimit(modelId: string, retryAfterSeconds?: number): void;
|
|
95
|
+
/**
|
|
96
|
+
* Record a server error (5xx) - treat as potential overload
|
|
97
|
+
*/
|
|
98
|
+
recordServerError(modelId: string): void;
|
|
99
|
+
/**
|
|
100
|
+
* Get the next available model from the rotation pool.
|
|
101
|
+
* Skips models that are in cooldown or have low health.
|
|
102
|
+
*/
|
|
103
|
+
getNextRotationModel(excludeModel?: string): string | null;
|
|
104
|
+
/**
|
|
105
|
+
* Get the best available model for a given requested model.
|
|
106
|
+
* Tries the requested model first, then falls back through the rotation pool.
|
|
107
|
+
*/
|
|
108
|
+
getBestAvailableModel(requestedModel: string): string;
|
|
109
|
+
/**
|
|
110
|
+
* Get rate limit status for all tracked models
|
|
111
|
+
*/
|
|
112
|
+
getStatus(): Array<{
|
|
113
|
+
model: string;
|
|
114
|
+
name: string;
|
|
115
|
+
health: number;
|
|
116
|
+
cooldown: number;
|
|
117
|
+
consecutive429s: number;
|
|
118
|
+
total429s: number;
|
|
119
|
+
totalSuccesses: number;
|
|
120
|
+
rpm: number;
|
|
121
|
+
avgResponseMs: number;
|
|
122
|
+
}>;
|
|
123
|
+
/**
|
|
124
|
+
* Get queue status
|
|
125
|
+
*/
|
|
126
|
+
getQueueStatus(): {
|
|
127
|
+
size: number;
|
|
128
|
+
oldestWaitMs: number;
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Reset all rate limit state (useful for testing)
|
|
132
|
+
*/
|
|
133
|
+
reset(): void;
|
|
134
|
+
/**
|
|
135
|
+
* Reset a specific model's rate limit state
|
|
136
|
+
*/
|
|
137
|
+
resetModel(modelId: string): void;
|
|
138
|
+
/**
|
|
139
|
+
* Update configuration at runtime
|
|
140
|
+
*/
|
|
141
|
+
updateConfig(updates: Partial<RateLimiterConfig>): void;
|
|
142
|
+
/**
|
|
143
|
+
* Print a formatted status table
|
|
144
|
+
*/
|
|
145
|
+
printStatus(): void;
|
|
146
|
+
private getOrCreateState;
|
|
147
|
+
private recordRequestStart;
|
|
148
|
+
private cleanupGlobalTimestamps;
|
|
149
|
+
private cleanupModelTimestamps;
|
|
150
|
+
private findAvailableModel;
|
|
151
|
+
private initializeRotationPool;
|
|
152
|
+
private enqueueRequest;
|
|
153
|
+
private startQueueProcessor;
|
|
154
|
+
private processQueue;
|
|
155
|
+
private recordRateLimitPattern;
|
|
156
|
+
private getHealthBar;
|
|
157
|
+
/**
|
|
158
|
+
* Cleanup - call when shutting down
|
|
159
|
+
*/
|
|
160
|
+
dispose(): void;
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=rate-limiter.d.ts.map
|