infinicode 2.3.4 → 2.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.opencode/plugins/home-logo-animation.tsx +5 -1
- package/.opencode/plugins/mesh-commands.tsx +4 -3
- package/.opencode/plugins/routing-mode-display.tsx +6 -2
- package/.opencode/tui.json +2 -1
- package/dist/cli.js +25 -1
- package/dist/commands/mesh-install.d.ts +10 -0
- package/dist/commands/mesh-install.js +160 -0
- package/dist/commands/run.js +196 -41
- package/dist/kernel/agents/backends/cli-backend.d.ts +37 -0
- package/dist/kernel/agents/backends/cli-backend.js +132 -0
- package/dist/kernel/agents/backends/detect.d.ts +5 -0
- package/dist/kernel/agents/backends/detect.js +40 -0
- package/dist/kernel/agents/backends/index.d.ts +11 -0
- package/dist/kernel/agents/backends/index.js +3 -0
- package/dist/kernel/agents/backends/parse-utils.d.ts +20 -0
- package/dist/kernel/agents/backends/parse-utils.js +72 -0
- package/dist/kernel/agents/backends/registry.d.ts +30 -0
- package/dist/kernel/agents/backends/registry.js +140 -0
- package/dist/kernel/agents/backends/types.d.ts +65 -0
- package/dist/kernel/agents/backends/types.js +1 -0
- package/dist/kernel/agents/index.d.ts +8 -0
- package/dist/kernel/agents/index.js +8 -0
- package/dist/kernel/federation/federation.d.ts +54 -1
- package/dist/kernel/federation/federation.js +180 -8
- package/dist/kernel/federation/types.d.ts +26 -1
- package/dist/kernel/free-providers.js +83 -0
- package/dist/kernel/gateway/index.d.ts +7 -0
- package/dist/kernel/gateway/index.js +7 -0
- package/dist/kernel/gateway/openai-gateway.d.ts +137 -0
- package/dist/kernel/gateway/openai-gateway.js +723 -0
- package/dist/kernel/index.d.ts +3 -1
- package/dist/kernel/index.js +5 -1
- package/dist/kernel/logger.d.ts +16 -3
- package/dist/kernel/logger.js +38 -0
- package/dist/kernel/mcp/mcp-server.js +47 -8
- package/dist/kernel/orchestrator.d.ts +4 -0
- package/dist/kernel/orchestrator.js +34 -0
- package/dist/kernel/provider-manager.d.ts +13 -0
- package/dist/kernel/provider-manager.js +58 -5
- package/dist/kernel/providers/openai-compatible-provider.d.ts +6 -0
- package/dist/kernel/providers/openai-compatible-provider.js +22 -3
- package/dist/kernel/recovery-manager.js +55 -15
- package/dist/kernel/router.js +15 -2
- package/dist/kernel/types.d.ts +12 -2
- package/package.json +1 -1
package/dist/kernel/index.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export { VerificationEngine } from './verification-engine.js';
|
|
|
22
22
|
export type { VerificationEngineDeps } from './verification-engine.js';
|
|
23
23
|
export { RecoveryManager } from './recovery-manager.js';
|
|
24
24
|
export type { RecoveryManagerDeps, RecoveryContext } from './recovery-manager.js';
|
|
25
|
-
export { ConsoleLogger, SilentLogger } from './logger.js';
|
|
25
|
+
export { ConsoleLogger, SilentLogger, FileLogger } from './logger.js';
|
|
26
26
|
export { OllamaProvider } from './providers/ollama-provider.js';
|
|
27
27
|
export type { OllamaProviderOptions } from './providers/ollama-provider.js';
|
|
28
28
|
export { OpenAICompatibleProvider } from './providers/openai-compatible-provider.js';
|
|
@@ -37,6 +37,8 @@ export type { MissionPlannerDeps, PlanOptions } from './planner.js';
|
|
|
37
37
|
export { normalizeCapabilities, CANONICAL_CAPABILITIES } from './capability-map.js';
|
|
38
38
|
export { phaseBenchmarkFloor, phaseLabel } from './scheduler.js';
|
|
39
39
|
export * from './federation/index.js';
|
|
40
|
+
export * from './agents/index.js';
|
|
41
|
+
export * from './gateway/index.js';
|
|
40
42
|
export { createMcpServer, serveMcpStdio } from './mcp/index.js';
|
|
41
43
|
export type { McpServerDeps } from './mcp/index.js';
|
|
42
44
|
export { GoalLoop } from './autonomy/index.js';
|
package/dist/kernel/index.js
CHANGED
|
@@ -18,7 +18,7 @@ export { PluginManager } from './plugin-manager.js';
|
|
|
18
18
|
export { PolicyEngine, PolicyPresets, POLICY_FACTORIES } from './policies.js';
|
|
19
19
|
export { VerificationEngine } from './verification-engine.js';
|
|
20
20
|
export { RecoveryManager } from './recovery-manager.js';
|
|
21
|
-
export { ConsoleLogger, SilentLogger } from './logger.js';
|
|
21
|
+
export { ConsoleLogger, SilentLogger, FileLogger } from './logger.js';
|
|
22
22
|
export { OllamaProvider } from './providers/ollama-provider.js';
|
|
23
23
|
export { OpenAICompatibleProvider } from './providers/openai-compatible-provider.js';
|
|
24
24
|
export { GeminiProvider } from './providers/gemini-provider.js';
|
|
@@ -29,6 +29,10 @@ export { normalizeCapabilities, CANONICAL_CAPABILITIES } from './capability-map.
|
|
|
29
29
|
export { phaseBenchmarkFloor, phaseLabel } from './scheduler.js';
|
|
30
30
|
// Federation — neutral device mesh (peers, telemetry, config sync, MOLTFED interop)
|
|
31
31
|
export * from './federation/index.js';
|
|
32
|
+
// Agents — pluggable executors (native kernel + third-party coding CLIs)
|
|
33
|
+
export * from './agents/index.js';
|
|
34
|
+
// Gateway — local OpenAI-compatible endpoint backed by kernel routing/recovery
|
|
35
|
+
export * from './gateway/index.js';
|
|
32
36
|
// MCP — north-facing control surface (spawn/dispatch/role/voice/map over the mesh)
|
|
33
37
|
export { createMcpServer, serveMcpStdio } from './mcp/index.js';
|
|
34
38
|
// Autonomy — proactive unattended goal loop (interval/idle/once triggers)
|
package/dist/kernel/logger.d.ts
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* OpenKernel — Logger
|
|
3
|
-
*/
|
|
4
1
|
import type { Logger } from './types.js';
|
|
5
2
|
export declare class ConsoleLogger implements Logger {
|
|
6
3
|
private prefix;
|
|
@@ -16,3 +13,19 @@ export declare class SilentLogger implements Logger {
|
|
|
16
13
|
warn(): void;
|
|
17
14
|
error(): void;
|
|
18
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Appends structured lines to a log file (best-effort — never throws into the
|
|
18
|
+
* caller). Used to give the kernel gateway a durable trace of routing/swap/
|
|
19
|
+
* truncation decisions during a TUI session, since its console is owned by the
|
|
20
|
+
* TUI. Read the file to see exactly why a turn died mid-task.
|
|
21
|
+
*/
|
|
22
|
+
export declare class FileLogger implements Logger {
|
|
23
|
+
private file;
|
|
24
|
+
private prefix;
|
|
25
|
+
constructor(file: string, prefix?: string);
|
|
26
|
+
private write;
|
|
27
|
+
debug(message: string, ...args: unknown[]): void;
|
|
28
|
+
info(message: string, ...args: unknown[]): void;
|
|
29
|
+
warn(message: string, ...args: unknown[]): void;
|
|
30
|
+
error(message: string, ...args: unknown[]): void;
|
|
31
|
+
}
|
package/dist/kernel/logger.js
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenKernel — Logger
|
|
3
|
+
*/
|
|
4
|
+
import { appendFileSync } from 'node:fs';
|
|
1
5
|
export class ConsoleLogger {
|
|
2
6
|
prefix;
|
|
3
7
|
constructor(prefix = 'openkernel') {
|
|
@@ -24,3 +28,37 @@ export class SilentLogger {
|
|
|
24
28
|
warn() { }
|
|
25
29
|
error() { }
|
|
26
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Appends structured lines to a log file (best-effort — never throws into the
|
|
33
|
+
* caller). Used to give the kernel gateway a durable trace of routing/swap/
|
|
34
|
+
* truncation decisions during a TUI session, since its console is owned by the
|
|
35
|
+
* TUI. Read the file to see exactly why a turn died mid-task.
|
|
36
|
+
*/
|
|
37
|
+
export class FileLogger {
|
|
38
|
+
file;
|
|
39
|
+
prefix;
|
|
40
|
+
constructor(file, prefix = 'gateway') {
|
|
41
|
+
this.file = file;
|
|
42
|
+
this.prefix = prefix;
|
|
43
|
+
}
|
|
44
|
+
write(level, message, args) {
|
|
45
|
+
try {
|
|
46
|
+
// NB: `fs` is imported at module top — a lazy `require()` here throws
|
|
47
|
+
// "require is not defined" under ESM (which this package runs as), and the
|
|
48
|
+
// throw was silently swallowed, so NO log was ever written. Static import fixes it.
|
|
49
|
+
const extra = args.length ? ' ' + args.map(a => (typeof a === 'string' ? a : JSON.stringify(a))).join(' ') : '';
|
|
50
|
+
const stamp = new Date().toISOString();
|
|
51
|
+
appendFileSync(this.file, `${stamp} ${level.padEnd(5)} [${this.prefix}] ${message}${extra}\n`);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
/* logging must never break the request path */
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
debug(message, ...args) {
|
|
58
|
+
if (process.env.OPENKERNEL_DEBUG === '1')
|
|
59
|
+
this.write('DEBUG', message, args);
|
|
60
|
+
}
|
|
61
|
+
info(message, ...args) { this.write('INFO', message, args); }
|
|
62
|
+
warn(message, ...args) { this.write('WARN', message, args); }
|
|
63
|
+
error(message, ...args) { this.write('ERROR', message, args); }
|
|
64
|
+
}
|
|
@@ -37,37 +37,76 @@ export function createMcpServer(deps) {
|
|
|
37
37
|
// ── Spawn + dispatch (parallel agents across devices, no SSH) ───────────────
|
|
38
38
|
server.registerTool('spawn_agent', {
|
|
39
39
|
title: 'Spawn agent on a device',
|
|
40
|
-
description: 'Hand a task to a device\'s local agent (by name or id). Non-blocking: returns a runId immediately — use follow_task to monitor. Pass wait:true to block until done (short tasks). Runs on the target with its persistent role.',
|
|
40
|
+
description: 'Hand a task to a device\'s local agent (by name or id). Non-blocking: returns a runId immediately — use follow_task to monitor. Pass wait:true to block until done (short tasks). Runs on the target with its persistent role. Set agent to run it with a third-party CLI on the target ("claude" | "codex" | "opencode") instead of the native kernel.',
|
|
41
41
|
inputSchema: {
|
|
42
42
|
task: z.string().describe('what the agent should do'),
|
|
43
43
|
device: z.string().optional().describe('target device name or node id'),
|
|
44
44
|
capabilities: z.array(z.string()).optional().describe('required capabilities, e.g. ["coding"]'),
|
|
45
|
+
agent: z.string().optional().describe('executor on the target: "kernel" (default), "claude", "codex", or "opencode"'),
|
|
45
46
|
role: z.string().optional().describe('override the role context for this task'),
|
|
46
47
|
wait: z.boolean().optional().describe('block until the run finishes (default false)'),
|
|
47
48
|
},
|
|
48
|
-
}, async ({ task, device, capabilities, role, wait }) => {
|
|
49
|
+
}, async ({ task, device, capabilities, agent, role, wait }) => {
|
|
50
|
+
const caps = [...(capabilities ?? []), ...(agent && agent !== 'kernel' ? [`backend:${agent}`] : [])];
|
|
49
51
|
const t = {
|
|
50
52
|
description: task.slice(0, 80),
|
|
51
53
|
prompt: task,
|
|
52
|
-
capabilities:
|
|
54
|
+
capabilities: caps,
|
|
55
|
+
agent,
|
|
53
56
|
role,
|
|
54
57
|
};
|
|
55
58
|
const res = await federation.dispatch(t, { preferName: device, preferNodeId: device }, { wait });
|
|
56
59
|
if (!res)
|
|
57
60
|
return text({ ok: false, error: 'no capable device online for this task' });
|
|
58
|
-
return text({ ok: true, peerId: res.peerId, runId: res.runId, status: res.record.status, record: res.record });
|
|
61
|
+
return text({ ok: true, peerId: res.peerId, runId: res.runId, agent: agent ?? 'kernel', status: res.record.status, record: res.record });
|
|
59
62
|
});
|
|
60
63
|
server.registerTool('dispatch', {
|
|
61
64
|
title: 'Dispatch to best device',
|
|
62
|
-
description: 'Route a task to the best-fit device by capability + load (no device named). Non-blocking; returns a runId. Pass wait:true to block.',
|
|
65
|
+
description: 'Route a task to the best-fit device by capability + load (no device named). Non-blocking; returns a runId. Pass wait:true to block. Set agent to require a device that has that third-party CLI ("claude" | "codex" | "opencode").',
|
|
63
66
|
inputSchema: {
|
|
64
67
|
task: z.string(),
|
|
65
68
|
capabilities: z.array(z.string()).optional(),
|
|
69
|
+
agent: z.string().optional().describe('executor: "kernel" (default), "claude", "codex", or "opencode"'),
|
|
66
70
|
wait: z.boolean().optional(),
|
|
67
71
|
},
|
|
68
|
-
}, async ({ task, capabilities, wait }) => {
|
|
69
|
-
const
|
|
70
|
-
|
|
72
|
+
}, async ({ task, capabilities, agent, wait }) => {
|
|
73
|
+
const caps = [...(capabilities ?? []), ...(agent && agent !== 'kernel' ? [`backend:${agent}`] : [])];
|
|
74
|
+
const res = await federation.dispatch({ description: task.slice(0, 80), prompt: task, capabilities: caps, agent }, undefined, { wait });
|
|
75
|
+
return text(res ? { ok: true, peerId: res.peerId, runId: res.runId, agent: agent ?? 'kernel', status: res.record.status, record: res.record } : { ok: false, error: 'no capable device' });
|
|
76
|
+
});
|
|
77
|
+
// ── Agent mailbox (CLI sessions message each other across devices) ──────────
|
|
78
|
+
server.registerTool('list_agents', {
|
|
79
|
+
title: 'List messageable agents',
|
|
80
|
+
description: 'Every CLI agent session across the mesh you can message — live or dormant-but-resumable. Each has an address (nodeId/sessionId) and the backend that runs it.',
|
|
81
|
+
inputSchema: {},
|
|
82
|
+
}, async () => text(await federation.agents()));
|
|
83
|
+
server.registerTool('send_message', {
|
|
84
|
+
title: 'Message an agent',
|
|
85
|
+
description: 'Send a message to a CLI agent session anywhere on the mesh (by address "nodeId/sessionId", or a sessionId/runId). Resumes that exact session so it keeps its context, and returns a runId to follow the reply. This is how one agent talks to another. Pass wait:true to block for the reply.',
|
|
86
|
+
inputSchema: {
|
|
87
|
+
address: z.string().describe('target agent: "nodeId/sessionId", a sessionId, or the runId that spawned it'),
|
|
88
|
+
text: z.string().describe('the message to deliver'),
|
|
89
|
+
wait: z.boolean().optional().describe('block until the agent replies (default false)'),
|
|
90
|
+
},
|
|
91
|
+
}, async ({ address, text: message, wait }) => {
|
|
92
|
+
const res = await federation.sendTo(address, message, { wait });
|
|
93
|
+
if (!res)
|
|
94
|
+
return text({ ok: false, error: `no agent found for address: ${address}` });
|
|
95
|
+
return text({ ok: true, runId: res.runId, nodeId: res.nodeId, status: res.record?.status ?? 'running', record: res.record });
|
|
96
|
+
});
|
|
97
|
+
server.registerTool('list_backends', {
|
|
98
|
+
title: 'List executors on a device',
|
|
99
|
+
description: 'Which executors a node can run tasks with — "kernel" plus any installed coding CLIs (claude/codex/opencode). Omit nodeId for this node.',
|
|
100
|
+
inputSchema: { nodeId: z.string().optional().describe('target node; omit for local') },
|
|
101
|
+
}, async ({ nodeId }) => {
|
|
102
|
+
if (!nodeId || nodeId === federation.identity.nodeId) {
|
|
103
|
+
return text({ nodeId: federation.identity.nodeId, backends: federation.availableBackends() });
|
|
104
|
+
}
|
|
105
|
+
const status = federation.nodeStatuses().find(s => s.nodeId === nodeId);
|
|
106
|
+
const backends = (status?.capabilities ?? [])
|
|
107
|
+
.filter(c => typeof c === 'string' && c.startsWith('backend:'))
|
|
108
|
+
.map(c => c.slice('backend:'.length));
|
|
109
|
+
return text({ nodeId, backends });
|
|
71
110
|
});
|
|
72
111
|
server.registerTool('follow_task', {
|
|
73
112
|
title: 'Follow a run',
|
|
@@ -39,6 +39,10 @@ export declare class Orchestrator {
|
|
|
39
39
|
/** How many times each task has been replanned (bounds the replan loop). */
|
|
40
40
|
private taskReplans;
|
|
41
41
|
private maxReplans;
|
|
42
|
+
/** How many times each task has parked waiting for capacity to return. */
|
|
43
|
+
private taskWaitRetries;
|
|
44
|
+
/** Generous cap so a long outage keeps the task alive without looping forever. */
|
|
45
|
+
private maxWaitRetries;
|
|
42
46
|
constructor(deps: OrchestratorDeps);
|
|
43
47
|
run(mission: Mission, input: MissionInput): Promise<Mission>;
|
|
44
48
|
private runTasks;
|
|
@@ -10,6 +10,10 @@ export class Orchestrator {
|
|
|
10
10
|
/** How many times each task has been replanned (bounds the replan loop). */
|
|
11
11
|
taskReplans = new Map();
|
|
12
12
|
maxReplans = 2;
|
|
13
|
+
/** How many times each task has parked waiting for capacity to return. */
|
|
14
|
+
taskWaitRetries = new Map();
|
|
15
|
+
/** Generous cap so a long outage keeps the task alive without looping forever. */
|
|
16
|
+
maxWaitRetries = 20;
|
|
13
17
|
constructor(deps) {
|
|
14
18
|
this.deps = deps;
|
|
15
19
|
}
|
|
@@ -237,6 +241,36 @@ export class Orchestrator {
|
|
|
237
241
|
task.error = message;
|
|
238
242
|
break;
|
|
239
243
|
}
|
|
244
|
+
case 'wait-retry': {
|
|
245
|
+
// Transient exhaustion (rate limit / all providers cooling down / outage).
|
|
246
|
+
// Park the task and retry after an escalating backoff — do NOT count it
|
|
247
|
+
// against the attempt limit, so a temporary outage can't fail the mission.
|
|
248
|
+
// Bounded by maxWaitRetries so a permanent dead-end still resolves.
|
|
249
|
+
const waits = (this.taskWaitRetries.get(task.id) ?? 0) + 1;
|
|
250
|
+
this.taskWaitRetries.set(task.id, waits);
|
|
251
|
+
if (waits > this.maxWaitRetries) {
|
|
252
|
+
task.status = 'FAILED';
|
|
253
|
+
task.error = `${message} (no capacity after ${waits} waits)`;
|
|
254
|
+
this.deps.eventBus.emit({
|
|
255
|
+
type: 'TASK_FAILED',
|
|
256
|
+
timestamp: Date.now(),
|
|
257
|
+
missionId: mission.id,
|
|
258
|
+
taskId: task.id,
|
|
259
|
+
workerId: instance.id,
|
|
260
|
+
data: { error: task.error, attempts: task.attempts, waits },
|
|
261
|
+
});
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
await this.deps.checkpoints.save(mission, this.collectWorkerStates(mission.id), 'crash');
|
|
265
|
+
const waitMs = Math.min(60_000, 5_000 * waits); // 5s → capped at 60s
|
|
266
|
+
this.deps.logger.warn(`Task ${task.id}: waiting ${Math.round(waitMs / 1000)}s for capacity (wait #${waits})`);
|
|
267
|
+
if (task.attempts > 0)
|
|
268
|
+
task.attempts--; // don't burn an attempt on a park
|
|
269
|
+
await this.sleep(waitMs);
|
|
270
|
+
task.status = 'PENDING';
|
|
271
|
+
task.error = message;
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
240
274
|
case 'abort':
|
|
241
275
|
default: {
|
|
242
276
|
task.status = 'FAILED';
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
import type { ProviderInfo, ProviderInterface, ProviderModel } from './types.js';
|
|
9
9
|
import type { EventBus } from './event-bus.js';
|
|
10
10
|
import type { Logger } from './types.js';
|
|
11
|
+
/** How long a provider is excluded from routing after each failure class. */
|
|
12
|
+
export declare const COOLDOWN_MS: Record<string, number>;
|
|
11
13
|
export declare class ProviderManager {
|
|
12
14
|
private eventBus;
|
|
13
15
|
private logger;
|
|
@@ -23,6 +25,17 @@ export declare class ProviderManager {
|
|
|
23
25
|
listInfos(): ProviderInfo[];
|
|
24
26
|
getAvailableModels(): Promise<ProviderModel[]>;
|
|
25
27
|
getHealthyProviders(): Promise<ProviderInterface[]>;
|
|
28
|
+
/** Is this provider currently parked on a recovery cooldown? */
|
|
29
|
+
isCoolingDown(info: ProviderInfo): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Park a provider out of routing for a window (out of credits, bad key, rate
|
|
32
|
+
* limited, down). The health check will NOT resurrect it until the cooldown
|
|
33
|
+
* expires — this is what stops a broke provider (whose /models still answers)
|
|
34
|
+
* from being re-picked on every task.
|
|
35
|
+
*/
|
|
36
|
+
markCooldown(providerId: string, ms: number, reason: string): void;
|
|
37
|
+
/** Apply the standard cooldown for a classified failure. Returns the ms used. */
|
|
38
|
+
penalize(providerId: string, failureType: string, reason?: string): number;
|
|
26
39
|
/** Record a successful completion for success-rate + quota tracking. */
|
|
27
40
|
recordSuccess(providerId: string): void;
|
|
28
41
|
/** Record a failed completion (non-429) for success-rate tracking. */
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
const QUOTA_WINDOW_MS = 60_000;
|
|
2
2
|
const QUOTA_MAX_PER_MIN = 60;
|
|
3
|
+
/** How long a provider is excluded from routing after each failure class. */
|
|
4
|
+
export const COOLDOWN_MS = {
|
|
5
|
+
'429': 15_000, // rate limit — clears fast
|
|
6
|
+
'provider-down': 60_000, // network blip / 5xx
|
|
7
|
+
'quota-exhausted': 30 * 60_000, // out of credits — long
|
|
8
|
+
'auth-failure': 6 * 60 * 60_000, // bad key — effectively out for the session
|
|
9
|
+
'content-filter': 0, // not the provider's fault — no cooldown, just rotate
|
|
10
|
+
};
|
|
3
11
|
// A real 1-token completion costs a little, so re-verify with a token at most
|
|
4
12
|
// this often; cheaper /models refreshes fill the gaps between verifications.
|
|
5
13
|
const VERIFY_INTERVAL_MS = 5 * 60_000;
|
|
@@ -48,7 +56,7 @@ export class ProviderManager {
|
|
|
48
56
|
async getAvailableModels() {
|
|
49
57
|
const models = [];
|
|
50
58
|
for (const provider of this.providers.values()) {
|
|
51
|
-
if (!provider.info.healthy)
|
|
59
|
+
if (!provider.info.healthy || this.isCoolingDown(provider.info))
|
|
52
60
|
continue;
|
|
53
61
|
try {
|
|
54
62
|
const caps = await provider.capabilities();
|
|
@@ -61,7 +69,39 @@ export class ProviderManager {
|
|
|
61
69
|
return models;
|
|
62
70
|
}
|
|
63
71
|
async getHealthyProviders() {
|
|
64
|
-
return this.list().filter(p => p.info.healthy);
|
|
72
|
+
return this.list().filter(p => p.info.healthy && !this.isCoolingDown(p.info));
|
|
73
|
+
}
|
|
74
|
+
/** Is this provider currently parked on a recovery cooldown? */
|
|
75
|
+
isCoolingDown(info) {
|
|
76
|
+
return info.cooldownUntil !== undefined && Date.now() < info.cooldownUntil;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Park a provider out of routing for a window (out of credits, bad key, rate
|
|
80
|
+
* limited, down). The health check will NOT resurrect it until the cooldown
|
|
81
|
+
* expires — this is what stops a broke provider (whose /models still answers)
|
|
82
|
+
* from being re-picked on every task.
|
|
83
|
+
*/
|
|
84
|
+
markCooldown(providerId, ms, reason) {
|
|
85
|
+
const info = this.providers.get(providerId)?.info;
|
|
86
|
+
if (!info || ms <= 0)
|
|
87
|
+
return;
|
|
88
|
+
info.cooldownUntil = Date.now() + ms;
|
|
89
|
+
info.healthy = false;
|
|
90
|
+
info.available = false;
|
|
91
|
+
info.error = reason;
|
|
92
|
+
this.logger.warn(`Provider ${providerId} cooled down ${Math.round(ms / 1000)}s: ${reason}`);
|
|
93
|
+
this.eventBus.emit({
|
|
94
|
+
type: 'PROVIDER_HEALTH',
|
|
95
|
+
timestamp: Date.now(),
|
|
96
|
+
data: { providerId, healthy: false, cooldownUntil: info.cooldownUntil, reason },
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
/** Apply the standard cooldown for a classified failure. Returns the ms used. */
|
|
100
|
+
penalize(providerId, failureType, reason) {
|
|
101
|
+
const ms = COOLDOWN_MS[failureType] ?? 0;
|
|
102
|
+
if (ms > 0)
|
|
103
|
+
this.markCooldown(providerId, ms, reason ?? failureType);
|
|
104
|
+
return ms;
|
|
65
105
|
}
|
|
66
106
|
/** Record a successful completion for success-rate + quota tracking. */
|
|
67
107
|
recordSuccess(providerId) {
|
|
@@ -107,10 +147,11 @@ export class ProviderManager {
|
|
|
107
147
|
if (info) {
|
|
108
148
|
info.quotaRemaining = Math.max(0, (info.quotaRemaining ?? 0) - 1);
|
|
109
149
|
info.successRate = s.totalRequests > 0 ? s.successfulRequests / s.totalRequests : undefined;
|
|
110
|
-
|
|
111
|
-
|
|
150
|
+
// Short cooldown so the next few turns skip this provider while the rate
|
|
151
|
+
// limit clears, instead of hammering it and burning attempts.
|
|
152
|
+
this.markCooldown(providerId, COOLDOWN_MS['429'], 'rate-limited');
|
|
153
|
+
if ((info.quotaRemaining ?? 0) <= 0)
|
|
112
154
|
info.error = 'rate-limited (quota exhausted)';
|
|
113
|
-
}
|
|
114
155
|
}
|
|
115
156
|
}
|
|
116
157
|
}
|
|
@@ -134,6 +175,13 @@ export class ProviderManager {
|
|
|
134
175
|
const provider = this.providers.get(providerId);
|
|
135
176
|
if (!provider)
|
|
136
177
|
return;
|
|
178
|
+
// Respect an active cooldown — do NOT resurrect a parked provider (out of
|
|
179
|
+
// credits / bad key), even though its cheap /models probe would succeed.
|
|
180
|
+
if (this.isCoolingDown(provider.info))
|
|
181
|
+
return;
|
|
182
|
+
if (provider.info.cooldownUntil !== undefined) {
|
|
183
|
+
provider.info.cooldownUntil = undefined; // expired — allow a fresh check
|
|
184
|
+
}
|
|
137
185
|
const start = Date.now();
|
|
138
186
|
const s = this.stats.get(providerId);
|
|
139
187
|
const now = Date.now();
|
|
@@ -164,6 +212,11 @@ export class ProviderManager {
|
|
|
164
212
|
await provider.capabilities();
|
|
165
213
|
latency = Date.now() - start;
|
|
166
214
|
}
|
|
215
|
+
// A cooldown may have been applied WHILE this async check was in flight
|
|
216
|
+
// (e.g. a credits failure on a concurrent request). Honor it — don't let a
|
|
217
|
+
// late-completing health probe resurrect a just-parked provider.
|
|
218
|
+
if (this.isCoolingDown(provider.info))
|
|
219
|
+
return;
|
|
167
220
|
provider.info.healthy = true;
|
|
168
221
|
provider.info.available = true;
|
|
169
222
|
provider.info.latencyMs = latency;
|
|
@@ -37,6 +37,12 @@ export declare class OpenAICompatibleProvider implements ProviderInterface {
|
|
|
37
37
|
}>;
|
|
38
38
|
private parseTokenLimit;
|
|
39
39
|
private fetchModels;
|
|
40
|
+
/**
|
|
41
|
+
* Infer capabilities for a live-fetched model from its id. Excludes non-chat
|
|
42
|
+
* endpoints (embeddings/audio/rerank/guard) from coding; everything else — a
|
|
43
|
+
* general instruct/chat LLM — is treated as coding+reasoning capable.
|
|
44
|
+
*/
|
|
45
|
+
private inferCapabilities;
|
|
40
46
|
private toProviderModel;
|
|
41
47
|
private headers;
|
|
42
48
|
private buildMessages;
|
|
@@ -183,15 +183,34 @@ export class OpenAICompatibleProvider {
|
|
|
183
183
|
return this.knownModels.map(m => this.toProviderModel(m.id, m.contextLength));
|
|
184
184
|
}
|
|
185
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Infer capabilities for a live-fetched model from its id. Excludes non-chat
|
|
188
|
+
* endpoints (embeddings/audio/rerank/guard) from coding; everything else — a
|
|
189
|
+
* general instruct/chat LLM — is treated as coding+reasoning capable.
|
|
190
|
+
*/
|
|
191
|
+
inferCapabilities(lower) {
|
|
192
|
+
const nonChat = /(embed|embedding|rerank|reranker|bge-|e5-|gte-|whisper|tts|text-to-speech|speech|audio|voice|moderat|guard|safety|\bocr\b|stable-diffusion|flux|image-gen|dall)/;
|
|
193
|
+
if (nonChat.test(lower))
|
|
194
|
+
return ['reasoning'];
|
|
195
|
+
const caps = ['coding', 'reasoning'];
|
|
196
|
+
if (/(vision|vl|multimodal|omni|image|-vlm|maverick|scout)/.test(lower))
|
|
197
|
+
caps.push('vision');
|
|
198
|
+
return caps;
|
|
199
|
+
}
|
|
186
200
|
toProviderModel(id, contextLength) {
|
|
187
201
|
const known = this.knownModels.find(m => m.id === id);
|
|
188
202
|
const lower = id.toLowerCase();
|
|
189
|
-
|
|
203
|
+
// Known models keep their curated tags. For a LIVE-fetched model, infer:
|
|
204
|
+
// modern instruct/chat LLMs (GLM, Kimi, DeepSeek, Qwen, Llama, Mistral…) are
|
|
205
|
+
// all coding-capable, so default to coding+reasoning — tagging them
|
|
206
|
+
// reasoning-only would drop their capabilityScore on coding routes and hand
|
|
207
|
+
// every build to whatever curated model happens to carry the 'coding' tag.
|
|
208
|
+
const capabilities = known?.capabilities ? [...known.capabilities] : this.inferCapabilities(lower);
|
|
190
209
|
if (!capabilities.includes('reasoning'))
|
|
191
210
|
capabilities.push('reasoning');
|
|
192
|
-
if (lower.includes('code') || lower.includes('coder'))
|
|
211
|
+
if ((lower.includes('code') || lower.includes('coder')) && !capabilities.includes('coding'))
|
|
193
212
|
capabilities.push('coding');
|
|
194
|
-
if (lower.includes('vision') || lower.includes('vl'))
|
|
213
|
+
if ((lower.includes('vision') || lower.includes('vl') || lower.includes('multimodal')) && !capabilities.includes('vision'))
|
|
195
214
|
capabilities.push('vision');
|
|
196
215
|
return {
|
|
197
216
|
id,
|
|
@@ -5,11 +5,25 @@ export class RecoveryManager {
|
|
|
5
5
|
}
|
|
6
6
|
classify(error) {
|
|
7
7
|
const lower = error.toLowerCase();
|
|
8
|
+
// Billing/credits FIRST — a 402 or "insufficient credits" must not be read as
|
|
9
|
+
// a transient rate limit; it needs a long provider cooldown, not a fast retry.
|
|
10
|
+
if (/402|payment required|insufficient (funds|credit|credits|balance|quota)|out of (credit|credits|quota)|no credit|quota exceeded|exceeded your current quota|billing|not enough (credit|credits|balance|funds)|add credits|purchase|top up|spending limit/.test(lower)) {
|
|
11
|
+
return 'quota-exhausted';
|
|
12
|
+
}
|
|
13
|
+
if (/401|403|unauthorized|forbidden|invalid api key|invalid.{0,4}key|incorrect api key|authentication|auth.{0,12}fail|permission denied|api key not/.test(lower)) {
|
|
14
|
+
return 'auth-failure';
|
|
15
|
+
}
|
|
8
16
|
if (/429|rate.?limit|too many requests/.test(lower))
|
|
9
17
|
return '429';
|
|
18
|
+
if (/context length|maximum context|context window|too many tokens|reduce.{0,24}(length|tokens)|token limit|maximum.{0,12}tokens|input is too long|string too long|prompt is too long/.test(lower)) {
|
|
19
|
+
return 'context-overflow';
|
|
20
|
+
}
|
|
21
|
+
if (/content (filter|policy|management)|moderation|flagged|safety (system|policy)|responsible ai|jailbreak/.test(lower)) {
|
|
22
|
+
return 'content-filter';
|
|
23
|
+
}
|
|
10
24
|
if (/timeout|timed out|deadline exceeded|abort/.test(lower))
|
|
11
25
|
return 'timeout';
|
|
12
|
-
if (/econnrefused|enotfound|unreachable|provider down|network|socket hang up|fetch failed/.test(lower)) {
|
|
26
|
+
if (/econnrefused|enotfound|unreachable|provider down|network|socket hang up|fetch failed|\b5\d\d\b|service unavailable|bad gateway|overloaded/.test(lower)) {
|
|
13
27
|
return 'provider-down';
|
|
14
28
|
}
|
|
15
29
|
if (/hallucinat|fabricat|invented|not grounded/.test(lower))
|
|
@@ -35,24 +49,50 @@ export class RecoveryManager {
|
|
|
35
49
|
let newWorkerType;
|
|
36
50
|
let escalateToBenchmark;
|
|
37
51
|
switch (failureType) {
|
|
52
|
+
case 'quota-exhausted':
|
|
53
|
+
case 'auth-failure':
|
|
54
|
+
case 'content-filter':
|
|
38
55
|
case '429':
|
|
39
56
|
case 'provider-down': {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
rotateToProviderId = alt.providerId;
|
|
46
|
-
rotateToModelId = alt.modelId;
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
action = belowLimit ? 'retry' : 'abort';
|
|
50
|
-
reason = `${failureType}: ${action} (no alternate provider)`;
|
|
51
|
-
}
|
|
57
|
+
// Park the failing provider so it isn't re-picked next task, then move the
|
|
58
|
+
// work to another provider. If none is available right now, WAIT and retry
|
|
59
|
+
// rather than abort — the credits/limit/outage is usually temporary.
|
|
60
|
+
if (ctx.currentProviderId) {
|
|
61
|
+
this.deps.providerManager.penalize(ctx.currentProviderId, failureType, `${failureType}: ${ctx.error.slice(0, 120)}`);
|
|
52
62
|
}
|
|
53
|
-
|
|
63
|
+
const alt = await this.findAlternateProvider(ctx);
|
|
64
|
+
if (alt) {
|
|
65
|
+
action = 'rotate-provider';
|
|
66
|
+
reason = `${failureType}: rotate to ${alt.providerId}/${alt.modelId}`;
|
|
67
|
+
rotateToProviderId = alt.providerId;
|
|
68
|
+
rotateToModelId = alt.modelId;
|
|
69
|
+
}
|
|
70
|
+
else if (failureType === 'auth-failure') {
|
|
71
|
+
// A bad key won't self-heal and there's nowhere else to go.
|
|
54
72
|
action = 'abort';
|
|
55
|
-
reason = `${failureType}:
|
|
73
|
+
reason = `${failureType}: no working provider (fix the API key)`;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
// Rate limit / credits / outage with no alternate → keep the task alive.
|
|
77
|
+
action = 'wait-retry';
|
|
78
|
+
reason = `${failureType}: no alternate provider — parking, will retry`;
|
|
79
|
+
}
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
case 'context-overflow': {
|
|
83
|
+
// The request is too big for this model — climb to a larger model, else
|
|
84
|
+
// replan (fresh routing / decomposition) rather than retry the same size.
|
|
85
|
+
const bigger = await this.findMoreCapableModel(ctx);
|
|
86
|
+
if (bigger) {
|
|
87
|
+
action = 'escalate-model';
|
|
88
|
+
reason = `context-overflow: escalate → ${bigger.providerId}/${bigger.modelId}`;
|
|
89
|
+
rotateToProviderId = bigger.providerId;
|
|
90
|
+
rotateToModelId = bigger.modelId;
|
|
91
|
+
escalateToBenchmark = bigger.benchmarkScore;
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
action = 'replan';
|
|
95
|
+
reason = 'context-overflow: at model ceiling → replan (decompose/compact)';
|
|
56
96
|
}
|
|
57
97
|
break;
|
|
58
98
|
}
|
package/dist/kernel/router.js
CHANGED
|
@@ -193,8 +193,21 @@ export class CapabilityRouter {
|
|
|
193
193
|
score = 44;
|
|
194
194
|
else
|
|
195
195
|
score = 58; // unknown-size cloud model — assume mid-tier
|
|
196
|
-
//
|
|
197
|
-
|
|
196
|
+
// Flagship frontier models — top-tier reasoning/coding that punch well above
|
|
197
|
+
// their (often unknown) parameter count. Scored above a 70B open model so the
|
|
198
|
+
// router actually reaches for them instead of defaulting to a fast mid model.
|
|
199
|
+
if (/(gpt-5|gpt-4\.1|o[1-9]\b|claude-?(3\.[57]|4|opus|sonnet|haiku-4)|deepseek-?(r1|v3|v3\.1|v4)|gemini-2(\.\d)?-pro|gemini-2\.5|glm-(4\.6|[5-9])|kimi|qwen-?3|qwen2\.5-?(72b|max)|minimax|nemotron-?(ultra|340b|4)|mistral-large-2|command-a|grok-[3-9])/.test(text)) {
|
|
200
|
+
// True frontier = unknown/large-size flagship MoEs (GLM-4.6, Kimi-K2,
|
|
201
|
+
// DeepSeek-V3). A flagship FAMILY at a small/mid param size (e.g. qwen3-27b)
|
|
202
|
+
// is fast but NOT frontier — a flat 91 floor let it tie those at 94 and, via
|
|
203
|
+
// first-match-wins, steal the default for whichever provider lists first
|
|
204
|
+
// (Groq). Floor mid-size flagships lower so genuine frontier wins outright.
|
|
205
|
+
score = Math.max(score, paramSize > 0 && paramSize < 30 ? 88 : 91);
|
|
206
|
+
}
|
|
207
|
+
// Strong models — capable, a tier below flagship. Llama 4 (Scout/Maverick) lands
|
|
208
|
+
// here, NOT flagship: bare `llama-4` in the top tier let a fast 17B MoE (Scout)
|
|
209
|
+
// tie GLM-4.6/Kimi at 94 and, via first-match-wins, steal the default for Groq.
|
|
210
|
+
else if (/(gpt-4o|gpt-oss|claude|deepseek|gemini-1\.5-pro|llama-4|llama-3\.3|qwen2?\.5|qwq|glm-4|nemotron|mistral-large|command-r-plus|ernie)/.test(text)) {
|
|
198
211
|
score = Math.max(score, 85);
|
|
199
212
|
}
|
|
200
213
|
// Small/fast variants are capable but not top-tier (avoid demoting MoE "air"
|
package/dist/kernel/types.d.ts
CHANGED
|
@@ -186,6 +186,13 @@ export interface ProviderInfo {
|
|
|
186
186
|
*/
|
|
187
187
|
maxRequestTokens?: number;
|
|
188
188
|
error?: string;
|
|
189
|
+
/**
|
|
190
|
+
* When set and in the future, the provider is on a recovery cooldown (out of
|
|
191
|
+
* credits, bad key, rate-limited, or down) and is excluded from routing until
|
|
192
|
+
* this time. Prevents a broke/dead provider — whose /models still responds —
|
|
193
|
+
* from being resurrected by the health check and re-picked every task.
|
|
194
|
+
*/
|
|
195
|
+
cooldownUntil?: number;
|
|
189
196
|
}
|
|
190
197
|
export interface ProviderCapabilities {
|
|
191
198
|
streaming: boolean;
|
|
@@ -407,8 +414,11 @@ export interface LlmJudgeOptions {
|
|
|
407
414
|
providerId?: string;
|
|
408
415
|
modelId?: string;
|
|
409
416
|
}
|
|
410
|
-
export type FailureType = '429' | 'timeout' | 'provider-down' | 'hallucination' | 'tool-failure' | 'verification-failure' | 'planning-failure' | 'browser-failure' | 'unknown';
|
|
411
|
-
export type RecoveryAction = 'retry' | 'rotate-provider' | 'escalate-model' | 'spawn-different-worker' | 'replan' | 'checkpoint' | 'continue'
|
|
417
|
+
export type FailureType = '429' | 'timeout' | 'provider-down' | 'quota-exhausted' | 'auth-failure' | 'context-overflow' | 'content-filter' | 'hallucination' | 'tool-failure' | 'verification-failure' | 'planning-failure' | 'browser-failure' | 'unknown';
|
|
418
|
+
export type RecoveryAction = 'retry' | 'rotate-provider' | 'escalate-model' | 'spawn-different-worker' | 'replan' | 'checkpoint' | 'continue'
|
|
419
|
+
/** Park the task and retry after a backoff — for transient exhaustion (rate
|
|
420
|
+
* limit / all providers cooling down) where the work must NOT be abandoned. */
|
|
421
|
+
| 'wait-retry' | 'abort';
|
|
412
422
|
export interface RecoveryDecision {
|
|
413
423
|
action: RecoveryAction;
|
|
414
424
|
reason: string;
|
package/package.json
CHANGED