@phuetz/code-buddy 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -164
- package/dist/agent/autonomous/agentic-coding-contract.d.ts +6 -6
- package/dist/agent/codebuddy-agent.d.ts +1 -1
- package/dist/agent/codebuddy-agent.js +101 -13
- package/dist/agent/execution/agent-executor.js +122 -12
- package/dist/agent/execution/tool-dependency-graph.js +7 -0
- package/dist/agent/execution/tool-hooks.d.ts +6 -0
- package/dist/agent/execution/tool-hooks.js +13 -5
- package/dist/agent/execution/tool-selection-strategy.d.ts +11 -2
- package/dist/agent/execution/tool-selection-strategy.js +67 -10
- package/dist/agent/extended-thinking.d.ts +8 -0
- package/dist/agent/extended-thinking.js +27 -0
- package/dist/agent/hermes-browser-backends.js +151 -11
- package/dist/agent/hermes-claw-migrate.d.ts +59 -0
- package/dist/agent/hermes-claw-migrate.js +427 -26
- package/dist/agent/hermes-parity-manifest.js +78 -71
- package/dist/agent/hermes-runtime-lifecycle.d.ts +67 -0
- package/dist/agent/hermes-runtime-lifecycle.js +468 -0
- package/dist/agent/lesson-auto-proposer.js +10 -0
- package/dist/agent/middleware/index.d.ts +1 -0
- package/dist/agent/middleware/index.js +1 -0
- package/dist/agent/middleware/session-duration.d.ts +36 -0
- package/dist/agent/middleware/session-duration.js +78 -0
- package/dist/agent/middleware/visual-validation-middleware.d.ts +17 -0
- package/dist/agent/middleware/visual-validation-middleware.js +67 -0
- package/dist/agent/model-benchmark.d.ts +77 -0
- package/dist/agent/model-benchmark.js +309 -0
- package/dist/agent/model-tier.d.ts +14 -0
- package/dist/agent/model-tier.js +71 -0
- package/dist/agent/multi-agent/agents/orchestrator-agent.js +17 -6
- package/dist/agent/multi-agent/multi-agent-system.js +3 -1
- package/dist/agent/multi-agent/session-fleet-bridge.d.ts +2 -0
- package/dist/agent/multi-agent/session-fleet-bridge.js +9 -1
- package/dist/agent/multi-agent/session-tools.js +22 -5
- package/dist/agent/reasoning/mcts.js +20 -6
- package/dist/agent/session-end-flush.d.ts +71 -0
- package/dist/agent/session-end-flush.js +233 -0
- package/dist/agent/streaming/streaming-handler.d.ts +7 -0
- package/dist/agent/streaming/streaming-handler.js +8 -1
- package/dist/agent/tool-executor.js +1 -0
- package/dist/agent/tool-handler.d.ts +5 -0
- package/dist/agent/tool-handler.js +130 -4
- package/dist/browser-automation/browser-operator-executor.js +15 -0
- package/dist/browser-automation/browser-use-runner.d.ts +96 -0
- package/dist/browser-automation/browser-use-runner.js +492 -0
- package/dist/browser-automation/camofox-runner.d.ts +107 -0
- package/dist/browser-automation/camofox-runner.js +287 -0
- package/dist/channels/dingtalk/index.js +3 -0
- package/dist/channels/discord/client.d.ts +9 -0
- package/dist/channels/discord/client.js +12 -0
- package/dist/channels/feishu/index.d.ts +167 -1
- package/dist/channels/feishu/index.js +432 -7
- package/dist/channels/gateway-lifecycle.d.ts +132 -0
- package/dist/channels/gateway-lifecycle.js +219 -0
- package/dist/channels/google-chat/index.d.ts +5 -1
- package/dist/channels/google-chat/index.js +5 -1
- package/dist/channels/imessage/index.d.ts +14 -0
- package/dist/channels/imessage/index.js +64 -17
- package/dist/channels/index.d.ts +4 -0
- package/dist/channels/index.js +4 -0
- package/dist/channels/irc/index.d.ts +73 -3
- package/dist/channels/irc/index.js +446 -11
- package/dist/channels/line/index.js +3 -0
- package/dist/channels/mattermost/index.d.ts +51 -4
- package/dist/channels/mattermost/index.js +303 -20
- package/dist/channels/nextcloud-talk/index.d.ts +89 -8
- package/dist/channels/nextcloud-talk/index.js +367 -16
- package/dist/channels/nostr/index.d.ts +121 -1
- package/dist/channels/nostr/index.js +396 -8
- package/dist/channels/ntfy/index.js +3 -0
- package/dist/channels/qq/index.js +3 -0
- package/dist/channels/slash-parity.d.ts +117 -0
- package/dist/channels/slash-parity.js +185 -0
- package/dist/channels/synology-chat/index.js +3 -0
- package/dist/channels/teams/index.d.ts +3 -0
- package/dist/channels/teams/index.js +3 -0
- package/dist/channels/telegram/client.d.ts +9 -0
- package/dist/channels/telegram/client.js +12 -0
- package/dist/channels/twilio-voice/index.js +3 -0
- package/dist/channels/webchat/index.d.ts +7 -1
- package/dist/channels/webchat/index.js +7 -1
- package/dist/channels/wecom/index.js +3 -0
- package/dist/channels/weixin/index.js +3 -0
- package/dist/channels/zalo/index.js +3 -0
- package/dist/cli/config-loader.js +1 -1
- package/dist/codebuddy/client.d.ts +30 -1
- package/dist/codebuddy/client.js +167 -20
- package/dist/codebuddy/providers/provider-chatgpt-responses.d.ts +5 -0
- package/dist/codebuddy/providers/provider-chatgpt-responses.js +1 -0
- package/dist/codebuddy/providers/provider-gemini-native.js +1 -1
- package/dist/codebuddy/providers/provider-openai-compat.d.ts +5 -0
- package/dist/codebuddy/providers/provider-openai-compat.js +82 -2
- package/dist/codebuddy/tool-definitions/computer-control-tools.js +32 -2
- package/dist/codebuddy/tool-definitions/core-tools.js +3 -3
- package/dist/codebuddy/tool-definitions/index.d.ts +2 -1
- package/dist/codebuddy/tool-definitions/index.js +2 -1
- package/dist/codebuddy/tool-definitions/multimodal-tools.d.ts +1 -0
- package/dist/codebuddy/tool-definitions/multimodal-tools.js +44 -1
- package/dist/codebuddy/tool-definitions/windows-tools.d.ts +2 -0
- package/dist/codebuddy/tool-definitions/windows-tools.js +34 -0
- package/dist/codebuddy/tools.d.ts +1 -1
- package/dist/codebuddy/tools.js +27 -24
- package/dist/commands/cli/hermes-commands.js +108 -0
- package/dist/commands/cli/native-engine-commands.js +107 -3
- package/dist/commands/cli/utility-commands.js +45 -0
- package/dist/commands/enhanced-command-handler.js +5 -0
- package/dist/commands/goal-cli.d.ts +71 -0
- package/dist/commands/goal-cli.js +280 -0
- package/dist/commands/handlers/goal-handler.d.ts +34 -0
- package/dist/commands/handlers/goal-handler.js +155 -0
- package/dist/commands/handlers/index.d.ts +1 -0
- package/dist/commands/handlers/index.js +2 -0
- package/dist/commands/handlers/infra-handlers.js +36 -0
- package/dist/commands/handlers/memory-handlers.js +96 -1
- package/dist/commands/headless-slash.d.ts +2 -0
- package/dist/commands/headless-slash.js +12 -0
- package/dist/commands/llm-provider-resolution.d.ts +5 -3
- package/dist/commands/llm-provider-resolution.js +87 -33
- package/dist/commands/ollama.d.ts +25 -0
- package/dist/commands/ollama.js +100 -0
- package/dist/commands/provider.d.ts +5 -0
- package/dist/commands/provider.js +106 -55
- package/dist/commands/slash/builtin-commands.js +20 -0
- package/dist/commands/spec-next.js +2 -1
- package/dist/commands/spec-plan.js +5 -16
- package/dist/commands/tunnel.d.ts +2 -0
- package/dist/commands/tunnel.js +48 -0
- package/dist/config/config-resolver.d.ts +2 -1
- package/dist/config/config-resolver.js +54 -41
- package/dist/config/constants.d.ts +28 -0
- package/dist/config/constants.js +7 -0
- package/dist/config/env-schema.js +770 -0
- package/dist/config/feature-flags.js +7 -0
- package/dist/config/model-tools.js +4 -4
- package/dist/config/toml-config.d.ts +16 -0
- package/dist/config/toml-config.js +3 -0
- package/dist/context/context-manager-v2.d.ts +39 -0
- package/dist/context/context-manager-v2.js +91 -0
- package/dist/daemon/agent-task-executor.js +12 -1
- package/dist/daemon/autonomous-daemon.d.ts +1 -1
- package/dist/daemon/autonomous-daemon.js +5 -3
- package/dist/daemon/autonomous-loop.d.ts +21 -1
- package/dist/daemon/autonomous-loop.js +69 -0
- package/dist/daemon/colab-goal.d.ts +38 -0
- package/dist/daemon/colab-goal.js +81 -0
- package/dist/daemon/cron-agent-bridge.d.ts +12 -3
- package/dist/daemon/cron-agent-bridge.js +25 -9
- package/dist/desktop/codebuddy-engine-adapter.d.ts +32 -0
- package/dist/desktop/codebuddy-engine-adapter.js +257 -80
- package/dist/desktop/engine-adapter.d.ts +14 -0
- package/dist/desktop-automation/automation-manager.js +16 -0
- package/dist/desktop-automation/omniparser-runner.d.ts +63 -0
- package/dist/desktop-automation/omniparser-runner.js +115 -0
- package/dist/desktop-automation/smart-snapshot.d.ts +11 -0
- package/dist/desktop-automation/smart-snapshot.js +72 -5
- package/dist/doctor/index.js +40 -23
- package/dist/events/types.d.ts +11 -0
- package/dist/fleet/colab-store.d.ts +21 -0
- package/dist/fleet/colab-store.js +28 -0
- package/dist/fleet/peer-session-bridge.d.ts +1 -1
- package/dist/fleet/peer-session-bridge.js +243 -2
- package/dist/fleet/peer-session-store.d.ts +3 -0
- package/dist/fleet/peer-tool-bridge.js +14 -0
- package/dist/fleet/privacy-lint.d.ts +8 -0
- package/dist/fleet/privacy-lint.js +22 -0
- package/dist/goals/goal-decomposer.d.ts +27 -0
- package/dist/goals/goal-decomposer.js +293 -0
- package/dist/goals/goal-judge-client.d.ts +15 -0
- package/dist/goals/goal-judge-client.js +45 -0
- package/dist/goals/goal-judge.d.ts +38 -0
- package/dist/goals/goal-judge.js +158 -0
- package/dist/goals/goal-loop.d.ts +37 -0
- package/dist/goals/goal-loop.js +101 -0
- package/dist/goals/goal-manager.d.ts +77 -0
- package/dist/goals/goal-manager.js +289 -0
- package/dist/goals/goal-state.d.ts +99 -0
- package/dist/goals/goal-state.js +315 -0
- package/dist/goals/goal-store.d.ts +28 -0
- package/dist/goals/goal-store.js +87 -0
- package/dist/goals/index.d.ts +5 -0
- package/dist/goals/index.js +6 -0
- package/dist/hooks/use-input-handler.js +36 -1
- package/dist/hooks/user-hooks.js +17 -3
- package/dist/index.js +506 -25
- package/dist/input/text-to-speech.d.ts +2 -6
- package/dist/input/text-to-speech.js +2 -27
- package/dist/integrations/screenpipe/screenpipe-client.d.ts +10 -0
- package/dist/integrations/screenpipe/screenpipe-client.js +12 -2
- package/dist/integrations/tailscale.d.ts +13 -0
- package/dist/integrations/tailscale.js +87 -34
- package/dist/knowledge/workspace-indexer.js +53 -9
- package/dist/mcp/client.js +1 -0
- package/dist/mcp/mcp-desktop-tools.d.ts +23 -0
- package/dist/mcp/mcp-desktop-tools.js +158 -0
- package/dist/mcp/mcp-server.js +2 -0
- package/dist/memory/enhanced-memory.d.ts +10 -0
- package/dist/memory/enhanced-memory.js +33 -2
- package/dist/memory/index.d.ts +3 -1
- package/dist/memory/index.js +4 -1
- package/dist/memory/memory-auto-proposer.d.ts +23 -0
- package/dist/memory/memory-auto-proposer.js +308 -0
- package/dist/memory/memory-candidate-queue.d.ts +92 -0
- package/dist/memory/memory-candidate-queue.js +261 -0
- package/dist/memory/persistent-memory.d.ts +45 -2
- package/dist/memory/persistent-memory.js +236 -40
- package/dist/ml/bayesian-qualifier.d.ts +1 -1
- package/dist/ml/bayesian-qualifier.js +35 -2
- package/dist/observability/run-store.d.ts +1 -1
- package/dist/openclaw/gateway-bridge.js +5 -0
- package/dist/plugins/marketplace.d.ts +1 -0
- package/dist/plugins/marketplace.js +7 -0
- package/dist/prompts/execution-discipline.d.ts +14 -0
- package/dist/prompts/execution-discipline.js +29 -0
- package/dist/prompts/variation-injector.js +9 -2
- package/dist/protocols/acp/acp-agentic-runner.d.ts +7 -6
- package/dist/protocols/acp/acp-agentic-runner.js +115 -18
- package/dist/protocols/acp/acp-session-store.d.ts +23 -0
- package/dist/protocols/acp/acp-session-store.js +77 -0
- package/dist/protocols/acp/acp-stdio-server.d.ts +7 -0
- package/dist/protocols/acp/acp-stdio-server.js +71 -16
- package/dist/providers/active-llm-registry.d.ts +37 -0
- package/dist/providers/active-llm-registry.js +186 -0
- package/dist/providers/auxiliary-provider.d.ts +25 -0
- package/dist/providers/auxiliary-provider.js +192 -0
- package/dist/providers/codex-oauth.d.ts +1 -1
- package/dist/providers/codex-oauth.js +27 -4
- package/dist/providers/index.d.ts +3 -0
- package/dist/providers/index.js +4 -0
- package/dist/providers/provider-catalog.d.ts +70 -0
- package/dist/providers/provider-catalog.js +738 -0
- package/dist/providers/provider-fallback.d.ts +35 -0
- package/dist/providers/provider-fallback.js +171 -0
- package/dist/providers/xai-oauth.d.ts +128 -0
- package/dist/providers/xai-oauth.js +735 -0
- package/dist/scheduler/cron-scheduler.d.ts +15 -3
- package/dist/scheduler/cron-scheduler.js +53 -7
- package/dist/scheduler/script-runner.d.ts +2 -0
- package/dist/scheduler/script-runner.js +19 -4
- package/dist/scheduler/watchdog-handlers.js +7 -20
- package/dist/search/usearch-index.js +7 -2
- package/dist/security/tool-policy/tool-groups.js +2 -0
- package/dist/server/index.js +17 -2
- package/dist/server/routes/mobile.d.ts +12 -4
- package/dist/server/routes/mobile.js +116 -1
- package/dist/server/tls-config.d.ts +35 -0
- package/dist/server/tls-config.js +142 -0
- package/dist/server/tunnel-manager.d.ts +20 -0
- package/dist/server/tunnel-manager.js +58 -0
- package/dist/server/websocket/fleet-bridge.d.ts +13 -1
- package/dist/server/websocket/fleet-bridge.js +16 -0
- package/dist/services/prompt-builder.d.ts +1 -0
- package/dist/services/prompt-builder.js +49 -8
- package/dist/shared/engine-types.d.ts +15 -1
- package/dist/sidecar/sidecar-bridge.d.ts +1 -0
- package/dist/sidecar/sidecar-bridge.js +21 -10
- package/dist/spec/spec-store.js +8 -1
- package/dist/talk-mode/providers/audioreader-tts.d.ts +1 -1
- package/dist/talk-mode/providers/audioreader-tts.js +8 -8
- package/dist/templates/project-scaffolding.js +3 -2
- package/dist/themes/theme-schema.d.ts +10 -10
- package/dist/tools/application-profiles.js +38 -0
- package/dist/tools/bash/bash-tool.d.ts +1 -0
- package/dist/tools/bash/bash-tool.js +16 -2
- package/dist/tools/bash/command-validator.js +3 -0
- package/dist/tools/bash/rtk-rewrite.d.ts +16 -0
- package/dist/tools/bash/rtk-rewrite.js +100 -0
- package/dist/tools/bash/streaming-executor.js +14 -1
- package/dist/tools/computer-control-tool.d.ts +16 -1
- package/dist/tools/computer-control-tool.js +317 -5
- package/dist/tools/document-generator.d.ts +14 -0
- package/dist/tools/document-generator.js +79 -0
- package/dist/tools/execute-code-rpc-invoker.d.ts +26 -2
- package/dist/tools/execute-code-rpc-invoker.js +46 -5
- package/dist/tools/metadata.js +28 -0
- package/dist/tools/office-macro-tool.d.ts +10 -0
- package/dist/tools/office-macro-tool.js +93 -0
- package/dist/tools/registry/index.d.ts +2 -1
- package/dist/tools/registry/index.js +2 -1
- package/dist/tools/registry/memory-tools.d.ts +20 -0
- package/dist/tools/registry/memory-tools.js +225 -3
- package/dist/tools/registry/multimodal-tools.js +2 -2
- package/dist/tools/registry/vision-tools.d.ts +33 -0
- package/dist/tools/registry/vision-tools.js +308 -6
- package/dist/tools/registry/windows-tools.d.ts +2 -0
- package/dist/tools/registry/windows-tools.js +47 -0
- package/dist/tools/route-peer-tool.js +14 -0
- package/dist/tools/screenshot-tool.js +14 -2
- package/dist/tools/text-to-speech-tool.d.ts +1 -1
- package/dist/tools/text-to-speech-tool.js +2 -12
- package/dist/tools/tool-selector.js +3 -59
- package/dist/utils/config-validation/schema.d.ts +40 -7
- package/dist/utils/config-validation/schema.js +130 -1
- package/dist/utils/cost-tracker.js +21 -1
- package/dist/utils/disk-guard.d.ts +180 -0
- package/dist/utils/disk-guard.js +404 -0
- package/dist/utils/interactive-setup.js +35 -2
- package/dist/utils/model-utils.js +11 -2
- package/dist/utils/output-sanitizer.js +11 -0
- package/dist/utils/provider-detector.d.ts +8 -7
- package/dist/utils/provider-detector.js +19 -77
- package/dist/utils/settings-manager.d.ts +7 -0
- package/dist/utils/token-display.d.ts +3 -3
- package/dist/utils/token-display.js +22 -4
- package/dist/wizard/onboarding.d.ts +39 -0
- package/dist/wizard/onboarding.js +433 -21
- package/dist/wizard/provider-onboarding.d.ts +8 -23
- package/dist/wizard/provider-onboarding.js +39 -158
- package/package.json +4 -2
- package/dist/plugins/provider-onboarding.d.ts +0 -23
- package/dist/plugins/provider-onboarding.js +0 -116
- package/dist/utils/kokoro-tts.d.ts +0 -8
- package/dist/utils/kokoro-tts.js +0 -67
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Visual Validation Middleware
|
|
3
|
+
*
|
|
4
|
+
* Checks if the agent has saved an Office document (Excel, PowerPoint, Word).
|
|
5
|
+
* If so, it warns the agent to consider running a visual validation step
|
|
6
|
+
* using `snapshot_with_screenshot` to verify the layout, fonts, and alignments.
|
|
7
|
+
*
|
|
8
|
+
* Priority 156
|
|
9
|
+
*/
|
|
10
|
+
import { logger } from '../../utils/logger.js';
|
|
11
|
+
export class VisualValidationMiddleware {
|
|
12
|
+
name = 'visual-validation';
|
|
13
|
+
priority = 156;
|
|
14
|
+
hasWarnedForFiles = new Set();
|
|
15
|
+
async afterTurn(context) {
|
|
16
|
+
if (process.platform !== 'win32') {
|
|
17
|
+
return { action: 'continue' };
|
|
18
|
+
}
|
|
19
|
+
const savedFiles = this.getRecentlySavedOfficeFiles(context);
|
|
20
|
+
if (savedFiles.length === 0) {
|
|
21
|
+
return { action: 'continue' };
|
|
22
|
+
}
|
|
23
|
+
// Filter out files we already warned about
|
|
24
|
+
const newFiles = savedFiles.filter((f) => !this.hasWarnedForFiles.has(f));
|
|
25
|
+
if (newFiles.length === 0) {
|
|
26
|
+
return { action: 'continue' };
|
|
27
|
+
}
|
|
28
|
+
for (const f of newFiles) {
|
|
29
|
+
this.hasWarnedForFiles.add(f);
|
|
30
|
+
}
|
|
31
|
+
logger.info('Visual validation middleware triggered', {
|
|
32
|
+
files: newFiles,
|
|
33
|
+
});
|
|
34
|
+
return {
|
|
35
|
+
action: 'warn',
|
|
36
|
+
message: `You just saved the following Office document(s): ${newFiles.join(', ')}.\n` +
|
|
37
|
+
`To ensure the best quality, please open the document and use the \`snapshot_with_screenshot\` action via \`computer_control\` to visually verify the layout, fonts, and alignment, and auto-correct any visual issues.`,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
getRecentlySavedOfficeFiles(context) {
|
|
41
|
+
const files = new Set();
|
|
42
|
+
// Scan recent tool calls in the context history
|
|
43
|
+
for (let i = context.history.length - 1; i >= Math.max(0, context.history.length - 10); i--) {
|
|
44
|
+
const msg = context.history[i];
|
|
45
|
+
if (!msg || !msg.toolCalls)
|
|
46
|
+
continue;
|
|
47
|
+
for (const call of msg.toolCalls) {
|
|
48
|
+
if (call.function.name === 'computer_control') {
|
|
49
|
+
try {
|
|
50
|
+
const args = JSON.parse(call.function.arguments);
|
|
51
|
+
const action = args.action;
|
|
52
|
+
if (['excel_save_workbook', 'powerpoint_save_presentation', 'word_save_document'].includes(action)) {
|
|
53
|
+
if (args.saveAsPath || args.filePath) {
|
|
54
|
+
files.add(args.saveAsPath || args.filePath);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
// Ignore parse errors
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return Array.from(files);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=visual-validation-middleware.js.map
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live model benchmarking for Tailnet Ollama peers.
|
|
3
|
+
*
|
|
4
|
+
* Measures actual request latency and simple compliance signals so the router
|
|
5
|
+
* can prefer the best network model instead of relying on a static ordering.
|
|
6
|
+
* The benchmark is intentionally lightweight: a few objective prompts, temp=0,
|
|
7
|
+
* streamed completions, and a persisted score cache that the model-tier ladder
|
|
8
|
+
* can use to rank candidates.
|
|
9
|
+
*/
|
|
10
|
+
export interface ModelBenchmarkCandidate {
|
|
11
|
+
model: string;
|
|
12
|
+
baseUrl: string;
|
|
13
|
+
label?: string;
|
|
14
|
+
}
|
|
15
|
+
export interface ModelBenchmarkPrompt {
|
|
16
|
+
name: string;
|
|
17
|
+
prompt: string;
|
|
18
|
+
maxTokens: number;
|
|
19
|
+
validate: (output: string) => boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface ModelBenchmarkRun {
|
|
22
|
+
promptName: string;
|
|
23
|
+
success: boolean;
|
|
24
|
+
compliance: boolean;
|
|
25
|
+
ttftMs: number;
|
|
26
|
+
totalMs: number;
|
|
27
|
+
outputChars: number;
|
|
28
|
+
outputTokensEstimate: number;
|
|
29
|
+
error?: string;
|
|
30
|
+
outputPreview?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface ModelBenchmarkSummary {
|
|
33
|
+
runs: number;
|
|
34
|
+
successes: number;
|
|
35
|
+
complianceRate: number;
|
|
36
|
+
avgTtftMs: number;
|
|
37
|
+
avgTotalMs: number;
|
|
38
|
+
avgOutputTokensEstimate: number;
|
|
39
|
+
score: number;
|
|
40
|
+
}
|
|
41
|
+
export interface ModelBenchmarkReport {
|
|
42
|
+
candidate: ModelBenchmarkCandidate;
|
|
43
|
+
runs: ModelBenchmarkRun[];
|
|
44
|
+
summary: ModelBenchmarkSummary;
|
|
45
|
+
}
|
|
46
|
+
export interface ModelBenchmarkIndexEntry {
|
|
47
|
+
model: string;
|
|
48
|
+
baseUrl: string;
|
|
49
|
+
label?: string;
|
|
50
|
+
score: number;
|
|
51
|
+
complianceRate: number;
|
|
52
|
+
avgTtftMs: number;
|
|
53
|
+
avgTotalMs: number;
|
|
54
|
+
updatedAt: string;
|
|
55
|
+
}
|
|
56
|
+
export interface ModelBenchmarkIndex {
|
|
57
|
+
updatedAt: string;
|
|
58
|
+
suite: string;
|
|
59
|
+
entries: ModelBenchmarkIndexEntry[];
|
|
60
|
+
}
|
|
61
|
+
export interface BenchmarkExecutionOptions {
|
|
62
|
+
runs?: number;
|
|
63
|
+
timeoutMs?: number;
|
|
64
|
+
fetchImpl?: typeof fetch;
|
|
65
|
+
}
|
|
66
|
+
export interface BenchmarkSuiteOptions extends BenchmarkExecutionOptions {
|
|
67
|
+
promptSet?: BenchmarkPromptSetName;
|
|
68
|
+
}
|
|
69
|
+
export type BenchmarkPromptSetName = 'balanced' | 'coding' | 'latency';
|
|
70
|
+
export declare const BENCHMARK_PROMPT_SETS: Record<BenchmarkPromptSetName, ModelBenchmarkPrompt[]>;
|
|
71
|
+
export declare function defaultBenchmarkIndexPath(homeDir?: string): string;
|
|
72
|
+
export declare function benchmarkCandidateKey(candidate: ModelBenchmarkCandidate): string;
|
|
73
|
+
export declare function benchmarkCandidates(candidates: ModelBenchmarkCandidate[], options?: BenchmarkSuiteOptions): Promise<ModelBenchmarkReport[]>;
|
|
74
|
+
export declare function summarizeBenchmarkRuns(runs: ModelBenchmarkRun[]): ModelBenchmarkSummary;
|
|
75
|
+
export declare function writeBenchmarkIndex(reports: ModelBenchmarkReport[], suite: string, indexPath?: string): Promise<ModelBenchmarkIndex>;
|
|
76
|
+
export declare function loadBenchmarkIndex(indexPath?: string): Promise<ModelBenchmarkIndex | null>;
|
|
77
|
+
export declare function loadBenchmarkScoreMap(indexPath?: string): Promise<Map<string, number>>;
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live model benchmarking for Tailnet Ollama peers.
|
|
3
|
+
*
|
|
4
|
+
* Measures actual request latency and simple compliance signals so the router
|
|
5
|
+
* can prefer the best network model instead of relying on a static ordering.
|
|
6
|
+
* The benchmark is intentionally lightweight: a few objective prompts, temp=0,
|
|
7
|
+
* streamed completions, and a persisted score cache that the model-tier ladder
|
|
8
|
+
* can use to rank candidates.
|
|
9
|
+
*/
|
|
10
|
+
import fs from 'node:fs/promises';
|
|
11
|
+
import os from 'node:os';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { normalizeBaseURL } from '../utils/base-url.js';
|
|
14
|
+
export const BENCHMARK_PROMPT_SETS = {
|
|
15
|
+
balanced: [
|
|
16
|
+
{
|
|
17
|
+
name: 'exact-ok',
|
|
18
|
+
prompt: 'Reply with exactly OK and nothing else.',
|
|
19
|
+
maxTokens: 8,
|
|
20
|
+
validate: (output) => output.trim() === 'OK',
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'json-object',
|
|
24
|
+
prompt: 'Return exactly this JSON object and nothing else: {"model":"darkstar","status":"ok"}',
|
|
25
|
+
maxTokens: 32,
|
|
26
|
+
validate: (output) => {
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(stripCodeFences(output));
|
|
29
|
+
return parsed?.model === 'darkstar' && parsed?.status === 'ok';
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: 'code-dedupe',
|
|
38
|
+
prompt: 'Write a TypeScript function named dedupeById that removes duplicate objects by id using a Map. Output only the function.',
|
|
39
|
+
maxTokens: 96,
|
|
40
|
+
validate: (output) => /dedupeById/.test(output) && /Map/.test(output) && /return/.test(output),
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
coding: [
|
|
44
|
+
{
|
|
45
|
+
name: 'json-object',
|
|
46
|
+
prompt: 'Return exactly this JSON object and nothing else: {"model":"darkstar","status":"ok"}',
|
|
47
|
+
maxTokens: 32,
|
|
48
|
+
validate: (output) => {
|
|
49
|
+
try {
|
|
50
|
+
const parsed = JSON.parse(stripCodeFences(output));
|
|
51
|
+
return parsed?.model === 'darkstar' && parsed?.status === 'ok';
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: 'code-dedupe',
|
|
60
|
+
prompt: 'Write a TypeScript function named dedupeById that removes duplicate objects by id using a Map. Output only the function.',
|
|
61
|
+
maxTokens: 96,
|
|
62
|
+
validate: (output) => /dedupeById/.test(output) && /Map/.test(output) && /return/.test(output),
|
|
63
|
+
},
|
|
64
|
+
],
|
|
65
|
+
latency: [
|
|
66
|
+
{
|
|
67
|
+
name: 'exact-ok',
|
|
68
|
+
prompt: 'Reply with exactly OK and nothing else.',
|
|
69
|
+
maxTokens: 8,
|
|
70
|
+
validate: (output) => output.trim() === 'OK',
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
name: 'short-json',
|
|
74
|
+
prompt: 'Return exactly {"ok":true} and nothing else.',
|
|
75
|
+
maxTokens: 16,
|
|
76
|
+
validate: (output) => {
|
|
77
|
+
try {
|
|
78
|
+
const parsed = JSON.parse(stripCodeFences(output));
|
|
79
|
+
return parsed?.ok === true;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
};
|
|
88
|
+
export function defaultBenchmarkIndexPath(homeDir = os.homedir()) {
|
|
89
|
+
return path.join(homeDir, '.codebuddy', 'model-benchmarks.json');
|
|
90
|
+
}
|
|
91
|
+
export function benchmarkCandidateKey(candidate) {
|
|
92
|
+
return `${normalizeBaseURL(candidate.baseUrl)}::${candidate.model}`;
|
|
93
|
+
}
|
|
94
|
+
export async function benchmarkCandidates(candidates, options = {}) {
|
|
95
|
+
const promptSet = options.promptSet ?? 'balanced';
|
|
96
|
+
const prompts = BENCHMARK_PROMPT_SETS[promptSet];
|
|
97
|
+
const runs = Math.max(1, options.runs ?? 1);
|
|
98
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
99
|
+
const reports = [];
|
|
100
|
+
for (const candidate of candidates) {
|
|
101
|
+
const candidateRuns = [];
|
|
102
|
+
for (let runIndex = 0; runIndex < runs; runIndex += 1) {
|
|
103
|
+
for (const prompt of prompts) {
|
|
104
|
+
candidateRuns.push(await runSinglePrompt(candidate, prompt, fetchImpl, options.timeoutMs));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
reports.push({
|
|
108
|
+
candidate,
|
|
109
|
+
runs: candidateRuns,
|
|
110
|
+
summary: summarizeBenchmarkRuns(candidateRuns),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
return reports;
|
|
114
|
+
}
|
|
115
|
+
export function summarizeBenchmarkRuns(runs) {
|
|
116
|
+
const runsCount = runs.length;
|
|
117
|
+
const successes = runs.filter((run) => run.success).length;
|
|
118
|
+
const complianceRuns = runs.filter((run) => run.success && run.compliance).length;
|
|
119
|
+
const successfulRuns = runs.filter((run) => run.success);
|
|
120
|
+
const avgTtftMs = average(successfulRuns.map((run) => run.ttftMs));
|
|
121
|
+
const avgTotalMs = average(successfulRuns.map((run) => run.totalMs));
|
|
122
|
+
const avgOutputTokensEstimate = average(successfulRuns.map((run) => run.outputTokensEstimate));
|
|
123
|
+
const complianceRate = runsCount > 0 ? complianceRuns / runsCount : 0;
|
|
124
|
+
// Weighted score: correctness dominates, then speed. Higher is better.
|
|
125
|
+
const failurePenalty = (runsCount - successes) * 1000;
|
|
126
|
+
const score = successes === 0
|
|
127
|
+
? -1_000_000 - failurePenalty
|
|
128
|
+
: (complianceRate * 1000)
|
|
129
|
+
- avgTtftMs
|
|
130
|
+
- (avgTotalMs * 0.5)
|
|
131
|
+
+ (successes / Math.max(1, runsCount)) * 100
|
|
132
|
+
- failurePenalty;
|
|
133
|
+
return {
|
|
134
|
+
runs: runsCount,
|
|
135
|
+
successes,
|
|
136
|
+
complianceRate,
|
|
137
|
+
avgTtftMs,
|
|
138
|
+
avgTotalMs,
|
|
139
|
+
avgOutputTokensEstimate,
|
|
140
|
+
score,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
export async function writeBenchmarkIndex(reports, suite, indexPath = defaultBenchmarkIndexPath()) {
|
|
144
|
+
const entries = reports
|
|
145
|
+
.map((report) => ({
|
|
146
|
+
model: report.candidate.model,
|
|
147
|
+
baseUrl: normalizeBaseURL(report.candidate.baseUrl),
|
|
148
|
+
...(report.candidate.label ? { label: report.candidate.label } : {}),
|
|
149
|
+
score: report.summary.score,
|
|
150
|
+
complianceRate: report.summary.complianceRate,
|
|
151
|
+
avgTtftMs: report.summary.avgTtftMs,
|
|
152
|
+
avgTotalMs: report.summary.avgTotalMs,
|
|
153
|
+
updatedAt: new Date().toISOString(),
|
|
154
|
+
}))
|
|
155
|
+
.sort((a, b) => b.score - a.score);
|
|
156
|
+
const index = {
|
|
157
|
+
updatedAt: new Date().toISOString(),
|
|
158
|
+
suite,
|
|
159
|
+
entries,
|
|
160
|
+
};
|
|
161
|
+
await fs.mkdir(path.dirname(indexPath), { recursive: true });
|
|
162
|
+
await fs.writeFile(indexPath, `${JSON.stringify(index, null, 2)}\n`, 'utf8');
|
|
163
|
+
return index;
|
|
164
|
+
}
|
|
165
|
+
export async function loadBenchmarkIndex(indexPath = defaultBenchmarkIndexPath()) {
|
|
166
|
+
try {
|
|
167
|
+
const raw = await fs.readFile(indexPath, 'utf8');
|
|
168
|
+
const parsed = JSON.parse(raw);
|
|
169
|
+
if (!parsed || !Array.isArray(parsed.entries))
|
|
170
|
+
return null;
|
|
171
|
+
return parsed;
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
export async function loadBenchmarkScoreMap(indexPath = defaultBenchmarkIndexPath()) {
|
|
178
|
+
const index = await loadBenchmarkIndex(indexPath);
|
|
179
|
+
const map = new Map();
|
|
180
|
+
for (const entry of index?.entries ?? []) {
|
|
181
|
+
map.set(`${normalizeBaseURL(entry.baseUrl)}::${entry.model}`, entry.score);
|
|
182
|
+
}
|
|
183
|
+
return map;
|
|
184
|
+
}
|
|
185
|
+
async function runSinglePrompt(candidate, prompt, fetchImpl, timeoutMs) {
|
|
186
|
+
const endpoint = `${normalizeBaseURL(candidate.baseUrl)}/chat/completions`;
|
|
187
|
+
const startedAt = performance.now();
|
|
188
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
189
|
+
try {
|
|
190
|
+
const response = await fetchImpl(endpoint, {
|
|
191
|
+
method: 'POST',
|
|
192
|
+
headers,
|
|
193
|
+
body: JSON.stringify({
|
|
194
|
+
model: candidate.model,
|
|
195
|
+
messages: [{ role: 'user', content: prompt.prompt }],
|
|
196
|
+
temperature: 0,
|
|
197
|
+
max_tokens: prompt.maxTokens,
|
|
198
|
+
stream: true,
|
|
199
|
+
stream_options: { include_usage: true },
|
|
200
|
+
}),
|
|
201
|
+
signal: AbortSignal.timeout(timeoutMs ?? 60_000),
|
|
202
|
+
});
|
|
203
|
+
if (!response.ok || !response.body) {
|
|
204
|
+
const body = await safeText(response);
|
|
205
|
+
return failureRun(prompt.name, startedAt, `HTTP ${response.status}: ${body.slice(0, 200)}`);
|
|
206
|
+
}
|
|
207
|
+
const streamed = await readOpenAIChatStream(response, startedAt);
|
|
208
|
+
const totalMs = performance.now() - startedAt;
|
|
209
|
+
const compliance = prompt.validate(streamed.output);
|
|
210
|
+
return {
|
|
211
|
+
promptName: prompt.name,
|
|
212
|
+
success: true,
|
|
213
|
+
compliance,
|
|
214
|
+
ttftMs: streamed.ttftMs ?? totalMs,
|
|
215
|
+
totalMs,
|
|
216
|
+
outputChars: streamed.output.length,
|
|
217
|
+
outputTokensEstimate: streamed.outputTokensEstimate ?? estimateTokens(streamed.output),
|
|
218
|
+
...(streamed.output.length ? { outputPreview: trimPreview(streamed.output) } : {}),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
return failureRun(prompt.name, startedAt, error instanceof Error ? error.message : String(error));
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
async function readOpenAIChatStream(response, startedAt) {
|
|
226
|
+
const reader = response.body?.getReader();
|
|
227
|
+
if (!reader) {
|
|
228
|
+
return { output: '', ttftMs: null, outputTokensEstimate: null };
|
|
229
|
+
}
|
|
230
|
+
const decoder = new TextDecoder();
|
|
231
|
+
let buffer = '';
|
|
232
|
+
let output = '';
|
|
233
|
+
let ttftMs = null;
|
|
234
|
+
let firstContentSeen = false;
|
|
235
|
+
while (true) {
|
|
236
|
+
const { done, value } = await reader.read();
|
|
237
|
+
if (done)
|
|
238
|
+
break;
|
|
239
|
+
buffer += decoder.decode(value, { stream: true });
|
|
240
|
+
let newlineIndex = buffer.indexOf('\n');
|
|
241
|
+
while (newlineIndex >= 0) {
|
|
242
|
+
const rawLine = buffer.slice(0, newlineIndex);
|
|
243
|
+
buffer = buffer.slice(newlineIndex + 1);
|
|
244
|
+
const line = rawLine.trim();
|
|
245
|
+
if (line.startsWith('data:')) {
|
|
246
|
+
const payload = line.slice(5).trim();
|
|
247
|
+
if (payload && payload !== '[DONE]') {
|
|
248
|
+
try {
|
|
249
|
+
const parsed = JSON.parse(payload);
|
|
250
|
+
const delta = parsed.choices?.[0]?.delta?.content ?? '';
|
|
251
|
+
if (delta) {
|
|
252
|
+
output += delta;
|
|
253
|
+
if (!firstContentSeen) {
|
|
254
|
+
ttftMs = performance.now() - startedAt;
|
|
255
|
+
firstContentSeen = true;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
// Ignore malformed SSE frames and continue reading the stream.
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
newlineIndex = buffer.indexOf('\n');
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
output,
|
|
269
|
+
ttftMs,
|
|
270
|
+
outputTokensEstimate: estimateTokens(output),
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
function failureRun(promptName, startedAt, error) {
|
|
274
|
+
const totalMs = performance.now() - startedAt;
|
|
275
|
+
return {
|
|
276
|
+
promptName,
|
|
277
|
+
success: false,
|
|
278
|
+
compliance: false,
|
|
279
|
+
ttftMs: totalMs,
|
|
280
|
+
totalMs,
|
|
281
|
+
outputChars: 0,
|
|
282
|
+
outputTokensEstimate: 0,
|
|
283
|
+
error,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
function average(values) {
|
|
287
|
+
if (values.length === 0)
|
|
288
|
+
return 0;
|
|
289
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
290
|
+
}
|
|
291
|
+
function estimateTokens(text) {
|
|
292
|
+
return Math.max(1, Math.ceil(text.trim().length / 4));
|
|
293
|
+
}
|
|
294
|
+
function trimPreview(text, maxChars = 200) {
|
|
295
|
+
const trimmed = text.trim().replace(/\s+/g, ' ');
|
|
296
|
+
return trimmed.length > maxChars ? `${trimmed.slice(0, maxChars)}...` : trimmed;
|
|
297
|
+
}
|
|
298
|
+
function stripCodeFences(text) {
|
|
299
|
+
return text.trim().replace(/^```(?:json|ts|typescript)?\s*/i, '').replace(/\s*```$/i, '');
|
|
300
|
+
}
|
|
301
|
+
async function safeText(response) {
|
|
302
|
+
try {
|
|
303
|
+
return await response.text();
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
return '';
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
//# sourceMappingURL=model-benchmark.js.map
|
|
@@ -32,6 +32,13 @@ export interface ModelTierConfig {
|
|
|
32
32
|
/** Paid cloud model, last resort (undefined = never escalate to paid). */
|
|
33
33
|
escalationModel?: string;
|
|
34
34
|
}
|
|
35
|
+
export interface LiveModelTierDiscoveryOptions {
|
|
36
|
+
/**
|
|
37
|
+
* When true, the resolver keeps the static `CODEBUDDY_NETWORK_MODELS`
|
|
38
|
+
* entries and appends live Tailnet peers discovered at runtime.
|
|
39
|
+
*/
|
|
40
|
+
augmentConfiguredNetworkModels?: boolean;
|
|
41
|
+
}
|
|
35
42
|
export interface EscalationSignal {
|
|
36
43
|
/** Caller explicitly wants the strongest available model for this turn. */
|
|
37
44
|
escalate?: boolean;
|
|
@@ -65,6 +72,13 @@ export interface AutonomousModelChoice {
|
|
|
65
72
|
* - escalation: `CODEBUDDY_ESCALATION_MODEL` → `GROK_MODEL` → none.
|
|
66
73
|
*/
|
|
67
74
|
export declare function resolveModelTierConfig(env?: NodeJS.ProcessEnv): ModelTierConfig;
|
|
75
|
+
/**
|
|
76
|
+
* Resolve the ladder and enrich the network rung with live Tailnet Ollama peers
|
|
77
|
+
* when the `tailscale` CLI is available. This keeps the system dynamic: the
|
|
78
|
+
* network tier grows as peers come and go instead of relying on a hardcoded
|
|
79
|
+
* host list.
|
|
80
|
+
*/
|
|
81
|
+
export declare function resolveLiveModelTierConfig(env?: NodeJS.ProcessEnv, options?: LiveModelTierDiscoveryOptions): Promise<ModelTierConfig>;
|
|
68
82
|
export declare function parseNetworkModels(raw: string | undefined): NetworkModel[];
|
|
69
83
|
/**
|
|
70
84
|
* Choose the model for an autonomous turn. Free-first: local by default, the
|
package/dist/agent/model-tier.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* Pure decision core: given the configured tiers and an escalation signal, it
|
|
15
15
|
* returns which model to use and why. It performs no inference/routing itself.
|
|
16
16
|
*/
|
|
17
|
+
import { normalizeBaseURL } from '../utils/base-url.js';
|
|
17
18
|
const PRIORITY_RANK = { critical: 3, high: 2, medium: 1, low: 0 };
|
|
18
19
|
const DEFAULT_LOCAL_MODEL = 'llama3.2';
|
|
19
20
|
const DEFAULT_LOCAL_BASE_URL = 'http://localhost:11434/v1';
|
|
@@ -41,6 +42,43 @@ export function resolveModelTierConfig(env = process.env) {
|
|
|
41
42
|
...(escalationModel ? { escalationModel } : {}),
|
|
42
43
|
};
|
|
43
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Resolve the ladder and enrich the network rung with live Tailnet Ollama peers
|
|
47
|
+
* when the `tailscale` CLI is available. This keeps the system dynamic: the
|
|
48
|
+
* network tier grows as peers come and go instead of relying on a hardcoded
|
|
49
|
+
* host list.
|
|
50
|
+
*/
|
|
51
|
+
export async function resolveLiveModelTierConfig(env = process.env, options = {}) {
|
|
52
|
+
const config = resolveModelTierConfig(env);
|
|
53
|
+
let discovered = [];
|
|
54
|
+
let benchmarkScores = new Map();
|
|
55
|
+
try {
|
|
56
|
+
const { TailscaleManager } = await import('../integrations/tailscale.js');
|
|
57
|
+
discovered = (await TailscaleManager.getInstance().discoverOllamaPeers()).flatMap((peer) => peer.models.map((model) => ({
|
|
58
|
+
model,
|
|
59
|
+
baseUrl: peer.baseURL,
|
|
60
|
+
label: peer.hostname,
|
|
61
|
+
})));
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
discovered = [];
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const { loadBenchmarkScoreMap } = await import('./model-benchmark.js');
|
|
68
|
+
benchmarkScores = await loadBenchmarkScoreMap();
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
benchmarkScores = new Map();
|
|
72
|
+
}
|
|
73
|
+
const configured = config.networkModels ?? [];
|
|
74
|
+
const merged = options.augmentConfiguredNetworkModels === false
|
|
75
|
+
? discovered
|
|
76
|
+
: mergeNetworkModels(configured, discovered);
|
|
77
|
+
return {
|
|
78
|
+
...config,
|
|
79
|
+
networkModels: rankNetworkModelsByBenchmark(merged, benchmarkScores),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
44
82
|
export function parseNetworkModels(raw) {
|
|
45
83
|
if (!raw?.trim())
|
|
46
84
|
return [];
|
|
@@ -56,6 +94,39 @@ export function parseNetworkModels(raw) {
|
|
|
56
94
|
function normalizeBaseUrl(url) {
|
|
57
95
|
return url.replace(/\/+$/, '');
|
|
58
96
|
}
|
|
97
|
+
function mergeNetworkModels(primary, secondary) {
|
|
98
|
+
const seen = new Set();
|
|
99
|
+
const merged = [];
|
|
100
|
+
for (const item of [...primary, ...secondary]) {
|
|
101
|
+
const key = `${item.model}@${normalizeBaseUrl(item.baseUrl)}`;
|
|
102
|
+
if (seen.has(key))
|
|
103
|
+
continue;
|
|
104
|
+
seen.add(key);
|
|
105
|
+
merged.push(item);
|
|
106
|
+
}
|
|
107
|
+
return merged;
|
|
108
|
+
}
|
|
109
|
+
function rankNetworkModelsByBenchmark(networkModels, benchmarkScores) {
|
|
110
|
+
if (networkModels.length === 0 || benchmarkScores.size === 0) {
|
|
111
|
+
return networkModels;
|
|
112
|
+
}
|
|
113
|
+
return networkModels
|
|
114
|
+
.map((model, index) => ({
|
|
115
|
+
model,
|
|
116
|
+
index,
|
|
117
|
+
score: benchmarkScores.get(`${normalizeBaseURL(model.baseUrl)}::${model.model}`),
|
|
118
|
+
}))
|
|
119
|
+
.sort((a, b) => {
|
|
120
|
+
const aHas = typeof a.score === 'number';
|
|
121
|
+
const bHas = typeof b.score === 'number';
|
|
122
|
+
if (aHas && bHas && a.score !== b.score)
|
|
123
|
+
return (b.score ?? 0) - (a.score ?? 0);
|
|
124
|
+
if (aHas !== bHas)
|
|
125
|
+
return aHas ? -1 : 1;
|
|
126
|
+
return a.index - b.index;
|
|
127
|
+
})
|
|
128
|
+
.map((entry) => entry.model);
|
|
129
|
+
}
|
|
59
130
|
/** Escalation rung 0 (local) / 1 (network) / 2 (paid) implied by the signal. */
|
|
60
131
|
function escalationLevel(signal, policy) {
|
|
61
132
|
let level = 0;
|
|
@@ -64,7 +64,12 @@ DECISION MAKING:
|
|
|
64
64
|
"bash",
|
|
65
65
|
"codebase_map",
|
|
66
66
|
],
|
|
67
|
-
|
|
67
|
+
// Bounded so a model that over-explores during planning/synthesis can't loop
|
|
68
|
+
// indefinitely on a large repo. With the "plan directly" task prompt, planning
|
|
69
|
+
// normally terminates in 1 round; if a model still exhausts this cap without
|
|
70
|
+
// emitting a <plan>, parsePlan() falls back to a default coder task so the
|
|
71
|
+
// workflow still delegates to the worker agents.
|
|
72
|
+
maxRounds: 8,
|
|
68
73
|
};
|
|
69
74
|
export class OrchestratorAgent extends BaseAgent {
|
|
70
75
|
executionPlan = null;
|
|
@@ -88,16 +93,18 @@ export class OrchestratorAgent extends BaseAgent {
|
|
|
88
93
|
const planningTask = {
|
|
89
94
|
id: createId("task"),
|
|
90
95
|
title: "Create Execution Plan",
|
|
91
|
-
description: `Analyze the following goal and
|
|
96
|
+
description: `Analyze the following goal and OUTPUT an execution plan using the <plan> XML format from your instructions.
|
|
92
97
|
|
|
93
98
|
GOAL: ${goal}
|
|
94
99
|
|
|
95
|
-
|
|
96
|
-
1. Breaks
|
|
100
|
+
Plan directly. Do NOT explore the codebase unless it is strictly required to plan — for self-contained tasks, skip exploration entirely and emit the plan in your first response. A good plan:
|
|
101
|
+
1. Breaks the goal into manageable phases
|
|
97
102
|
2. Identifies specific tasks within each phase
|
|
98
103
|
3. Assigns each task to the most appropriate agent
|
|
99
104
|
4. Considers dependencies and parallelization opportunities
|
|
100
|
-
5. Includes validation and review steps
|
|
105
|
+
5. Includes validation and review steps
|
|
106
|
+
|
|
107
|
+
Respond with ONLY the <plan>...</plan> block and no other prose.`,
|
|
101
108
|
status: "in_progress",
|
|
102
109
|
priority: "critical",
|
|
103
110
|
assignedTo: "orchestrator",
|
|
@@ -153,7 +160,11 @@ Explore the codebase to understand the current state, then create a comprehensiv
|
|
|
153
160
|
description: taskDescMatch?.[1]?.trim() ?? "",
|
|
154
161
|
status: "pending",
|
|
155
162
|
priority: priority,
|
|
156
|
-
|
|
163
|
+
// Normalize role casing: the planning prompt lists specialists as
|
|
164
|
+
// "Coder"/"Reviewer"/"Tester" (capitalized), so models often emit
|
|
165
|
+
// `agent="Coder"`, but the agent registry is keyed lowercase. Without
|
|
166
|
+
// this, `agents.get("Coder")` misses and the worker never runs.
|
|
167
|
+
assignedTo: (agent ?? "coder").toLowerCase(),
|
|
157
168
|
dependencies: [],
|
|
158
169
|
subtasks: [],
|
|
159
170
|
artifacts: [],
|
|
@@ -507,7 +507,9 @@ export class MultiAgentSystem extends EventEmitter {
|
|
|
507
507
|
this.addTimelineEvent("task_started", `Reallocated: ${task.title} → ${resolvedRole}`, { task, originalRole: task.assignedTo, resolvedRole });
|
|
508
508
|
task.assignedTo = resolvedRole;
|
|
509
509
|
}
|
|
510
|
-
|
|
510
|
+
// Case-insensitive lookup: plans may carry capitalized roles ("Coder").
|
|
511
|
+
const agent = this.agents.get(task.assignedTo) ??
|
|
512
|
+
this.agents.get(String(task.assignedTo).toLowerCase());
|
|
511
513
|
if (!agent) {
|
|
512
514
|
errors.push(`No agent found for role: ${task.assignedTo}`);
|
|
513
515
|
return;
|
|
@@ -28,6 +28,8 @@
|
|
|
28
28
|
* could fill the WS send buffer.
|
|
29
29
|
*/
|
|
30
30
|
import type { SessionRegistry } from './session-registry.js';
|
|
31
|
+
export type FleetEventBroadcaster = (type: any, payload: any, agentId?: string) => void;
|
|
32
|
+
export declare function registerSessionFleetBroadcaster(broadcaster: FleetEventBroadcaster): void;
|
|
31
33
|
export interface SessionFleetBridgeHandle {
|
|
32
34
|
/** Detach listeners. Idempotent. */
|
|
33
35
|
disable: () => void;
|
|
@@ -27,7 +27,15 @@
|
|
|
27
27
|
* - No back-pressure (same as (d).1 deferral). Bursty session activity
|
|
28
28
|
* could fill the WS send buffer.
|
|
29
29
|
*/
|
|
30
|
-
|
|
30
|
+
let _fleetBroadcaster = null;
|
|
31
|
+
export function registerSessionFleetBroadcaster(broadcaster) {
|
|
32
|
+
_fleetBroadcaster = broadcaster;
|
|
33
|
+
}
|
|
34
|
+
function broadcastFleetEvent(type, payload, agentId) {
|
|
35
|
+
if (_fleetBroadcaster) {
|
|
36
|
+
_fleetBroadcaster(type, payload, agentId);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
31
39
|
function isFleetStreamEnabled() {
|
|
32
40
|
const v = process.env.CODEBUDDY_FLEET_STREAM;
|
|
33
41
|
return v === '1' || v === 'true' || v === 'TRUE';
|