@claude-flow/cli 3.10.46 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/analysis/analyze-code-quality.md +178 -178
- package/.claude/agents/analysis/code-analyzer.md +209 -209
- package/.claude/agents/analysis/code-review/analyze-code-quality.md +178 -178
- package/.claude/agents/architecture/arch-system-design.md +156 -156
- package/.claude/agents/architecture/system-design/arch-system-design.md +154 -154
- package/.claude/agents/browser/browser-agent.yaml +182 -182
- package/.claude/agents/consensus/byzantine-coordinator.md +62 -62
- package/.claude/agents/consensus/crdt-synchronizer.md +996 -996
- package/.claude/agents/consensus/gossip-coordinator.md +62 -62
- package/.claude/agents/consensus/performance-benchmarker.md +850 -850
- package/.claude/agents/consensus/quorum-manager.md +822 -822
- package/.claude/agents/consensus/raft-manager.md +62 -62
- package/.claude/agents/consensus/security-manager.md +621 -621
- package/.claude/agents/core/planner.md +374 -374
- package/.claude/agents/custom/test-long-runner.md +44 -44
- package/.claude/agents/data/data-ml-model.md +444 -444
- package/.claude/agents/data/ml/data-ml-model.md +192 -192
- package/.claude/agents/development/backend/dev-backend-api.md +141 -141
- package/.claude/agents/development/dev-backend-api.md +344 -344
- package/.claude/agents/devops/ci-cd/ops-cicd-github.md +163 -163
- package/.claude/agents/devops/ops-cicd-github.md +164 -164
- package/.claude/agents/documentation/api-docs/docs-api-openapi.md +173 -173
- package/.claude/agents/documentation/docs-api-openapi.md +354 -354
- package/.claude/agents/flow-nexus/app-store.md +87 -87
- package/.claude/agents/flow-nexus/authentication.md +68 -68
- package/.claude/agents/flow-nexus/challenges.md +80 -80
- package/.claude/agents/flow-nexus/neural-network.md +87 -87
- package/.claude/agents/flow-nexus/payments.md +82 -82
- package/.claude/agents/flow-nexus/sandbox.md +75 -75
- package/.claude/agents/flow-nexus/swarm.md +75 -75
- package/.claude/agents/flow-nexus/user-tools.md +95 -95
- package/.claude/agents/flow-nexus/workflow.md +83 -83
- package/.claude/agents/github/code-review-swarm.md +377 -377
- package/.claude/agents/github/github-modes.md +172 -172
- package/.claude/agents/github/issue-tracker.md +575 -575
- package/.claude/agents/github/multi-repo-swarm.md +552 -552
- package/.claude/agents/github/pr-manager.md +437 -437
- package/.claude/agents/github/project-board-sync.md +508 -508
- package/.claude/agents/github/release-manager.md +604 -604
- package/.claude/agents/github/release-swarm.md +582 -582
- package/.claude/agents/github/repo-architect.md +397 -397
- package/.claude/agents/github/swarm-issue.md +572 -572
- package/.claude/agents/github/swarm-pr.md +427 -427
- package/.claude/agents/github/sync-coordinator.md +451 -451
- package/.claude/agents/github/workflow-automation.md +902 -902
- package/.claude/agents/goal/agent.md +815 -815
- package/.claude/agents/optimization/benchmark-suite.md +664 -664
- package/.claude/agents/optimization/load-balancer.md +430 -430
- package/.claude/agents/optimization/performance-monitor.md +671 -671
- package/.claude/agents/optimization/resource-allocator.md +673 -673
- package/.claude/agents/optimization/topology-optimizer.md +807 -807
- package/.claude/agents/payments/agentic-payments.md +126 -126
- package/.claude/agents/sona/sona-learning-optimizer.md +74 -74
- package/.claude/agents/sparc/architecture.md +698 -698
- package/.claude/agents/sparc/pseudocode.md +519 -519
- package/.claude/agents/sparc/refinement.md +801 -801
- package/.claude/agents/sparc/specification.md +477 -477
- package/.claude/agents/specialized/mobile/spec-mobile-react-native.md +224 -224
- package/.claude/agents/specialized/spec-mobile-react-native.md +226 -226
- package/.claude/agents/sublinear/consensus-coordinator.md +337 -337
- package/.claude/agents/sublinear/matrix-optimizer.md +184 -184
- package/.claude/agents/sublinear/pagerank-analyzer.md +298 -298
- package/.claude/agents/sublinear/performance-optimizer.md +367 -367
- package/.claude/agents/sublinear/trading-predictor.md +245 -245
- package/.claude/agents/swarm/adaptive-coordinator.md +1126 -1126
- package/.claude/agents/swarm/hierarchical-coordinator.md +709 -709
- package/.claude/agents/swarm/mesh-coordinator.md +962 -962
- package/.claude/agents/templates/automation-smart-agent.md +204 -204
- package/.claude/agents/templates/base-template-generator.md +289 -289
- package/.claude/agents/templates/coordinator-swarm-init.md +89 -89
- package/.claude/agents/templates/github-pr-manager.md +176 -176
- package/.claude/agents/templates/implementer-sparc-coder.md +258 -258
- package/.claude/agents/templates/memory-coordinator.md +186 -186
- package/.claude/agents/templates/orchestrator-task.md +138 -138
- package/.claude/agents/templates/performance-analyzer.md +198 -198
- package/.claude/agents/templates/sparc-coordinator.md +513 -513
- package/.claude/agents/testing/production-validator.md +394 -394
- package/.claude/agents/testing/tdd-london-swarm.md +243 -243
- package/.claude/agents/v3/aidefence-guardian.md +282 -282
- package/.claude/agents/v3/claims-authorizer.md +208 -208
- package/.claude/agents/v3/collective-intelligence-coordinator.md +993 -993
- package/.claude/agents/v3/ddd-domain-expert.md +220 -220
- package/.claude/agents/v3/injection-analyst.md +236 -236
- package/.claude/agents/v3/performance-engineer.md +1233 -1233
- package/.claude/agents/v3/pii-detector.md +151 -151
- package/.claude/agents/v3/reasoningbank-learner.md +213 -213
- package/.claude/agents/v3/security-architect-aidefence.md +410 -410
- package/.claude/agents/v3/security-architect.md +867 -867
- package/.claude/agents/v3/swarm-memory-manager.md +157 -157
- package/.claude/agents/v3/v3-integration-architect.md +205 -205
- package/.claude/commands/agents/README.md +50 -50
- package/.claude/commands/agents/agent-capabilities.md +140 -140
- package/.claude/commands/agents/agent-coordination.md +28 -28
- package/.claude/commands/agents/agent-spawning.md +28 -28
- package/.claude/commands/agents/agent-types.md +216 -216
- package/.claude/commands/agents/health.md +139 -139
- package/.claude/commands/agents/list.md +100 -100
- package/.claude/commands/agents/logs.md +130 -130
- package/.claude/commands/agents/metrics.md +122 -122
- package/.claude/commands/agents/pool.md +127 -127
- package/.claude/commands/agents/spawn.md +140 -140
- package/.claude/commands/agents/status.md +115 -115
- package/.claude/commands/agents/stop.md +102 -102
- package/.claude/commands/analysis/COMMAND_COMPLIANCE_REPORT.md +53 -53
- package/.claude/commands/analysis/README.md +9 -9
- package/.claude/commands/analysis/bottleneck-detect.md +162 -162
- package/.claude/commands/analysis/performance-bottlenecks.md +58 -58
- package/.claude/commands/analysis/performance-report.md +25 -25
- package/.claude/commands/analysis/token-efficiency.md +44 -44
- package/.claude/commands/analysis/token-usage.md +25 -25
- package/.claude/commands/automation/README.md +9 -9
- package/.claude/commands/automation/auto-agent.md +122 -122
- package/.claude/commands/automation/self-healing.md +105 -105
- package/.claude/commands/automation/session-memory.md +89 -89
- package/.claude/commands/automation/smart-agents.md +72 -72
- package/.claude/commands/automation/smart-spawn.md +25 -25
- package/.claude/commands/automation/workflow-select.md +25 -25
- package/.claude/commands/claude-flow-help.md +103 -103
- package/.claude/commands/claude-flow-memory.md +107 -107
- package/.claude/commands/claude-flow-swarm.md +205 -205
- package/.claude/commands/coordination/README.md +9 -9
- package/.claude/commands/coordination/agent-spawn.md +25 -25
- package/.claude/commands/coordination/init.md +44 -44
- package/.claude/commands/coordination/orchestrate.md +43 -43
- package/.claude/commands/coordination/spawn.md +45 -45
- package/.claude/commands/coordination/swarm-init.md +85 -85
- package/.claude/commands/coordination/task-orchestrate.md +25 -25
- package/.claude/commands/github/README.md +11 -11
- package/.claude/commands/github/code-review-swarm.md +513 -513
- package/.claude/commands/github/code-review.md +25 -25
- package/.claude/commands/github/github-modes.md +146 -146
- package/.claude/commands/github/github-swarm.md +121 -121
- package/.claude/commands/github/issue-tracker.md +291 -291
- package/.claude/commands/github/issue-triage.md +25 -25
- package/.claude/commands/github/multi-repo-swarm.md +518 -518
- package/.claude/commands/github/pr-enhance.md +26 -26
- package/.claude/commands/github/pr-manager.md +169 -169
- package/.claude/commands/github/project-board-sync.md +470 -470
- package/.claude/commands/github/release-manager.md +339 -339
- package/.claude/commands/github/release-swarm.md +543 -543
- package/.claude/commands/github/repo-analyze.md +25 -25
- package/.claude/commands/github/repo-architect.md +366 -366
- package/.claude/commands/github/swarm-issue.md +484 -484
- package/.claude/commands/github/swarm-pr.md +287 -287
- package/.claude/commands/github/sync-coordinator.md +302 -302
- package/.claude/commands/github/workflow-automation.md +441 -441
- package/.claude/commands/hive-mind/README.md +17 -17
- package/.claude/commands/hive-mind/hive-mind-consensus.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-init.md +18 -18
- package/.claude/commands/hive-mind/hive-mind-memory.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-metrics.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-resume.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-sessions.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-spawn.md +21 -21
- package/.claude/commands/hive-mind/hive-mind-status.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-stop.md +8 -8
- package/.claude/commands/hive-mind/hive-mind-wizard.md +8 -8
- package/.claude/commands/hive-mind/hive-mind.md +27 -27
- package/.claude/commands/hooks/README.md +11 -11
- package/.claude/commands/hooks/overview.md +57 -57
- package/.claude/commands/hooks/post-edit.md +117 -117
- package/.claude/commands/hooks/post-task.md +112 -112
- package/.claude/commands/hooks/pre-edit.md +113 -113
- package/.claude/commands/hooks/pre-task.md +111 -111
- package/.claude/commands/hooks/session-end.md +118 -118
- package/.claude/commands/hooks/setup.md +102 -102
- package/.claude/commands/memory/README.md +9 -9
- package/.claude/commands/memory/memory-persist.md +25 -25
- package/.claude/commands/memory/memory-search.md +25 -25
- package/.claude/commands/memory/memory-usage.md +25 -25
- package/.claude/commands/memory/neural.md +47 -47
- package/.claude/commands/monitoring/README.md +9 -9
- package/.claude/commands/monitoring/agent-metrics.md +25 -25
- package/.claude/commands/monitoring/agents.md +44 -44
- package/.claude/commands/monitoring/real-time-view.md +25 -25
- package/.claude/commands/monitoring/status.md +46 -46
- package/.claude/commands/monitoring/swarm-monitor.md +25 -25
- package/.claude/commands/optimization/README.md +9 -9
- package/.claude/commands/optimization/auto-topology.md +61 -61
- package/.claude/commands/optimization/cache-manage.md +25 -25
- package/.claude/commands/optimization/parallel-execute.md +25 -25
- package/.claude/commands/optimization/parallel-execution.md +49 -49
- package/.claude/commands/optimization/topology-optimize.md +25 -25
- package/.claude/commands/pair/README.md +260 -260
- package/.claude/commands/pair/commands.md +545 -545
- package/.claude/commands/pair/config.md +509 -509
- package/.claude/commands/pair/examples.md +511 -511
- package/.claude/commands/pair/modes.md +347 -347
- package/.claude/commands/pair/session.md +406 -406
- package/.claude/commands/pair/start.md +208 -208
- package/.claude/commands/sparc/analyzer.md +51 -51
- package/.claude/commands/sparc/architect.md +53 -53
- package/.claude/commands/sparc/ask.md +97 -97
- package/.claude/commands/sparc/batch-executor.md +54 -54
- package/.claude/commands/sparc/code.md +89 -89
- package/.claude/commands/sparc/coder.md +54 -54
- package/.claude/commands/sparc/debug.md +83 -83
- package/.claude/commands/sparc/debugger.md +54 -54
- package/.claude/commands/sparc/designer.md +53 -53
- package/.claude/commands/sparc/devops.md +109 -109
- package/.claude/commands/sparc/docs-writer.md +80 -80
- package/.claude/commands/sparc/documenter.md +54 -54
- package/.claude/commands/sparc/innovator.md +54 -54
- package/.claude/commands/sparc/integration.md +83 -83
- package/.claude/commands/sparc/mcp.md +117 -117
- package/.claude/commands/sparc/memory-manager.md +54 -54
- package/.claude/commands/sparc/optimizer.md +54 -54
- package/.claude/commands/sparc/orchestrator.md +131 -131
- package/.claude/commands/sparc/post-deployment-monitoring-mode.md +83 -83
- package/.claude/commands/sparc/refinement-optimization-mode.md +83 -83
- package/.claude/commands/sparc/researcher.md +54 -54
- package/.claude/commands/sparc/reviewer.md +54 -54
- package/.claude/commands/sparc/security-review.md +80 -80
- package/.claude/commands/sparc/sparc-modes.md +174 -174
- package/.claude/commands/sparc/sparc.md +111 -111
- package/.claude/commands/sparc/spec-pseudocode.md +80 -80
- package/.claude/commands/sparc/supabase-admin.md +348 -348
- package/.claude/commands/sparc/swarm-coordinator.md +54 -54
- package/.claude/commands/sparc/tdd.md +54 -54
- package/.claude/commands/sparc/tester.md +54 -54
- package/.claude/commands/sparc/tutorial.md +79 -79
- package/.claude/commands/sparc/workflow-manager.md +54 -54
- package/.claude/commands/sparc.md +166 -166
- package/.claude/commands/stream-chain/pipeline.md +120 -120
- package/.claude/commands/stream-chain/run.md +69 -69
- package/.claude/commands/swarm/README.md +15 -15
- package/.claude/commands/swarm/analysis.md +95 -95
- package/.claude/commands/swarm/development.md +96 -96
- package/.claude/commands/swarm/examples.md +168 -168
- package/.claude/commands/swarm/maintenance.md +102 -102
- package/.claude/commands/swarm/optimization.md +117 -117
- package/.claude/commands/swarm/research.md +136 -136
- package/.claude/commands/swarm/swarm-analysis.md +8 -8
- package/.claude/commands/swarm/swarm-background.md +8 -8
- package/.claude/commands/swarm/swarm-init.md +19 -19
- package/.claude/commands/swarm/swarm-modes.md +8 -8
- package/.claude/commands/swarm/swarm-monitor.md +8 -8
- package/.claude/commands/swarm/swarm-spawn.md +19 -19
- package/.claude/commands/swarm/swarm-status.md +8 -8
- package/.claude/commands/swarm/swarm-strategies.md +8 -8
- package/.claude/commands/swarm/swarm.md +87 -87
- package/.claude/commands/swarm/testing.md +131 -131
- package/.claude/commands/training/README.md +9 -9
- package/.claude/commands/training/model-update.md +25 -25
- package/.claude/commands/training/neural-patterns.md +107 -107
- package/.claude/commands/training/neural-train.md +75 -75
- package/.claude/commands/training/pattern-learn.md +25 -25
- package/.claude/commands/training/specialization.md +62 -62
- package/.claude/commands/truth/start.md +142 -142
- package/.claude/commands/verify/check.md +49 -49
- package/.claude/commands/verify/start.md +127 -127
- package/.claude/commands/workflows/README.md +9 -9
- package/.claude/commands/workflows/development.md +77 -77
- package/.claude/commands/workflows/research.md +62 -62
- package/.claude/commands/workflows/workflow-create.md +25 -25
- package/.claude/commands/workflows/workflow-execute.md +25 -25
- package/.claude/commands/workflows/workflow-export.md +25 -25
- package/.claude/helpers/README.md +96 -96
- package/.claude/helpers/adr-compliance.sh +186 -186
- package/.claude/helpers/auto-commit.sh +178 -178
- package/.claude/helpers/auto-memory-hook.mjs +368 -368
- package/.claude/helpers/checkpoint-manager.sh +251 -251
- package/.claude/helpers/daemon-manager.sh +252 -252
- package/.claude/helpers/ddd-tracker.sh +144 -144
- package/.claude/helpers/github-safe.js +156 -156
- package/.claude/helpers/github-setup.sh +45 -45
- package/.claude/helpers/guidance-hook.sh +13 -13
- package/.claude/helpers/guidance-hooks.sh +102 -102
- package/.claude/helpers/health-monitor.sh +108 -108
- package/.claude/helpers/hook-handler.cjs +286 -286
- package/.claude/helpers/intelligence.cjs +1031 -1031
- package/.claude/helpers/learning-hooks.sh +329 -329
- package/.claude/helpers/learning-optimizer.sh +127 -127
- package/.claude/helpers/learning-service.mjs +1144 -1144
- package/.claude/helpers/memory.js +83 -83
- package/.claude/helpers/metrics-db.mjs +488 -488
- package/.claude/helpers/pattern-consolidator.sh +86 -86
- package/.claude/helpers/perf-worker.sh +160 -160
- package/.claude/helpers/post-commit +16 -16
- package/.claude/helpers/pre-commit +26 -26
- package/.claude/helpers/quick-start.sh +19 -19
- package/.claude/helpers/router.js +105 -105
- package/.claude/helpers/security-scanner.sh +127 -127
- package/.claude/helpers/session.js +157 -157
- package/.claude/helpers/setup-mcp.sh +18 -18
- package/.claude/helpers/standard-checkpoint-hooks.sh +189 -189
- package/.claude/helpers/statusline-hook.sh +21 -21
- package/.claude/helpers/statusline.cjs +604 -604
- package/.claude/helpers/statusline.js +352 -352
- package/.claude/helpers/swarm-comms.sh +353 -353
- package/.claude/helpers/swarm-hooks.sh +761 -761
- package/.claude/helpers/swarm-monitor.sh +210 -210
- package/.claude/helpers/sync-v3-metrics.sh +245 -245
- package/.claude/helpers/update-v3-progress.sh +165 -165
- package/.claude/helpers/v3-quick-status.sh +57 -57
- package/.claude/helpers/v3.sh +110 -110
- package/.claude/helpers/validate-v3-config.sh +215 -215
- package/.claude/helpers/worker-manager.sh +170 -170
- package/.claude/settings.json +182 -182
- package/.claude/skills/agentdb-advanced/SKILL.md +550 -550
- package/.claude/skills/agentdb-learning/SKILL.md +545 -545
- package/.claude/skills/agentdb-memory-patterns/SKILL.md +339 -339
- package/.claude/skills/agentdb-optimization/SKILL.md +509 -509
- package/.claude/skills/agentdb-vector-search/SKILL.md +339 -339
- package/.claude/skills/browser/SKILL.md +204 -204
- package/.claude/skills/dual-mode/README.md +71 -71
- package/.claude/skills/dual-mode/dual-collect.md +103 -103
- package/.claude/skills/dual-mode/dual-coordinate.md +85 -85
- package/.claude/skills/dual-mode/dual-spawn.md +81 -81
- package/.claude/skills/flow-nexus-neural/SKILL.md +727 -727
- package/.claude/skills/flow-nexus-platform/SKILL.md +1154 -1154
- package/.claude/skills/flow-nexus-swarm/SKILL.md +604 -604
- package/.claude/skills/github-code-review/SKILL.md +1125 -1125
- package/.claude/skills/github-multi-repo/SKILL.md +862 -862
- package/.claude/skills/github-project-management/SKILL.md +1262 -1262
- package/.claude/skills/github-release-management/SKILL.md +1064 -1064
- package/.claude/skills/github-workflow-automation/SKILL.md +1047 -1047
- package/.claude/skills/hooks-automation/SKILL.md +1201 -1201
- package/.claude/skills/pair-programming/SKILL.md +1202 -1202
- package/.claude/skills/reasoningbank-agentdb/SKILL.md +446 -446
- package/.claude/skills/reasoningbank-intelligence/SKILL.md +201 -201
- package/.claude/skills/skill-builder/SKILL.md +910 -910
- package/.claude/skills/sparc-methodology/SKILL.md +1106 -1106
- package/.claude/skills/stream-chain/SKILL.md +560 -560
- package/.claude/skills/swarm-advanced/SKILL.md +970 -970
- package/.claude/skills/swarm-orchestration/SKILL.md +179 -179
- package/.claude/skills/v3-cli-modernization/SKILL.md +871 -871
- package/.claude/skills/v3-core-implementation/SKILL.md +796 -796
- package/.claude/skills/v3-ddd-architecture/SKILL.md +441 -441
- package/.claude/skills/v3-integration-deep/SKILL.md +240 -240
- package/.claude/skills/v3-mcp-optimization/SKILL.md +776 -776
- package/.claude/skills/v3-memory-unification/SKILL.md +173 -173
- package/.claude/skills/v3-performance-optimization/SKILL.md +389 -389
- package/.claude/skills/v3-security-overhaul/SKILL.md +81 -81
- package/.claude/skills/v3-swarm-coordination/SKILL.md +339 -339
- package/.claude/skills/verification-quality/SKILL.md +691 -691
- package/README.md +412 -412
- package/bin/cli.js +314 -314
- package/bin/mcp-server.js +224 -224
- package/bin/preinstall.cjs +2 -2
- package/dist/src/benchmarks/gaia-critic.js +24 -24
- package/dist/src/commands/completions.js +409 -409
- package/dist/src/commands/daemon.js +44 -44
- package/dist/src/commands/doctor.d.ts.map +1 -1
- package/dist/src/commands/doctor.js +175 -1
- package/dist/src/commands/doctor.js.map +1 -1
- package/dist/src/commands/eject.d.ts +33 -0
- package/dist/src/commands/eject.d.ts.map +1 -0
- package/dist/src/commands/eject.js +195 -0
- package/dist/src/commands/eject.js.map +1 -0
- package/dist/src/commands/embeddings.js +26 -26
- package/dist/src/commands/hive-mind.js +97 -97
- package/dist/src/commands/hooks.d.ts.map +1 -1
- package/dist/src/commands/hooks.js +11 -10
- package/dist/src/commands/hooks.js.map +1 -1
- package/dist/src/commands/index.d.ts.map +1 -1
- package/dist/src/commands/index.js +4 -0
- package/dist/src/commands/index.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +3 -0
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/metaharness.d.ts +39 -0
- package/dist/src/commands/metaharness.d.ts.map +1 -0
- package/dist/src/commands/metaharness.js +194 -0
- package/dist/src/commands/metaharness.js.map +1 -0
- package/dist/src/commands/neural.d.ts.map +1 -1
- package/dist/src/commands/neural.js +2351 -1
- package/dist/src/commands/neural.js.map +1 -1
- package/dist/src/commands/ruvector/backup.js +23 -23
- package/dist/src/commands/ruvector/benchmark.js +31 -31
- package/dist/src/commands/ruvector/import.js +14 -14
- package/dist/src/commands/ruvector/init.js +115 -115
- package/dist/src/commands/ruvector/migrate.js +99 -99
- package/dist/src/commands/ruvector/optimize.js +51 -51
- package/dist/src/commands/ruvector/setup.js +624 -624
- package/dist/src/commands/ruvector/status.js +38 -38
- package/dist/src/init/claudemd-generator.js +231 -231
- package/dist/src/init/executor.js +453 -453
- package/dist/src/init/helpers-generator.js +727 -727
- package/dist/src/init/statusline-generator.js +717 -717
- package/dist/src/mcp-client.d.ts.map +1 -1
- package/dist/src/mcp-client.js +4 -0
- package/dist/src/mcp-client.js.map +1 -1
- package/dist/src/mcp-tools/agent-execute-core.d.ts +23 -1
- package/dist/src/mcp-tools/agent-execute-core.d.ts.map +1 -1
- package/dist/src/mcp-tools/agent-execute-core.js +136 -9
- package/dist/src/mcp-tools/agent-execute-core.js.map +1 -1
- package/dist/src/mcp-tools/agent-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/agent-tools.js +48 -4
- package/dist/src/mcp-tools/agent-tools.js.map +1 -1
- package/dist/src/mcp-tools/agentdb-tools.js +15 -15
- package/dist/src/mcp-tools/hooks-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-tools.js +196 -0
- package/dist/src/mcp-tools/hooks-tools.js.map +1 -1
- package/dist/src/mcp-tools/index.d.ts +1 -0
- package/dist/src/mcp-tools/index.d.ts.map +1 -1
- package/dist/src/mcp-tools/index.js +2 -0
- package/dist/src/mcp-tools/index.js.map +1 -1
- package/dist/src/mcp-tools/metaharness-tools.d.ts +31 -0
- package/dist/src/mcp-tools/metaharness-tools.d.ts.map +1 -0
- package/dist/src/mcp-tools/metaharness-tools.js +372 -0
- package/dist/src/mcp-tools/metaharness-tools.js.map +1 -0
- package/dist/src/memory/graph-edge-writer.js +18 -18
- package/dist/src/memory/memory-bridge.js +84 -84
- package/dist/src/memory/memory-initializer.js +393 -393
- package/dist/src/memory/rabitq-index.js +5 -5
- package/dist/src/plugins/store/discovery.d.ts.map +1 -1
- package/dist/src/plugins/store/discovery.js +3 -0
- package/dist/src/plugins/store/discovery.js.map +1 -1
- package/dist/src/plugins/store/types.d.ts +6 -1
- package/dist/src/plugins/store/types.d.ts.map +1 -1
- package/dist/src/runtime/headless.js +28 -28
- package/dist/src/ruvector/enhanced-model-router.d.ts +5 -0
- package/dist/src/ruvector/enhanced-model-router.d.ts.map +1 -1
- package/dist/src/ruvector/enhanced-model-router.js +18 -1
- package/dist/src/ruvector/enhanced-model-router.js.map +1 -1
- package/dist/src/ruvector/model-prices.d.ts +50 -0
- package/dist/src/ruvector/model-prices.d.ts.map +1 -0
- package/dist/src/ruvector/model-prices.js +72 -0
- package/dist/src/ruvector/model-prices.js.map +1 -0
- package/dist/src/ruvector/model-router.d.ts +147 -8
- package/dist/src/ruvector/model-router.d.ts.map +1 -1
- package/dist/src/ruvector/model-router.js +455 -18
- package/dist/src/ruvector/model-router.js.map +1 -1
- package/dist/src/ruvector/neural-router.d.ts +172 -39
- package/dist/src/ruvector/neural-router.d.ts.map +1 -1
- package/dist/src/ruvector/neural-router.js +780 -91
- package/dist/src/ruvector/neural-router.js.map +1 -1
- package/dist/src/ruvector/router-calibrator.d.ts +65 -0
- package/dist/src/ruvector/router-calibrator.d.ts.map +1 -0
- package/dist/src/ruvector/router-calibrator.js +120 -0
- package/dist/src/ruvector/router-calibrator.js.map +1 -0
- package/dist/src/ruvector/router-parallel-recorder.d.ts +127 -0
- package/dist/src/ruvector/router-parallel-recorder.d.ts.map +1 -0
- package/dist/src/ruvector/router-parallel-recorder.js +183 -0
- package/dist/src/ruvector/router-parallel-recorder.js.map +1 -0
- package/dist/src/ruvector/router-trajectory.d.ts +137 -45
- package/dist/src/ruvector/router-trajectory.d.ts.map +1 -1
- package/dist/src/ruvector/router-trajectory.js +226 -63
- package/dist/src/ruvector/router-trajectory.js.map +1 -1
- package/dist/src/ruvector/task-embedder.d.ts +56 -0
- package/dist/src/ruvector/task-embedder.d.ts.map +1 -0
- package/dist/src/ruvector/task-embedder.js +237 -0
- package/dist/src/ruvector/task-embedder.js.map +1 -0
- package/dist/src/services/headless-worker-executor.js +84 -84
- package/dist/src/transfer/deploy-seraphine.js +23 -23
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +138 -134
- package/scripts/beir-bootstrap-significance.mjs +162 -162
- package/scripts/bench-rvagent.mjs +312 -312
- package/scripts/benchmark-codemods.mjs +127 -127
- package/scripts/benchmark-cross-repo.mjs +206 -206
- package/scripts/benchmark-pretrained-retrieval.mjs +305 -305
- package/scripts/benchmark-self-learning.mjs +241 -241
- package/scripts/benchmark-trajectory-mrr.mjs +191 -191
- package/scripts/deploy-ipfs-node.sh +153 -153
- package/scripts/grid-search-retrieval.mjs +216 -216
- package/scripts/postinstall.cjs +153 -153
- package/scripts/pretrain-from-github.mjs +358 -358
- package/scripts/publish-registry.ts +345 -345
- package/scripts/publish.sh +57 -57
- package/scripts/run-beir-bge.mjs +324 -324
- package/scripts/run-beir-hybrid.mjs +261 -261
- package/scripts/run-beir-lucene-bm25.mjs +154 -154
- package/scripts/run-beir-nfcorpus.mjs +231 -231
- package/scripts/run-beir-rrf-ablation.mjs +221 -221
- package/scripts/setup-ipfs-registry.md +366 -366
- package/dist/src/ruvector/flash-attention.d.ts +0 -195
- package/dist/src/ruvector/flash-attention.d.ts.map +0 -1
- package/dist/src/ruvector/flash-attention.js +0 -643
- package/dist/src/ruvector/flash-attention.js.map +0 -1
- package/dist/src/ruvector/moe-router.d.ts +0 -206
- package/dist/src/ruvector/moe-router.d.ts.map +0 -1
- package/dist/src/ruvector/moe-router.js +0 -626
- package/dist/src/ruvector/moe-router.js.map +0 -1
- package/dist/src/services/event-stream.d.ts +0 -25
- package/dist/src/services/event-stream.js +0 -27
- package/dist/src/services/loop-worker-runner.d.ts +0 -16
- package/dist/src/services/loop-worker-runner.js +0 -34
- package/dist/src/services/runtime-capabilities.d.ts +0 -22
- package/dist/src/services/runtime-capabilities.js +0 -45
|
@@ -1516,11 +1516,2361 @@ const benchmarkCommand = {
|
|
|
1516
1516
|
}
|
|
1517
1517
|
},
|
|
1518
1518
|
};
|
|
1519
|
+
// ============================================================================
|
|
1520
|
+
// ADR-148 — `neural router` subcommand tree: status / train / reload
|
|
1521
|
+
// ============================================================================
|
|
1522
|
+
const routerStatusCommand = {
|
|
1523
|
+
name: 'status',
|
|
1524
|
+
description: 'Show the cost-optimal neural router state (ADR-148) — gate, backend, artifact, counters',
|
|
1525
|
+
options: [
|
|
1526
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
1527
|
+
],
|
|
1528
|
+
examples: [
|
|
1529
|
+
{ command: 'claude-flow neural router status', description: 'Show router state' },
|
|
1530
|
+
{ command: 'CLAUDE_FLOW_ROUTER_NEURAL=1 claude-flow neural router status', description: 'Show status with gate open' },
|
|
1531
|
+
],
|
|
1532
|
+
action: async (ctx) => {
|
|
1533
|
+
const format = ctx.flags.format || 'table';
|
|
1534
|
+
const { neuralRouterStatus } = await import('../ruvector/neural-router.js');
|
|
1535
|
+
const { getModelRouterStats } = await import('../ruvector/model-router.js');
|
|
1536
|
+
const status = await neuralRouterStatus();
|
|
1537
|
+
const stats = getModelRouterStats();
|
|
1538
|
+
const payload = { neuralRouter: status, modelRouter: stats };
|
|
1539
|
+
if (format === 'json') {
|
|
1540
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
1541
|
+
return { success: true, data: payload };
|
|
1542
|
+
}
|
|
1543
|
+
output.writeln();
|
|
1544
|
+
output.writeln(output.bold('Cost-Optimal Neural Router (ADR-148)'));
|
|
1545
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
1546
|
+
output.writeln(` Gate (CLAUDE_FLOW_ROUTER_NEURAL=1): ${status.enabled ? output.success('open') : output.warning('closed')}`);
|
|
1547
|
+
output.writeln(` Backend available: ${status.available ? output.success('yes') : output.warning('no')}`);
|
|
1548
|
+
output.writeln(` Active backend (routedBy): ${status.routedBy ?? '—'}`);
|
|
1549
|
+
output.writeln(` Reason: ${status.reason}`);
|
|
1550
|
+
output.writeln(` Quality bar: ${status.config.qualityBar}`);
|
|
1551
|
+
output.writeln(` Seed corpus path: ${status.config.seedCorpusPath}`);
|
|
1552
|
+
output.writeln(` Bundled KRR artifact path: ${status.config.bundledKrrPath}`);
|
|
1553
|
+
output.writeln(` User artifact (modelPath): ${status.config.modelPath ?? '—'}`);
|
|
1554
|
+
output.writeln();
|
|
1555
|
+
output.writeln(output.bold('Counters (process-local since last reset)'));
|
|
1556
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
1557
|
+
output.writeln(` Total decisions: ${stats.totalDecisions}`);
|
|
1558
|
+
output.writeln(` routedBy: heuristic=${stats.routedByCounts.heuristic} hybrid=${stats.routedByCounts.hybrid} bandit-fallback=${stats.routedByCounts['bandit-fallback']}`);
|
|
1559
|
+
output.writeln(` neuralBackend: knn=${stats.neuralBackendCounts['metaharness-knn']} krr=${stats.neuralBackendCounts['metaharness-krr']} fastgrnn=${stats.neuralBackendCounts.fastgrnn}`);
|
|
1560
|
+
output.writeln(` A/B mode: ${stats.ab.comparisons} comparisons, ${stats.ab.disagreements} disagreements (${(stats.ab.disagreementRate * 100).toFixed(1)}%)`);
|
|
1561
|
+
output.writeln();
|
|
1562
|
+
return { success: true, data: payload };
|
|
1563
|
+
},
|
|
1564
|
+
};
|
|
1565
|
+
const routerTrainCommand = {
|
|
1566
|
+
name: 'train',
|
|
1567
|
+
description: 'Train a KRR router artifact from a DRACO-shaped JSON corpus (or the bundled seed) — pure TS, no native deps',
|
|
1568
|
+
options: [
|
|
1569
|
+
{ name: 'corpus', short: 'c', type: 'string', description: 'Path to DRACO rows JSON ({embedding, scores}). Defaults to the bundled seed corpus.' },
|
|
1570
|
+
{ name: 'out', short: 'o', type: 'string', description: 'Output path for the trained KRR JSON.' },
|
|
1571
|
+
{ name: 'quality-bar', short: 'q', type: 'number', description: 'qualityBar for cost-optimal selection (default 0.8)', default: '0.8' },
|
|
1572
|
+
],
|
|
1573
|
+
examples: [
|
|
1574
|
+
{ command: 'claude-flow neural router train', description: 'Train from the bundled seed and write to ./router.krr.json' },
|
|
1575
|
+
{ command: 'claude-flow neural router train -c ./my-corpus.json -o ./my-router.krr.json', description: 'Train from a custom corpus' },
|
|
1576
|
+
],
|
|
1577
|
+
action: async (ctx) => {
|
|
1578
|
+
const corpusPath = ctx.flags.corpus;
|
|
1579
|
+
const outPath = ctx.flags.out || './router.krr.json';
|
|
1580
|
+
const qualityBar = parseFloat(ctx.flags['quality-bar'] || '0.8') || 0.8;
|
|
1581
|
+
let mh;
|
|
1582
|
+
try {
|
|
1583
|
+
mh = await import('@metaharness/router');
|
|
1584
|
+
}
|
|
1585
|
+
catch {
|
|
1586
|
+
output.printError('@metaharness/router is not installed. `npm install @metaharness/router@^0.3.2` then re-run.');
|
|
1587
|
+
return { success: false, exitCode: 1 };
|
|
1588
|
+
}
|
|
1589
|
+
const { neuralRouterStatus } = await import('../ruvector/neural-router.js');
|
|
1590
|
+
const status = await neuralRouterStatus();
|
|
1591
|
+
const fs = await import('node:fs');
|
|
1592
|
+
const seedPath = corpusPath ?? status.config.seedCorpusPath;
|
|
1593
|
+
if (!fs.existsSync(seedPath)) {
|
|
1594
|
+
output.printError(`Corpus not found at ${seedPath}`);
|
|
1595
|
+
return { success: false, exitCode: 1 };
|
|
1596
|
+
}
|
|
1597
|
+
const rows = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
|
1598
|
+
output.writeln();
|
|
1599
|
+
output.writeln(output.bold('Training KRR router (ADR-148)'));
|
|
1600
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
1601
|
+
output.writeln(` Corpus: ${seedPath} (${rows.length} rows, dim=${rows[0]?.embedding?.length ?? '?'})`);
|
|
1602
|
+
output.writeln(` Output: ${outPath}`);
|
|
1603
|
+
output.writeln(` qualityBar: ${qualityBar}`);
|
|
1604
|
+
output.writeln();
|
|
1605
|
+
const spinner = output.createSpinner({ text: 'Fitting Beta-Bernoulli KRR with leave-one-out CV…', spinner: 'dots' });
|
|
1606
|
+
spinner.start();
|
|
1607
|
+
const t0 = performance.now();
|
|
1608
|
+
const { router, lambda, looQuality } = mh.trainRouter(rows, { haiku: 1, sonnet: 3, opus: 15 }, {
|
|
1609
|
+
qualityBar,
|
|
1610
|
+
lambdas: [1e-4, 1e-3, 1e-2, 1e-1, 1e0],
|
|
1611
|
+
});
|
|
1612
|
+
const ms = performance.now() - t0;
|
|
1613
|
+
spinner.succeed(`Trained in ${ms.toFixed(0)}ms (λ=${lambda.toExponential(3)}, looQuality=${looQuality.toFixed(4)})`);
|
|
1614
|
+
fs.writeFileSync(outPath, JSON.stringify(router.toJSON()));
|
|
1615
|
+
const bytes = fs.statSync(outPath).size;
|
|
1616
|
+
output.writeln(`Wrote ${bytes} bytes → ${outPath}`);
|
|
1617
|
+
output.writeln();
|
|
1618
|
+
output.writeln(output.dim('Use: CLAUDE_FLOW_ROUTER_NEURAL=1 CLAUDE_FLOW_ROUTER_MODEL_PATH=' + outPath + ' …'));
|
|
1619
|
+
output.writeln();
|
|
1620
|
+
return { success: true, data: { lambda, looQuality, trainMs: ms, modelPath: outPath, modelBytes: bytes } };
|
|
1621
|
+
},
|
|
1622
|
+
};
|
|
1623
|
+
// ADR-149 iter 19 — CLI surface for the iter 18 trajectory consumer. Pairs
|
|
1624
|
+
// decision+outcome rows from the JSONL recorder into seed-rows-compatible
|
|
1625
|
+
// training rows that `neural router train` (above) can consume.
|
|
1626
|
+
const routerTrainFromTrajectoriesCommand = {
|
|
1627
|
+
name: 'train-from-trajectories',
|
|
1628
|
+
description: 'Pair production decision+outcome JSONL rows into seed-corpus-shaped training rows (ADR-149 iter 18/19)',
|
|
1629
|
+
options: [
|
|
1630
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/model-router-trajectories.jsonl)' },
|
|
1631
|
+
{ name: 'write', short: 'w', type: 'string', description: 'Write paired rows to this path (seed-rows.json-compatible JSON array)' },
|
|
1632
|
+
{ name: 'union', short: 'u', type: 'string', description: 'Union paired rows with an existing seed-rows.json — production rows win on task-text collision' },
|
|
1633
|
+
{ name: 'filter-source', type: 'string', description: 'Only keep outcomes whose source matches (e.g. llm-judge, agent-execute)' },
|
|
1634
|
+
{ name: 'min-quality', type: 'number', description: 'Drop pairs whose MAX outcome score is below this threshold (default 0 — keep failures, they are training signal)', default: '0' },
|
|
1635
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
1636
|
+
],
|
|
1637
|
+
examples: [
|
|
1638
|
+
{ command: 'claude-flow neural router train-from-trajectories', description: 'Show pairing stats from default trajectory path' },
|
|
1639
|
+
{ command: 'claude-flow neural router train-from-trajectories -w production-rows.json', description: 'Emit a corpus to feed `router train`' },
|
|
1640
|
+
{ command: 'claude-flow neural router train-from-trajectories -u assets/model-router/seed-rows.json -w merged.json', description: 'Union production rows with the bundled seed corpus' },
|
|
1641
|
+
{ command: 'claude-flow neural router train-from-trajectories --filter-source llm-judge -w high-signal.json', description: 'Keep only judge-graded rows (drop coarse agent-execute baseline)' },
|
|
1642
|
+
],
|
|
1643
|
+
action: async (ctx) => {
|
|
1644
|
+
const fs = await import('node:fs');
|
|
1645
|
+
const path = await import('node:path');
|
|
1646
|
+
const { pairTrajectoryRows } = await import('../ruvector/router-trajectory.js');
|
|
1647
|
+
const inPath = ctx.flags.in
|
|
1648
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
1649
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
1650
|
+
const writePath = ctx.flags.write;
|
|
1651
|
+
const unionPath = ctx.flags.union;
|
|
1652
|
+
// Argv parser may camelCase hyphenated flags — accept both spellings.
|
|
1653
|
+
const filterSource = (ctx.flags['filter-source'] ?? ctx.flags.filterSource);
|
|
1654
|
+
const minQuality = parseFloat((ctx.flags['min-quality'] ?? ctx.flags.minQuality) || '0') || 0;
|
|
1655
|
+
const fmt = ctx.flags.format || 'table';
|
|
1656
|
+
if (!fs.existsSync(inPath)) {
|
|
1657
|
+
const msg = `Trajectory file not found at ${inPath}`;
|
|
1658
|
+
if (fmt === 'json') {
|
|
1659
|
+
output.writeln(JSON.stringify({ error: msg, hint: 'Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 and run any agent_spawn → executeAgentTask flow to accumulate rows.' }, null, 2));
|
|
1660
|
+
}
|
|
1661
|
+
else {
|
|
1662
|
+
output.printError(msg);
|
|
1663
|
+
output.writeln(output.dim(' Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 to enable trajectory recording.'));
|
|
1664
|
+
}
|
|
1665
|
+
return { success: false, exitCode: 1 };
|
|
1666
|
+
}
|
|
1667
|
+
const text = fs.readFileSync(inPath, 'utf8');
|
|
1668
|
+
const lines = text.split('\n').filter(l => l.trim().length > 0);
|
|
1669
|
+
const allRows = [];
|
|
1670
|
+
let malformed = 0;
|
|
1671
|
+
for (const l of lines) {
|
|
1672
|
+
try {
|
|
1673
|
+
allRows.push(JSON.parse(l));
|
|
1674
|
+
}
|
|
1675
|
+
catch {
|
|
1676
|
+
malformed++;
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
const { pairs: rawPairs, stats } = pairTrajectoryRows(allRows);
|
|
1680
|
+
let pairs = rawPairs;
|
|
1681
|
+
let filteredCount = pairs.length;
|
|
1682
|
+
if (filterSource)
|
|
1683
|
+
pairs = pairs.filter(p => p.source === filterSource);
|
|
1684
|
+
if (minQuality > 0)
|
|
1685
|
+
pairs = pairs.filter(p => Math.max(...Object.values(p.scores)) >= minQuality);
|
|
1686
|
+
filteredCount = pairs.length;
|
|
1687
|
+
const corpusRows = pairs.map(p => ({
|
|
1688
|
+
task: p.task,
|
|
1689
|
+
embedding: p.embedding,
|
|
1690
|
+
scores: p.scores,
|
|
1691
|
+
tier: p.tier,
|
|
1692
|
+
}));
|
|
1693
|
+
let unionRows = corpusRows;
|
|
1694
|
+
let unioned = false;
|
|
1695
|
+
let seedKept = 0;
|
|
1696
|
+
if (unionPath) {
|
|
1697
|
+
if (!fs.existsSync(unionPath)) {
|
|
1698
|
+
output.printError(`--union path ${unionPath} not found`);
|
|
1699
|
+
return { success: false, exitCode: 1 };
|
|
1700
|
+
}
|
|
1701
|
+
const seedRows = JSON.parse(fs.readFileSync(unionPath, 'utf8'));
|
|
1702
|
+
const productionTasks = new Set(corpusRows.map(r => r.task));
|
|
1703
|
+
const kept = seedRows.filter(r => !productionTasks.has(r.task));
|
|
1704
|
+
seedKept = kept.length;
|
|
1705
|
+
unionRows = [...kept, ...corpusRows];
|
|
1706
|
+
unioned = true;
|
|
1707
|
+
}
|
|
1708
|
+
if (writePath) {
|
|
1709
|
+
fs.writeFileSync(writePath, JSON.stringify(unionRows));
|
|
1710
|
+
}
|
|
1711
|
+
const data = {
|
|
1712
|
+
input: inPath,
|
|
1713
|
+
malformed,
|
|
1714
|
+
stats,
|
|
1715
|
+
afterFilters: filteredCount,
|
|
1716
|
+
unioned,
|
|
1717
|
+
seedKept: unioned ? seedKept : undefined,
|
|
1718
|
+
finalRows: unionRows.length,
|
|
1719
|
+
written: writePath,
|
|
1720
|
+
};
|
|
1721
|
+
if (fmt === 'json') {
|
|
1722
|
+
output.writeln(JSON.stringify(data, null, 2));
|
|
1723
|
+
return { success: true, data };
|
|
1724
|
+
}
|
|
1725
|
+
output.writeln();
|
|
1726
|
+
output.writeln(output.bold('Trajectory → Training-row pairing (ADR-149 iter 18/19)'));
|
|
1727
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
1728
|
+
output.writeln(` Input file: ${inPath}`);
|
|
1729
|
+
output.writeln(` Total rows: ${stats.totalRows} (${stats.decisions} decision, ${stats.outcomes} outcome${malformed > 0 ? `, ${malformed} malformed` : ''})`);
|
|
1730
|
+
output.writeln(` Paired: ${stats.paired}`);
|
|
1731
|
+
output.writeln(` Dropped (no embed): ${stats.droppedNoEmbedding}`);
|
|
1732
|
+
output.writeln(` Dropped (no match): ${stats.droppedNoMatch}`);
|
|
1733
|
+
if (Object.keys(stats.bySource).length > 0)
|
|
1734
|
+
output.writeln(` By source: ${JSON.stringify(stats.bySource)}`);
|
|
1735
|
+
if (Object.keys(stats.byTier).length > 0)
|
|
1736
|
+
output.writeln(` By tier: ${JSON.stringify(stats.byTier)}`);
|
|
1737
|
+
output.writeln(` After filters: ${filteredCount}`);
|
|
1738
|
+
if (unioned)
|
|
1739
|
+
output.writeln(` Final (unioned): ${unionRows.length} (seed kept=${seedKept}, production=${corpusRows.length})`);
|
|
1740
|
+
if (writePath)
|
|
1741
|
+
output.writeln(` Written: ${writePath}`);
|
|
1742
|
+
output.writeln();
|
|
1743
|
+
if (stats.totalRows === 0) {
|
|
1744
|
+
output.writeln(output.dim(' Empty trajectory file. Enable recording with `export CLAUDE_FLOW_ROUTER_TRAJECTORY=1` and run agent_spawn flows.'));
|
|
1745
|
+
}
|
|
1746
|
+
else if (pairs.length > 0 && writePath) {
|
|
1747
|
+
output.writeln(output.dim(` Next: claude-flow neural router train -c ${writePath} -o router.krr.json`));
|
|
1748
|
+
}
|
|
1749
|
+
output.writeln();
|
|
1750
|
+
return { success: true, data };
|
|
1751
|
+
},
|
|
1752
|
+
};
|
|
1753
|
+
const routerReloadCommand = {
|
|
1754
|
+
name: 'reload',
|
|
1755
|
+
description: 'Force-reload the neural router (clears in-process backend cache; next call re-reads artifact/corpus)',
|
|
1756
|
+
options: [],
|
|
1757
|
+
examples: [
|
|
1758
|
+
{ command: 'claude-flow neural router reload', description: 'Refresh backend caches after retraining an artifact' },
|
|
1759
|
+
],
|
|
1760
|
+
action: async () => {
|
|
1761
|
+
const { __resetNeuralRouterForTests, neuralRouterStatus } = await import('../ruvector/neural-router.js');
|
|
1762
|
+
__resetNeuralRouterForTests();
|
|
1763
|
+
output.writeln(output.success('Neural router backend cache cleared.'));
|
|
1764
|
+
const status = await neuralRouterStatus();
|
|
1765
|
+
output.writeln(output.dim(` Active backend now: ${status.routedBy ?? '—'} (${status.reason})`));
|
|
1766
|
+
return { success: true, data: status };
|
|
1767
|
+
},
|
|
1768
|
+
};
|
|
1769
|
+
/**
|
|
1770
|
+
* ADR-149 iter 8 — `neural router models`: list the candidate registry
|
|
1771
|
+
* with measured per-tier scores from the most recent seed-corpus bench,
|
|
1772
|
+
* latency, and cost. Reads:
|
|
1773
|
+
* - assets/model-router/seed-rows.json (which model ids exist in the corpus)
|
|
1774
|
+
* - latest docs/benchmarks/runs/seed-corpus-*.json (per-candidate aggregates)
|
|
1775
|
+
* - assets/model-router/openrouter-alts.json (tier mapping + alt rankings)
|
|
1776
|
+
* Falls back gracefully when files are missing.
|
|
1777
|
+
*/
|
|
1778
|
+
const routerModelsCommand = {
|
|
1779
|
+
name: 'models',
|
|
1780
|
+
description: 'List the cost-optimal router registry with measured per-tier scores, latency, and cost (ADR-149)',
|
|
1781
|
+
options: [
|
|
1782
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
1783
|
+
],
|
|
1784
|
+
examples: [
|
|
1785
|
+
{ command: 'claude-flow neural router models', description: 'Show candidate registry with measured stats' },
|
|
1786
|
+
{ command: 'claude-flow neural router models -f json', description: 'Machine-readable JSON' },
|
|
1787
|
+
],
|
|
1788
|
+
action: async (ctx) => {
|
|
1789
|
+
const fmt = ctx.flags.format || 'table';
|
|
1790
|
+
const fs = await import('node:fs');
|
|
1791
|
+
const path = await import('node:path');
|
|
1792
|
+
const { neuralRouterStatus } = await import('../ruvector/neural-router.js');
|
|
1793
|
+
const status = await neuralRouterStatus();
|
|
1794
|
+
const seedPath = status.config.seedCorpusPath;
|
|
1795
|
+
// Best-effort: the candidate set is the union of ids appearing in
|
|
1796
|
+
// seed-rows.json scores keys. The latest measurement run's
|
|
1797
|
+
// perCandidate array is the authoritative source for measured stats.
|
|
1798
|
+
const candidates = new Map();
|
|
1799
|
+
try {
|
|
1800
|
+
if (fs.existsSync(seedPath)) {
|
|
1801
|
+
const rows = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
|
1802
|
+
for (const r of rows) {
|
|
1803
|
+
for (const id of Object.keys(r.scores ?? {})) {
|
|
1804
|
+
if (!candidates.has(id))
|
|
1805
|
+
candidates.set(id, { id, tier: 'unknown' });
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
catch { /* keep going */ }
|
|
1811
|
+
// Pull latest FULL-CORPUS measurement for per-tier scores + latency.
|
|
1812
|
+
// Prefer files with cheap+mid+strong all populated (40-row+ runs); fall
|
|
1813
|
+
// back to the most-recent partial file if no full run is found.
|
|
1814
|
+
try {
|
|
1815
|
+
const benchDir = path.resolve(process.cwd(), 'docs', 'benchmarks', 'runs');
|
|
1816
|
+
if (fs.existsSync(benchDir)) {
|
|
1817
|
+
const files = fs.readdirSync(benchDir)
|
|
1818
|
+
.filter(f => f.startsWith('seed-corpus-') && f.endsWith('.json'))
|
|
1819
|
+
.sort()
|
|
1820
|
+
.reverse();
|
|
1821
|
+
// Scan files newest-first; pick the first one that has all three
|
|
1822
|
+
// tiers populated for the median model.
|
|
1823
|
+
let chosen = null;
|
|
1824
|
+
for (const f of files) {
|
|
1825
|
+
try {
|
|
1826
|
+
const data = JSON.parse(fs.readFileSync(path.join(benchDir, f), 'utf8'));
|
|
1827
|
+
const sample = data.perCandidate?.[0];
|
|
1828
|
+
if (sample && sample.cheap_avg_score != null && sample.mid_avg_score != null && sample.strong_avg_score != null) {
|
|
1829
|
+
chosen = data;
|
|
1830
|
+
break;
|
|
1831
|
+
}
|
|
1832
|
+
if (!chosen)
|
|
1833
|
+
chosen = data; // fallback to newest
|
|
1834
|
+
}
|
|
1835
|
+
catch { /* skip malformed */ }
|
|
1836
|
+
}
|
|
1837
|
+
if (chosen) {
|
|
1838
|
+
for (const raw of chosen.perCandidate ?? []) {
|
|
1839
|
+
const r = raw;
|
|
1840
|
+
const existing = candidates.get(r.id);
|
|
1841
|
+
const c = existing ?? { id: r.id, tier: 'unknown' };
|
|
1842
|
+
c.tier = (r.tier ?? c.tier) || 'unknown';
|
|
1843
|
+
c.cost_in = r.cost_per_m_tok_in;
|
|
1844
|
+
c.cost_out = r.cost_per_m_tok_out;
|
|
1845
|
+
c.cheap = r.cheap_avg_score ?? undefined;
|
|
1846
|
+
c.mid = r.mid_avg_score ?? undefined;
|
|
1847
|
+
c.strong = r.strong_avg_score ?? undefined;
|
|
1848
|
+
c.overall = r.overall_avg_score ?? undefined;
|
|
1849
|
+
c.latency_ms = r.latency_mean_ms ?? undefined;
|
|
1850
|
+
candidates.set(r.id, c);
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
catch { /* keep going */ }
|
|
1856
|
+
const rows = Array.from(candidates.values()).sort((a, b) => (b.overall ?? -1) - (a.overall ?? -1));
|
|
1857
|
+
if (fmt === 'json') {
|
|
1858
|
+
output.writeln(JSON.stringify({ count: rows.length, candidates: rows }, null, 2));
|
|
1859
|
+
return { success: true, data: { count: rows.length, candidates: rows } };
|
|
1860
|
+
}
|
|
1861
|
+
output.writeln();
|
|
1862
|
+
output.writeln(output.bold('Cost-Optimal Router Registry (ADR-149)'));
|
|
1863
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
1864
|
+
output.writeln(` Source: ${seedPath}`);
|
|
1865
|
+
output.writeln(` Candidates: ${rows.length}`);
|
|
1866
|
+
output.writeln();
|
|
1867
|
+
if (rows.length === 0) {
|
|
1868
|
+
output.writeln(output.warning(' No candidates — corpus not generated or measurement not run.'));
|
|
1869
|
+
output.writeln(output.dim(' Run: node scripts/gen-seed-corpus-v2.mjs && OPENROUTER_API_KEY=... node scripts/benchmark-seed-corpus.mjs --live'));
|
|
1870
|
+
return { success: true, data: { count: 0 } };
|
|
1871
|
+
}
|
|
1872
|
+
const fmtPct = (v) => v == null ? '—'.padStart(6) : `${(v * 100).toFixed(1)}%`.padStart(6);
|
|
1873
|
+
const fmtCost = (v) => v == null ? '—' : `$${v.toFixed(2)}`;
|
|
1874
|
+
const fmtLat = (v) => v == null ? '—' : `${v.toFixed(0)}ms`;
|
|
1875
|
+
output.writeln(' | Candidate | Tier | Cheap | Mid | Strong | Overall | $/Mtok in/out | Latency |');
|
|
1876
|
+
output.writeln(' |--------------------------------------------|--------|--------|--------|--------|---------|---------------|---------|');
|
|
1877
|
+
for (const c of rows) {
|
|
1878
|
+
output.writeln(` | ${c.id.padEnd(42)} | ${c.tier.padEnd(6)} | ${fmtPct(c.cheap)} | ${fmtPct(c.mid)} | ${fmtPct(c.strong)} | ${fmtPct(c.overall)} | ${fmtCost(c.cost_in).padStart(5)}/${fmtCost(c.cost_out).padEnd(6)} | ${fmtLat(c.latency_ms).padStart(7)} |`);
|
|
1879
|
+
}
|
|
1880
|
+
output.writeln();
|
|
1881
|
+
output.writeln(output.dim(' Sorted by overall score desc. Empty cells = no measurement on that tier.'));
|
|
1882
|
+
output.writeln(output.dim(' To re-measure: OPENROUTER_API_KEY=... node scripts/benchmark-seed-corpus.mjs --live'));
|
|
1883
|
+
output.writeln();
|
|
1884
|
+
return { success: true, data: { count: rows.length, candidates: rows } };
|
|
1885
|
+
},
|
|
1886
|
+
};
|
|
1887
|
+
// ADR-149 iter 28 — observability into recorded routing decisions. The
|
|
1888
|
+
// trajectory JSONL (iter 17+) records every decision the router makes;
|
|
1889
|
+
// this subcommand exposes that data as filtered + aggregated views so
|
|
1890
|
+
// operators don't have to grep the file.
|
|
1891
|
+
const routerDecisionsCommand = {
|
|
1892
|
+
name: 'decisions',
|
|
1893
|
+
description: 'Query the routing-decision JSONL (iter 17+): filter, aggregate, paginate (ADR-149 iter 28)',
|
|
1894
|
+
options: [
|
|
1895
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/model-router-trajectories.jsonl)' },
|
|
1896
|
+
{ name: 'since', short: 's', type: 'string', description: 'Time window suffix: 1h, 24h, 7d, 30d (default: all)' },
|
|
1897
|
+
{ name: 'routed-by', type: 'string', description: 'Filter by decision mechanism: hybrid | bandit-fallback | heuristic' },
|
|
1898
|
+
{ name: 'model', short: 'm', type: 'string', description: 'Filter by chosen model id (substring match, e.g. haiku, gpt-4)' },
|
|
1899
|
+
{ name: 'bucket', type: 'string', description: 'Filter by complexity bucket: cheap (< 0.34) | mid (< 0.67) | strong (≥ 0.67) — iter 58' },
|
|
1900
|
+
{ name: 'task-hash', type: 'string', description: 'Filter by exact task_hash (FNV-1a-32). Use for incident investigation — iter 59' },
|
|
1901
|
+
{ name: 'limit', short: 'l', type: 'number', description: 'Max recent decisions to list (default 20)', default: '20' },
|
|
1902
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
1903
|
+
],
|
|
1904
|
+
examples: [
|
|
1905
|
+
{ command: 'claude-flow neural router decisions', description: 'Aggregate stats + 20 most-recent decisions' },
|
|
1906
|
+
{ command: 'claude-flow neural router decisions --since 24h', description: 'Last 24 hours only' },
|
|
1907
|
+
{ command: 'claude-flow neural router decisions --routed-by bandit-fallback', description: 'Find decisions where neural backend failed' },
|
|
1908
|
+
{ command: 'claude-flow neural router decisions --model haiku --format json', description: 'All haiku picks, JSON output' },
|
|
1909
|
+
],
|
|
1910
|
+
action: async (ctx) => {
|
|
1911
|
+
const fs = await import('node:fs');
|
|
1912
|
+
const path = await import('node:path');
|
|
1913
|
+
const inPath = ctx.flags.in
|
|
1914
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
1915
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
1916
|
+
const since = ctx.flags.since;
|
|
1917
|
+
const routedByFilter = (ctx.flags['routed-by'] ?? ctx.flags.routedBy);
|
|
1918
|
+
const modelFilter = ctx.flags.model;
|
|
1919
|
+
const bucketFilterRaw = ctx.flags.bucket?.toLowerCase();
|
|
1920
|
+
const bucketFilter = bucketFilterRaw === 'cheap' || bucketFilterRaw === 'mid' || bucketFilterRaw === 'strong'
|
|
1921
|
+
? bucketFilterRaw : undefined;
|
|
1922
|
+
if (bucketFilterRaw && !bucketFilter) {
|
|
1923
|
+
output.printError(`--bucket must be one of: cheap | mid | strong (got "${bucketFilterRaw}")`);
|
|
1924
|
+
return { success: false, exitCode: 1 };
|
|
1925
|
+
}
|
|
1926
|
+
// iter 59 — task_hash filter for incident investigation. Accepts the 8-char
|
|
1927
|
+
// FNV-1a-32 hex format the trajectory recorder uses.
|
|
1928
|
+
const taskHashFilter = (ctx.flags['task-hash'] ?? ctx.flags.taskHash)?.toLowerCase();
|
|
1929
|
+
if (taskHashFilter && !/^[0-9a-f]{8}$/.test(taskHashFilter)) {
|
|
1930
|
+
output.printError(`--task-hash must be an 8-char hex string (got "${taskHashFilter}")`);
|
|
1931
|
+
return { success: false, exitCode: 1 };
|
|
1932
|
+
}
|
|
1933
|
+
const limit = parseInt(ctx.flags.limit || '20', 10) || 20;
|
|
1934
|
+
const fmt = ctx.flags.format || 'table';
|
|
1935
|
+
if (!fs.existsSync(inPath)) {
|
|
1936
|
+
const msg = `Trajectory file not found at ${inPath}`;
|
|
1937
|
+
if (fmt === 'json') {
|
|
1938
|
+
output.writeln(JSON.stringify({ error: msg, hint: 'Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 to enable recording.' }, null, 2));
|
|
1939
|
+
}
|
|
1940
|
+
else {
|
|
1941
|
+
output.printError(msg);
|
|
1942
|
+
output.writeln(output.dim(' Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 to enable trajectory recording.'));
|
|
1943
|
+
}
|
|
1944
|
+
return { success: false, exitCode: 1 };
|
|
1945
|
+
}
|
|
1946
|
+
const text = fs.readFileSync(inPath, 'utf8');
|
|
1947
|
+
const lines = text.split('\n').filter(l => l.trim().length > 0);
|
|
1948
|
+
const decisions = [];
|
|
1949
|
+
const outcomesByHash = new Map();
|
|
1950
|
+
let malformed = 0;
|
|
1951
|
+
for (const l of lines) {
|
|
1952
|
+
try {
|
|
1953
|
+
const row = JSON.parse(l);
|
|
1954
|
+
if (row.type === 'decision') {
|
|
1955
|
+
decisions.push(row);
|
|
1956
|
+
}
|
|
1957
|
+
else if (row.type === 'outcome') {
|
|
1958
|
+
const arr = outcomesByHash.get(row.task_hash) ?? [];
|
|
1959
|
+
arr.push({
|
|
1960
|
+
task_hash: row.task_hash,
|
|
1961
|
+
ts: row.ts,
|
|
1962
|
+
quality: row.quality,
|
|
1963
|
+
cost_usd: row.cost_usd,
|
|
1964
|
+
tokens: row.tokens,
|
|
1965
|
+
model_id: row.model_id,
|
|
1966
|
+
});
|
|
1967
|
+
outcomesByHash.set(row.task_hash, arr);
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
catch {
|
|
1971
|
+
malformed++;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
// Time-window filter.
|
|
1975
|
+
let cutoffMs = null;
|
|
1976
|
+
if (since) {
|
|
1977
|
+
const m = since.match(/^(\d+)([hdmw])$/);
|
|
1978
|
+
if (m) {
|
|
1979
|
+
const n = parseInt(m[1], 10);
|
|
1980
|
+
const unitMs = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]] ?? 0;
|
|
1981
|
+
cutoffMs = Date.now() - n * unitMs;
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
let filtered = decisions;
|
|
1985
|
+
if (cutoffMs !== null) {
|
|
1986
|
+
filtered = filtered.filter(d => Date.parse(d.ts) >= cutoffMs);
|
|
1987
|
+
}
|
|
1988
|
+
if (routedByFilter) {
|
|
1989
|
+
filtered = filtered.filter(d => d.routed_by === routedByFilter);
|
|
1990
|
+
}
|
|
1991
|
+
if (bucketFilter) {
|
|
1992
|
+
filtered = filtered.filter(d => {
|
|
1993
|
+
const bucket = d.complexity < 0.34 ? 'cheap' : d.complexity < 0.67 ? 'mid' : 'strong';
|
|
1994
|
+
return bucket === bucketFilter;
|
|
1995
|
+
});
|
|
1996
|
+
}
|
|
1997
|
+
if (taskHashFilter) {
|
|
1998
|
+
filtered = filtered.filter(d => d.task_hash === taskHashFilter);
|
|
1999
|
+
}
|
|
2000
|
+
if (modelFilter) {
|
|
2001
|
+
const needle = modelFilter.toLowerCase();
|
|
2002
|
+
filtered = filtered.filter(d => {
|
|
2003
|
+
const id = (d.openrouter_model ?? d.model).toLowerCase();
|
|
2004
|
+
return id.includes(needle);
|
|
2005
|
+
});
|
|
2006
|
+
}
|
|
2007
|
+
// Aggregate.
|
|
2008
|
+
const byRoutedBy = {};
|
|
2009
|
+
const byModel = {};
|
|
2010
|
+
const byTier = { cheap: 0, mid: 0, strong: 0 };
|
|
2011
|
+
// iter 31 — cost aggregation. Sum cost_usd from paired outcome rows.
|
|
2012
|
+
let costTotalUsd = 0;
|
|
2013
|
+
let costPairedCount = 0;
|
|
2014
|
+
const costByModel = {};
|
|
2015
|
+
const costByTier = { cheap: 0, mid: 0, strong: 0 };
|
|
2016
|
+
// iter 65 — track per-hash decision-iteration so we can pair the N-th
|
|
2017
|
+
// decision for a hash with the N-th outcome (instead of always pulling
|
|
2018
|
+
// the latest outcome). Iter 17's recorder writes decision then outcome
|
|
2019
|
+
// in chronological order, so this index-pairing matches what the agent
|
|
2020
|
+
// actually dispatched.
|
|
2021
|
+
const decisionIndexByHash = new Map();
|
|
2022
|
+
for (const d of filtered) {
|
|
2023
|
+
byRoutedBy[d.routed_by] = (byRoutedBy[d.routed_by] ?? 0) + 1;
|
|
2024
|
+
const id = d.openrouter_model ?? d.model;
|
|
2025
|
+
byModel[id] = (byModel[id] ?? 0) + 1;
|
|
2026
|
+
const tier = d.complexity < 0.34 ? 'cheap' : d.complexity < 0.67 ? 'mid' : 'strong';
|
|
2027
|
+
byTier[tier]++;
|
|
2028
|
+
// iter 31 + iter 65 — JOIN to the i-th OUTCOME row for this task_hash
|
|
2029
|
+
// (where i is the i-th DECISION for this hash). Avoids double-counting
|
|
2030
|
+
// costs when a task ran multiple times.
|
|
2031
|
+
const outcomeArr = outcomesByHash.get(d.task_hash);
|
|
2032
|
+
if (outcomeArr && outcomeArr.length > 0) {
|
|
2033
|
+
const seen = decisionIndexByHash.get(d.task_hash) ?? 0;
|
|
2034
|
+
const out = outcomeArr[Math.min(seen, outcomeArr.length - 1)];
|
|
2035
|
+
decisionIndexByHash.set(d.task_hash, seen + 1);
|
|
2036
|
+
if (out?.cost_usd != null) {
|
|
2037
|
+
costTotalUsd += out.cost_usd;
|
|
2038
|
+
costPairedCount++;
|
|
2039
|
+
const modelKey = out.model_id ?? id;
|
|
2040
|
+
costByModel[modelKey] = (costByModel[modelKey] ?? 0) + out.cost_usd;
|
|
2041
|
+
costByTier[tier] += out.cost_usd;
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
const fallbackRate = filtered.length > 0
|
|
2046
|
+
? ((byRoutedBy['bandit-fallback'] ?? 0) / filtered.length) * 100
|
|
2047
|
+
: 0;
|
|
2048
|
+
// Sort by ts ascending so "most recent" is well-defined regardless of
|
|
2049
|
+
// file order (rotation, concurrent writes can break monotonicity).
|
|
2050
|
+
const sorted = [...filtered].sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2051
|
+
const recent = sorted.slice(-limit).reverse(); // newest first
|
|
2052
|
+
const payload = {
|
|
2053
|
+
input: inPath,
|
|
2054
|
+
totalRows: lines.length,
|
|
2055
|
+
decisionRows: decisions.length,
|
|
2056
|
+
malformed,
|
|
2057
|
+
filtered: filtered.length,
|
|
2058
|
+
filters: { since, routedBy: routedByFilter, model: modelFilter, bucket: bucketFilter, taskHash: taskHashFilter },
|
|
2059
|
+
aggregates: {
|
|
2060
|
+
byRoutedBy, byModel, byTier,
|
|
2061
|
+
fallbackRatePct: Math.round(fallbackRate * 100) / 100,
|
|
2062
|
+
// iter 31 — cost aggregates. Only populated when outcome rows
|
|
2063
|
+
// carry cost_usd (post-iter-31 trajectories).
|
|
2064
|
+
...(costPairedCount > 0 ? {
|
|
2065
|
+
costTotalUsd: Math.round(costTotalUsd * 1000000) / 1000000,
|
|
2066
|
+
costPairedCount,
|
|
2067
|
+
costByModel: Object.fromEntries(Object.entries(costByModel).map(([k, v]) => [k, Math.round(v * 1000000) / 1000000])),
|
|
2068
|
+
costByTier: Object.fromEntries(Object.entries(costByTier).map(([k, v]) => [k, Math.round(v * 1000000) / 1000000])),
|
|
2069
|
+
avgCostPerCall: Math.round((costTotalUsd / costPairedCount) * 1000000) / 1000000,
|
|
2070
|
+
} : {}),
|
|
2071
|
+
},
|
|
2072
|
+
recent,
|
|
2073
|
+
};
|
|
2074
|
+
if (fmt === 'json') {
|
|
2075
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
2076
|
+
return { success: true, data: payload };
|
|
2077
|
+
}
|
|
2078
|
+
output.writeln();
|
|
2079
|
+
output.writeln(output.bold('Routing decisions (ADR-149 iter 17+, query iter 28)'));
|
|
2080
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
2081
|
+
output.writeln(` Input: ${inPath}`);
|
|
2082
|
+
output.writeln(` JSONL rows: ${lines.length} (${decisions.length} decision, ${malformed} malformed)`);
|
|
2083
|
+
if (since || routedByFilter || modelFilter || bucketFilter || taskHashFilter) {
|
|
2084
|
+
output.writeln(` Filters: ${[since && `since=${since}`, routedByFilter && `routed-by=${routedByFilter}`, modelFilter && `model~${modelFilter}`, bucketFilter && `bucket=${bucketFilter}`, taskHashFilter && `task_hash=${taskHashFilter}`].filter(Boolean).join(', ')}`);
|
|
2085
|
+
}
|
|
2086
|
+
output.writeln(` After filters: ${filtered.length}`);
|
|
2087
|
+
output.writeln('');
|
|
2088
|
+
if (filtered.length === 0) {
|
|
2089
|
+
output.writeln(output.dim(' No decisions match the filters.'));
|
|
2090
|
+
output.writeln('');
|
|
2091
|
+
return { success: true, data: payload };
|
|
2092
|
+
}
|
|
2093
|
+
output.writeln(` Fallback rate: ${fallbackRate.toFixed(2)}% (neural backend → bandit when prediction unusable)`);
|
|
2094
|
+
output.writeln('');
|
|
2095
|
+
output.writeln(' By routed_by:');
|
|
2096
|
+
for (const [k, v] of Object.entries(byRoutedBy).sort((a, b) => b[1] - a[1])) {
|
|
2097
|
+
const pct = ((v / filtered.length) * 100).toFixed(1).padStart(5);
|
|
2098
|
+
output.writeln(` ${k.padEnd(18)} ${String(v).padStart(6)} ${pct}%`);
|
|
2099
|
+
}
|
|
2100
|
+
output.writeln('');
|
|
2101
|
+
output.writeln(' By model:');
|
|
2102
|
+
for (const [k, v] of Object.entries(byModel).sort((a, b) => b[1] - a[1])) {
|
|
2103
|
+
const pct = ((v / filtered.length) * 100).toFixed(1).padStart(5);
|
|
2104
|
+
output.writeln(` ${k.padEnd(42)} ${String(v).padStart(6)} ${pct}%`);
|
|
2105
|
+
}
|
|
2106
|
+
output.writeln('');
|
|
2107
|
+
output.writeln(' By tier (complexity bucket):');
|
|
2108
|
+
for (const [k, v] of Object.entries(byTier)) {
|
|
2109
|
+
const pct = filtered.length > 0 ? ((v / filtered.length) * 100).toFixed(1).padStart(5) : ' 0.0';
|
|
2110
|
+
output.writeln(` ${k.padEnd(8)} ${String(v).padStart(6)} ${pct}%`);
|
|
2111
|
+
}
|
|
2112
|
+
output.writeln('');
|
|
2113
|
+
// iter 31 — cost block (only when outcome rows carry cost_usd).
|
|
2114
|
+
if (costPairedCount > 0) {
|
|
2115
|
+
output.writeln(' Cost (USD, from paired outcomes):');
|
|
2116
|
+
output.writeln(` Total: $${costTotalUsd.toFixed(4)} across ${costPairedCount} paired decisions`);
|
|
2117
|
+
output.writeln(` Avg per call: $${(costTotalUsd / costPairedCount).toFixed(6)}`);
|
|
2118
|
+
output.writeln(' By model:');
|
|
2119
|
+
for (const [k, v] of Object.entries(costByModel).sort((a, b) => b[1] - a[1])) {
|
|
2120
|
+
output.writeln(` ${k.padEnd(40)} $${v.toFixed(4)}`);
|
|
2121
|
+
}
|
|
2122
|
+
output.writeln(' By tier:');
|
|
2123
|
+
for (const [k, v] of Object.entries(costByTier)) {
|
|
2124
|
+
output.writeln(` ${k.padEnd(8)} $${v.toFixed(4)}`);
|
|
2125
|
+
}
|
|
2126
|
+
output.writeln('');
|
|
2127
|
+
}
|
|
2128
|
+
// iter 59 — incident detail mode. When --task-hash is set, render each
|
|
2129
|
+
// matching decision with FULL task text, paired outcome, complexity,
|
|
2130
|
+
// ab_pair, ensemble_disagreement. Operators investigating "why was
|
|
2131
|
+
// THIS task routed to X?" want maximum context per decision.
|
|
2132
|
+
if (taskHashFilter && filtered.length > 0) {
|
|
2133
|
+
// iter 64 — sort occurrences newest-first by ts. JSONL insertion order
|
|
2134
|
+
// may not match chronological order after rotation or out-of-order writes.
|
|
2135
|
+
const incidentSorted = [...filtered].sort((a, b) => b.ts.localeCompare(a.ts));
|
|
2136
|
+
output.writeln(output.bold(` Incident detail for task_hash=${taskHashFilter} (${filtered.length} occurrence(s), newest first):`));
|
|
2137
|
+
for (const d of incidentSorted) {
|
|
2138
|
+
// iter 65 — outcomesByHash is now an array of all outcomes per hash.
|
|
2139
|
+
// For the incident detail at this decision's timestamp, pick the
|
|
2140
|
+
// outcome closest in time (typically the corresponding one written
|
|
2141
|
+
// by iter 17's recorder immediately after the decision).
|
|
2142
|
+
const allOutcomes = outcomesByHash.get(d.task_hash) ?? [];
|
|
2143
|
+
const dts = Date.parse(d.ts);
|
|
2144
|
+
let out;
|
|
2145
|
+
let bestDelta = Infinity;
|
|
2146
|
+
for (const o of allOutcomes) {
|
|
2147
|
+
const delta = Math.abs(Date.parse(o.ts) - dts);
|
|
2148
|
+
if (delta < bestDelta) {
|
|
2149
|
+
bestDelta = delta;
|
|
2150
|
+
out = o;
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
const bucket = d.complexity < 0.34 ? 'cheap' : d.complexity < 0.67 ? 'mid' : 'strong';
|
|
2154
|
+
output.writeln('');
|
|
2155
|
+
output.writeln(` ts: ${d.ts}`);
|
|
2156
|
+
output.writeln(` task: "${d.task ?? '<task text not stored>'}"`);
|
|
2157
|
+
output.writeln(` complexity: ${d.complexity.toFixed(3)} (bucket: ${bucket})`);
|
|
2158
|
+
output.writeln(` picked model: ${d.openrouter_model ?? d.model}`);
|
|
2159
|
+
output.writeln(` routed_by: ${d.routed_by}${d.neural_backend ? ` via ${d.neural_backend}` : ''}`);
|
|
2160
|
+
output.writeln(` confidence: ${d.confidence.toFixed(3)} uncertainty: ${d.uncertainty.toFixed(3)}`);
|
|
2161
|
+
const apd = d.ab_pair;
|
|
2162
|
+
if (apd) {
|
|
2163
|
+
output.writeln(` ab_pair: bandit=${apd.bandit_pick} hybrid=${apd.hybrid_pick} disagree=${apd.disagree}`);
|
|
2164
|
+
}
|
|
2165
|
+
const ed = d.ensemble_disagreement;
|
|
2166
|
+
if (typeof ed === 'number') {
|
|
2167
|
+
output.writeln(` ensemble disagree: ${ed.toFixed(4)}`);
|
|
2168
|
+
}
|
|
2169
|
+
if (out) {
|
|
2170
|
+
output.writeln(` outcome: quality=${out.quality ?? '—'} cost_usd=${out.cost_usd != null ? '$' + out.cost_usd.toFixed(6) : '—'} source=${out.source ?? '—'}`);
|
|
2171
|
+
}
|
|
2172
|
+
else {
|
|
2173
|
+
output.writeln(` outcome: ${output.warning('(no paired outcome row)')}`);
|
|
2174
|
+
}
|
|
2175
|
+
}
|
|
2176
|
+
output.writeln('');
|
|
2177
|
+
return { success: true, data: payload };
|
|
2178
|
+
}
|
|
2179
|
+
output.writeln(` ${Math.min(limit, recent.length)} most-recent decisions (newest first):`);
|
|
2180
|
+
output.writeln(' ' + 'ts'.padEnd(20) + 'routed_by'.padEnd(18) + 'model'.padEnd(34) + 'conf');
|
|
2181
|
+
for (const d of recent) {
|
|
2182
|
+
const ts = d.ts.slice(0, 19);
|
|
2183
|
+
const id = (d.openrouter_model ?? d.model).slice(0, 32);
|
|
2184
|
+
output.writeln(` ${ts.padEnd(20)}${d.routed_by.padEnd(18)}${id.padEnd(34)}${d.confidence.toFixed(2)}`);
|
|
2185
|
+
}
|
|
2186
|
+
output.writeln('');
|
|
2187
|
+
return { success: true, data: payload };
|
|
2188
|
+
},
|
|
2189
|
+
};
|
|
2190
|
+
// ADR-149 iter 30 — forward-direction observability. Pairs with iter 28's
|
|
2191
|
+
// backward-direction `decisions` query. Lets operators inspect what the
|
|
2192
|
+
// router would pick for a hypothetical task WITHOUT actually dispatching.
|
|
2193
|
+
const routerDecideCommand = {
|
|
2194
|
+
name: 'decide',
|
|
2195
|
+
description: 'Show what the router would pick for a given task — no dispatch, just inspection (ADR-149 iter 30)',
|
|
2196
|
+
options: [
|
|
2197
|
+
{ name: 'task', short: 't', type: 'string', description: 'Task text (alternatively pass as positional arg)' },
|
|
2198
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
2199
|
+
],
|
|
2200
|
+
examples: [
|
|
2201
|
+
{ command: 'claude-flow neural router decide "fix typo in cache.ts"', description: 'See the routing decision for a small task' },
|
|
2202
|
+
{ command: 'claude-flow neural router decide -t "design distributed consensus" -f json', description: 'JSON output for the routing decision' },
|
|
2203
|
+
{ command: 'CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK=5 claude-flow neural router decide -t "..."', description: 'See iter 29 cost-ceiling mode in action' },
|
|
2204
|
+
],
|
|
2205
|
+
action: async (ctx) => {
|
|
2206
|
+
const task = ctx.flags.task ?? (ctx.args && ctx.args[0]) ?? null;
|
|
2207
|
+
const fmt = ctx.flags.format || 'table';
|
|
2208
|
+
if (!task || typeof task !== 'string' || task.trim().length === 0) {
|
|
2209
|
+
output.printError('Provide a task: --task "..." or as a positional argument');
|
|
2210
|
+
return { success: false, exitCode: 1 };
|
|
2211
|
+
}
|
|
2212
|
+
const { analyzeTaskComplexity, routeToModelFull } = await import('../ruvector/model-router.js');
|
|
2213
|
+
const { embedTaskWithCache } = await import('../ruvector/task-embedder.js');
|
|
2214
|
+
const { neuralRouterStatus, tryCostOptimalRoute } = await import('../ruvector/neural-router.js');
|
|
2215
|
+
const t0 = performance.now();
|
|
2216
|
+
const complexity = analyzeTaskComplexity(task);
|
|
2217
|
+
let embedding;
|
|
2218
|
+
try {
|
|
2219
|
+
embedding = await embedTaskWithCache(task);
|
|
2220
|
+
}
|
|
2221
|
+
catch { /* embedder may be absent */ }
|
|
2222
|
+
const result = await routeToModelFull(task, embedding);
|
|
2223
|
+
const status = await neuralRouterStatus();
|
|
2224
|
+
// iter 45 — also fetch the neural-layer result so we can surface the
|
|
2225
|
+
// ensemble-disagreement diagnostic. Extra inference but `decide` is an
|
|
2226
|
+
// operator inspection tool, not the hot path.
|
|
2227
|
+
let neuralResult = null;
|
|
2228
|
+
if (embedding) {
|
|
2229
|
+
const bucket = complexity.score < 0.34 ? 'low' : complexity.score < 0.67 ? 'med' : 'high';
|
|
2230
|
+
try {
|
|
2231
|
+
neuralResult = await tryCostOptimalRoute(embedding, { complexityBucket: bucket });
|
|
2232
|
+
}
|
|
2233
|
+
catch { /* gated off or backend absent */ }
|
|
2234
|
+
}
|
|
2235
|
+
const ms = performance.now() - t0;
|
|
2236
|
+
const payload = {
|
|
2237
|
+
task: task.length > 200 ? task.slice(0, 200) + '…' : task,
|
|
2238
|
+
taskLength: task.length,
|
|
2239
|
+
hasEmbedding: !!embedding,
|
|
2240
|
+
embeddingDim: embedding?.length ?? null,
|
|
2241
|
+
complexity: {
|
|
2242
|
+
score: complexity.score,
|
|
2243
|
+
bucket: complexity.score < 0.34 ? 'low' : complexity.score < 0.67 ? 'med' : 'high',
|
|
2244
|
+
features: complexity.features,
|
|
2245
|
+
indicators: complexity.indicators,
|
|
2246
|
+
},
|
|
2247
|
+
decision: {
|
|
2248
|
+
model: result.model,
|
|
2249
|
+
modelId: result.modelId,
|
|
2250
|
+
provider: result.provider,
|
|
2251
|
+
openrouterModel: result.openrouterModel,
|
|
2252
|
+
confidence: result.confidence,
|
|
2253
|
+
uncertainty: result.uncertainty,
|
|
2254
|
+
routedBy: result.routedBy,
|
|
2255
|
+
neuralBackend: result.neuralBackend,
|
|
2256
|
+
costMultiplier: result.costMultiplier,
|
|
2257
|
+
reasoning: result.reasoning,
|
|
2258
|
+
// iter 45 — ensemble disagreement diagnostic (always set when both
|
|
2259
|
+
// unified KRR + bucket specialist are loaded; observable signal for
|
|
2260
|
+
// tuning iter 44's threshold). null when not applicable.
|
|
2261
|
+
ensembleDisagreement: neuralResult?.ensembleDisagreement ?? null,
|
|
2262
|
+
},
|
|
2263
|
+
alternatives: result.alternatives,
|
|
2264
|
+
backend: {
|
|
2265
|
+
enabled: status.enabled,
|
|
2266
|
+
available: status.available,
|
|
2267
|
+
routedBy: status.routedBy,
|
|
2268
|
+
reason: status.reason,
|
|
2269
|
+
},
|
|
2270
|
+
activeEnv: {
|
|
2271
|
+
CLAUDE_FLOW_ROUTER_NEURAL: process.env.CLAUDE_FLOW_ROUTER_NEURAL ?? null,
|
|
2272
|
+
CLAUDE_FLOW_ROUTER_CALIBRATE: process.env.CLAUDE_FLOW_ROUTER_CALIBRATE ?? null,
|
|
2273
|
+
CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK: process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK ?? null,
|
|
2274
|
+
CLAUDE_FLOW_ROUTER_LATENCY_BUDGET_MS: process.env.CLAUDE_FLOW_ROUTER_LATENCY_BUDGET_MS ?? null,
|
|
2275
|
+
CLAUDE_FLOW_ROUTER_QUALITY_BAR: process.env.CLAUDE_FLOW_ROUTER_QUALITY_BAR ?? null,
|
|
2276
|
+
CLAUDE_FLOW_ROUTER_BANDIT_PER_MODEL: process.env.CLAUDE_FLOW_ROUTER_BANDIT_PER_MODEL ?? null,
|
|
2277
|
+
CLAUDE_FLOW_ROUTER_ENSEMBLE_UNCERTAINTY_THRESHOLD: process.env.CLAUDE_FLOW_ROUTER_ENSEMBLE_UNCERTAINTY_THRESHOLD ?? null,
|
|
2278
|
+
},
|
|
2279
|
+
elapsedMs: Math.round(ms * 100) / 100,
|
|
2280
|
+
};
|
|
2281
|
+
if (fmt === 'json') {
|
|
2282
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
2283
|
+
return { success: true, data: payload };
|
|
2284
|
+
}
|
|
2285
|
+
output.writeln();
|
|
2286
|
+
output.writeln(output.bold('Routing decision preview (ADR-149 iter 30 — no dispatch)'));
|
|
2287
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
2288
|
+
output.writeln(` Task: "${payload.task}"`);
|
|
2289
|
+
output.writeln(` Length: ${payload.taskLength} chars`);
|
|
2290
|
+
output.writeln(` Embedding: ${payload.hasEmbedding ? `${payload.embeddingDim} dims` : output.warning('not available (embedder absent / disabled)')}`);
|
|
2291
|
+
output.writeln('');
|
|
2292
|
+
output.writeln(output.bold(' Complexity:'));
|
|
2293
|
+
output.writeln(` score: ${complexity.score.toFixed(3)}`);
|
|
2294
|
+
output.writeln(` bucket: ${payload.complexity.bucket}`);
|
|
2295
|
+
output.writeln(` features: ${Object.entries(complexity.features).map(([k, v]) => `${k}=${typeof v === 'number' ? v.toFixed(2) : v}`).join(', ')}`);
|
|
2296
|
+
if (complexity.indicators.high.length + complexity.indicators.medium.length + complexity.indicators.low.length > 0) {
|
|
2297
|
+
output.writeln(` indicators: high=[${complexity.indicators.high.slice(0, 4).join(',')}] medium=[${complexity.indicators.medium.slice(0, 4).join(',')}] low=[${complexity.indicators.low.slice(0, 4).join(',')}]`);
|
|
2298
|
+
}
|
|
2299
|
+
output.writeln('');
|
|
2300
|
+
output.writeln(output.bold(' Decision:'));
|
|
2301
|
+
output.writeln(` model: ${output.success(result.model)}${result.modelId ? ` (id=${result.modelId})` : ''}`);
|
|
2302
|
+
output.writeln(` routed_by: ${result.routedBy}${result.neuralBackend ? ` via ${result.neuralBackend}` : ''}`);
|
|
2303
|
+
output.writeln(` confidence: ${result.confidence.toFixed(3)} uncertainty: ${result.uncertainty.toFixed(3)}`);
|
|
2304
|
+
output.writeln(` cost mult: ${result.costMultiplier.toFixed(2)}×`);
|
|
2305
|
+
if (result.provider === 'openrouter' && result.openrouterModel) {
|
|
2306
|
+
output.writeln(` via: openrouter → ${result.openrouterModel}`);
|
|
2307
|
+
}
|
|
2308
|
+
output.writeln(` reasoning: ${result.reasoning}`);
|
|
2309
|
+
if (neuralResult?.ensembleDisagreement !== undefined) {
|
|
2310
|
+
const d = neuralResult.ensembleDisagreement;
|
|
2311
|
+
const annotation = d > 0.20 ? output.warning(' ⚠ high — consider tuning iter 44 threshold')
|
|
2312
|
+
: d > 0.10 ? output.dim(' (moderate)')
|
|
2313
|
+
: output.dim(' (low — predictions agree)');
|
|
2314
|
+
output.writeln(` ensemble disagreement: ${d.toFixed(4)}${annotation}`);
|
|
2315
|
+
}
|
|
2316
|
+
output.writeln('');
|
|
2317
|
+
if (result.alternatives.length > 0) {
|
|
2318
|
+
output.writeln(output.bold(' Alternatives (model: score):'));
|
|
2319
|
+
for (const a of result.alternatives) {
|
|
2320
|
+
output.writeln(` ${a.model.padEnd(8)} ${a.score.toFixed(4)}`);
|
|
2321
|
+
}
|
|
2322
|
+
output.writeln('');
|
|
2323
|
+
}
|
|
2324
|
+
output.writeln(output.bold(' Backend state:'));
|
|
2325
|
+
output.writeln(` enabled: ${status.enabled}`);
|
|
2326
|
+
output.writeln(` available: ${status.available}`);
|
|
2327
|
+
output.writeln(` routedBy: ${status.routedBy ?? '—'}`);
|
|
2328
|
+
output.writeln(` reason: ${status.reason}`);
|
|
2329
|
+
output.writeln('');
|
|
2330
|
+
const activeEnvKeys = Object.entries(payload.activeEnv).filter(([, v]) => v !== null);
|
|
2331
|
+
if (activeEnvKeys.length > 0) {
|
|
2332
|
+
output.writeln(output.bold(' Active env overrides:'));
|
|
2333
|
+
for (const [k, v] of activeEnvKeys)
|
|
2334
|
+
output.writeln(` ${k}=${v}`);
|
|
2335
|
+
output.writeln('');
|
|
2336
|
+
}
|
|
2337
|
+
output.writeln(output.dim(` Total decision time: ${ms.toFixed(1)}ms`));
|
|
2338
|
+
output.writeln('');
|
|
2339
|
+
return { success: true, data: payload };
|
|
2340
|
+
},
|
|
2341
|
+
};
|
|
2342
|
+
// ADR-149 iter 32 — counterfactual cost-savings analysis. Iter 31 added
|
|
2343
|
+
// cost_usd to outcome rows; this subcommand consumes that and asks "what
|
|
2344
|
+
// would each decision have cost on the heuristic-only path?" — surfacing
|
|
2345
|
+
// the actual production savings the router delivers.
|
|
2346
|
+
const routerCostSavingsCommand = {
|
|
2347
|
+
name: 'cost-savings',
|
|
2348
|
+
description: 'Compute actual vs heuristic-counterfactual cost from paired decision+outcome rows (ADR-149 iter 32)',
|
|
2349
|
+
options: [
|
|
2350
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/...)' },
|
|
2351
|
+
{ name: 'since', short: 's', type: 'string', description: 'Time window suffix: 1h, 24h, 7d, 30d' },
|
|
2352
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
2353
|
+
{ name: 'top-n', type: 'number', description: 'Show top-N largest individual savings (default 5)', default: '5' },
|
|
2354
|
+
{ name: 'baseline', short: 'b', type: 'string', description: 'Counterfactual baseline: heuristic | always-haiku | always-sonnet | always-opus | always-gpt-4.1 | all (default: all)', default: 'all' },
|
|
2355
|
+
{ name: 'window', short: 'w', type: 'string', description: 'Bin decisions into successive windows of this duration (e.g. 1h, 24h, 7d). Output adds a trend table — iter 34 drift detection.' },
|
|
2356
|
+
{ name: 'task-hash', type: 'string', description: 'Filter to specific task_hash (FNV-1a-32 hex). For per-task cost investigation — iter 61.' },
|
|
2357
|
+
{ name: 'alert-on-drop-pct', type: 'number', description: 'Exit 1 if the most recent window\'s savings% falls > N points below the mean of prior windows. Requires --window. Default off. (ADR-149 iter 50)' },
|
|
2358
|
+
],
|
|
2359
|
+
examples: [
|
|
2360
|
+
{ command: 'claude-flow neural router cost-savings', description: 'All-time, all baselines (heuristic + Sonnet-always + Opus-always)' },
|
|
2361
|
+
{ command: 'claude-flow neural router cost-savings --baseline always-sonnet', description: 'Compare only against Sonnet-always' },
|
|
2362
|
+
{ command: 'claude-flow neural router cost-savings --window 24h', description: 'Daily savings trend — iter 34 drift detection' },
|
|
2363
|
+
{ command: 'claude-flow neural router cost-savings --since 7d --format json | jq .baselines.heuristic.savings.totalUsd', description: 'Pipe-friendly headline' },
|
|
2364
|
+
],
|
|
2365
|
+
action: async (ctx) => {
|
|
2366
|
+
const fs = await import('node:fs');
|
|
2367
|
+
const path = await import('node:path');
|
|
2368
|
+
const { MODEL_PRICES } = await import('../ruvector/model-prices.js');
|
|
2369
|
+
const inPath = ctx.flags.in
|
|
2370
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
2371
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
2372
|
+
const since = ctx.flags.since;
|
|
2373
|
+
const fmt = ctx.flags.format || 'table';
|
|
2374
|
+
const topN = parseInt((ctx.flags['top-n'] ?? ctx.flags.topN) || '5', 10) || 5;
|
|
2375
|
+
if (!fs.existsSync(inPath)) {
|
|
2376
|
+
const msg = `Trajectory file not found at ${inPath}`;
|
|
2377
|
+
if (fmt === 'json') {
|
|
2378
|
+
output.writeln(JSON.stringify({ error: msg }, null, 2));
|
|
2379
|
+
}
|
|
2380
|
+
else {
|
|
2381
|
+
output.printError(msg);
|
|
2382
|
+
}
|
|
2383
|
+
return { success: false, exitCode: 1 };
|
|
2384
|
+
}
|
|
2385
|
+
// iter 62 — preserve ALL outcomes (don't dedup by task_hash). Decisions
|
|
2386
|
+
// can still dedup because all occurrences of the same task have the
|
|
2387
|
+
// same embedding/complexity (it's the same task!), but different runs
|
|
2388
|
+
// produce different token counts and costs. The Array preserves those.
|
|
2389
|
+
const decisions = new Map();
|
|
2390
|
+
const outcomes = [];
|
|
2391
|
+
let malformed = 0;
|
|
2392
|
+
for (const l of fs.readFileSync(inPath, 'utf8').split('\n')) {
|
|
2393
|
+
if (!l.trim())
|
|
2394
|
+
continue;
|
|
2395
|
+
try {
|
|
2396
|
+
const r = JSON.parse(l);
|
|
2397
|
+
if (r.type === 'decision')
|
|
2398
|
+
decisions.set(r.task_hash, r);
|
|
2399
|
+
else if (r.type === 'outcome')
|
|
2400
|
+
outcomes.push(r);
|
|
2401
|
+
}
|
|
2402
|
+
catch {
|
|
2403
|
+
malformed++;
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
// Time-window filter (on the OUTCOME ts, since that's when cost was incurred).
|
|
2407
|
+
let cutoffMs = null;
|
|
2408
|
+
if (since) {
|
|
2409
|
+
const m = since.match(/^(\d+)([hdmw])$/);
|
|
2410
|
+
if (m) {
|
|
2411
|
+
const n = parseInt(m[1], 10);
|
|
2412
|
+
const unitMs = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]] ?? 0;
|
|
2413
|
+
cutoffMs = Date.now() - n * unitMs;
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
// iter 33 — multi-baseline counterfactuals. Each per-call row carries
|
|
2417
|
+
// the actual cost AND a map of counterfactual costs (one per baseline),
|
|
2418
|
+
// so a single LOO over the trajectory data computes all baselines at once.
|
|
2419
|
+
const baselineArg = ctx.flags.baseline ?? 'all';
|
|
2420
|
+
const allBaselines = ['heuristic', 'always-haiku', 'always-sonnet', 'always-opus', 'always-gpt-4.1'];
|
|
2421
|
+
const baselines = baselineArg === 'all' ? allBaselines : [baselineArg];
|
|
2422
|
+
// Map each baseline label → the modelId used for pricing.
|
|
2423
|
+
const baselineModelFor = (baseline, dec) => {
|
|
2424
|
+
switch (baseline) {
|
|
2425
|
+
case 'heuristic':
|
|
2426
|
+
return dec.ab_pair?.bandit_pick
|
|
2427
|
+
?? (dec.complexity < 0.34 ? 'haiku' : dec.complexity < 0.67 ? 'sonnet' : 'opus');
|
|
2428
|
+
case 'always-haiku': return 'haiku';
|
|
2429
|
+
case 'always-sonnet': return 'sonnet';
|
|
2430
|
+
case 'always-opus': return 'opus';
|
|
2431
|
+
case 'always-gpt-4.1': return 'openai/gpt-4.1';
|
|
2432
|
+
default: return baseline; // operator passes raw modelId
|
|
2433
|
+
}
|
|
2434
|
+
};
|
|
2435
|
+
const perCall = [];
|
|
2436
|
+
let droppedNoOutcomeCost = 0;
|
|
2437
|
+
let droppedNoDecision = 0;
|
|
2438
|
+
let droppedNoTokens = 0;
|
|
2439
|
+
// iter 61 — task-hash filter for per-task cost investigation.
|
|
2440
|
+
const taskHashFilter = (ctx.flags['task-hash'] ?? ctx.flags.taskHash)?.toLowerCase();
|
|
2441
|
+
if (taskHashFilter && !/^[0-9a-f]{8}$/.test(taskHashFilter)) {
|
|
2442
|
+
output.printError(`--task-hash must be an 8-char hex string (got "${taskHashFilter}")`);
|
|
2443
|
+
return { success: false, exitCode: 1 };
|
|
2444
|
+
}
|
|
2445
|
+
// iter 62 — iterate ALL outcomes (Array), not just one per hash. Multiple
|
|
2446
|
+
// occurrences of the same task contribute separately to the aggregate.
|
|
2447
|
+
for (const out of outcomes) {
|
|
2448
|
+
if (cutoffMs !== null && Date.parse(out.ts) < cutoffMs)
|
|
2449
|
+
continue;
|
|
2450
|
+
if (taskHashFilter && out.task_hash !== taskHashFilter)
|
|
2451
|
+
continue;
|
|
2452
|
+
if (out.cost_usd == null) {
|
|
2453
|
+
droppedNoOutcomeCost++;
|
|
2454
|
+
continue;
|
|
2455
|
+
}
|
|
2456
|
+
const dec = decisions.get(out.task_hash);
|
|
2457
|
+
if (!dec) {
|
|
2458
|
+
droppedNoDecision++;
|
|
2459
|
+
continue;
|
|
2460
|
+
}
|
|
2461
|
+
if (!out.tokens) {
|
|
2462
|
+
droppedNoTokens++;
|
|
2463
|
+
continue;
|
|
2464
|
+
}
|
|
2465
|
+
const actualCost = out.cost_usd;
|
|
2466
|
+
const actualModel = out.model_id ?? dec.openrouter_model ?? dec.model;
|
|
2467
|
+
const tier = dec.complexity < 0.34 ? 'cheap' : dec.complexity < 0.67 ? 'mid' : 'strong';
|
|
2468
|
+
// Compute every baseline at once. Cost = (input × $/Mtok_in + output ×
|
|
2469
|
+
// $/Mtok_out) / 1e6 — same formula iter 31's costUsd() uses.
|
|
2470
|
+
const counterfactuals = {};
|
|
2471
|
+
for (const b of baselines) {
|
|
2472
|
+
const m = baselineModelFor(b, dec);
|
|
2473
|
+
const p = MODEL_PRICES[m] ?? { in: 1, out: 1 };
|
|
2474
|
+
const cost = (out.tokens.input * p.in + out.tokens.output * p.out) / 1_000_000;
|
|
2475
|
+
counterfactuals[b] = { model: m, cost, savings: cost - actualCost };
|
|
2476
|
+
}
|
|
2477
|
+
perCall.push({
|
|
2478
|
+
task_hash: out.task_hash, ts: out.ts,
|
|
2479
|
+
actualModel, actualCost,
|
|
2480
|
+
counterfactuals,
|
|
2481
|
+
tokens: out.tokens,
|
|
2482
|
+
complexity: dec.complexity, tier,
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
const round6 = (x) => Math.round(x * 1_000_000) / 1_000_000;
|
|
2486
|
+
const round2 = (x) => Math.round(x * 100) / 100;
|
|
2487
|
+
const totalActual = perCall.reduce((s, p) => s + p.actualCost, 0);
|
|
2488
|
+
const baselineAggs = {};
|
|
2489
|
+
for (const b of baselines) {
|
|
2490
|
+
let totalCf = 0;
|
|
2491
|
+
const byTier = {
|
|
2492
|
+
cheap: { actual: 0, counterfactual: 0, n: 0 },
|
|
2493
|
+
mid: { actual: 0, counterfactual: 0, n: 0 },
|
|
2494
|
+
strong: { actual: 0, counterfactual: 0, n: 0 },
|
|
2495
|
+
};
|
|
2496
|
+
for (const c of perCall) {
|
|
2497
|
+
const cf = c.counterfactuals[b]?.cost ?? 0;
|
|
2498
|
+
totalCf += cf;
|
|
2499
|
+
byTier[c.tier].actual += c.actualCost;
|
|
2500
|
+
byTier[c.tier].counterfactual += cf;
|
|
2501
|
+
byTier[c.tier].n += 1;
|
|
2502
|
+
}
|
|
2503
|
+
const savings = totalCf - totalActual;
|
|
2504
|
+
const pct = totalCf > 0 ? (savings / totalCf) * 100 : 0;
|
|
2505
|
+
baselineAggs[b] = {
|
|
2506
|
+
totalUsd: round6(savings),
|
|
2507
|
+
savingsPct: round2(pct),
|
|
2508
|
+
actualUsd: round6(totalActual),
|
|
2509
|
+
counterfactualUsd: round6(totalCf),
|
|
2510
|
+
byTier: Object.fromEntries(Object.entries(byTier).map(([k, v]) => [k, {
|
|
2511
|
+
n: v.n,
|
|
2512
|
+
actualUsd: round6(v.actual),
|
|
2513
|
+
counterfactualUsd: round6(v.counterfactual),
|
|
2514
|
+
savingsUsd: round6(v.counterfactual - v.actual),
|
|
2515
|
+
savingsPct: v.counterfactual > 0 ? round2(((v.counterfactual - v.actual) / v.counterfactual) * 100) : 0,
|
|
2516
|
+
}])),
|
|
2517
|
+
};
|
|
2518
|
+
}
|
|
2519
|
+
// Top-N largest individual savings — use the first baseline's per-call savings.
|
|
2520
|
+
const primaryBaseline = baselines[0];
|
|
2521
|
+
const topSavings = [...perCall].sort((a, b) => (b.counterfactuals[primaryBaseline]?.savings ?? 0) - (a.counterfactuals[primaryBaseline]?.savings ?? 0)).slice(0, topN);
|
|
2522
|
+
const payload = {
|
|
2523
|
+
input: inPath,
|
|
2524
|
+
filters: { since, taskHash: taskHashFilter },
|
|
2525
|
+
pairs: perCall.length,
|
|
2526
|
+
dropped: { noOutcomeCost: droppedNoOutcomeCost, noDecision: droppedNoDecision, noTokens: droppedNoTokens },
|
|
2527
|
+
baselines: Object.fromEntries(Object.entries(baselineAggs).map(([k, v]) => [k, {
|
|
2528
|
+
savings: { totalUsd: v.totalUsd, savingsPct: v.savingsPct, actualUsd: v.actualUsd, counterfactualUsd: v.counterfactualUsd },
|
|
2529
|
+
byTier: v.byTier,
|
|
2530
|
+
}])),
|
|
2531
|
+
// Back-compat: top-level `savings` and `byTier` mirror the PRIMARY baseline
|
|
2532
|
+
// (first in the requested list, which for --baseline all is "heuristic" — same
|
|
2533
|
+
// as iter 32's output shape). Iter 32 callers parsing the old shape keep working.
|
|
2534
|
+
savings: baselineAggs[primaryBaseline]
|
|
2535
|
+
? { totalUsd: baselineAggs[primaryBaseline].totalUsd, savingsPct: baselineAggs[primaryBaseline].savingsPct, actualUsd: baselineAggs[primaryBaseline].actualUsd, counterfactualUsd: baselineAggs[primaryBaseline].counterfactualUsd }
|
|
2536
|
+
: { totalUsd: 0, savingsPct: 0, actualUsd: round6(totalActual), counterfactualUsd: 0 },
|
|
2537
|
+
byTier: baselineAggs[primaryBaseline]?.byTier ?? {},
|
|
2538
|
+
topSavings: topSavings.map(t => ({
|
|
2539
|
+
ts: t.ts, actualModel: t.actualModel,
|
|
2540
|
+
counterfactualModel: t.counterfactuals[primaryBaseline]?.model ?? '—',
|
|
2541
|
+
actualUsd: round6(t.actualCost),
|
|
2542
|
+
counterfactualUsd: round6(t.counterfactuals[primaryBaseline]?.cost ?? 0),
|
|
2543
|
+
savingsUsd: round6(t.counterfactuals[primaryBaseline]?.savings ?? 0),
|
|
2544
|
+
})),
|
|
2545
|
+
};
|
|
2546
|
+
// iter 34 — windowed drift detection. When --window is set, bin the
|
|
2547
|
+
// paired calls into successive duration windows and emit a trend table
|
|
2548
|
+
// (one row per window). Useful for "is the router degrading over time?"
|
|
2549
|
+
// — a sudden drop in savings %% across windows surfaces calibration drift,
|
|
2550
|
+
// workload shifts, or model deprecation.
|
|
2551
|
+
const windowArg = ctx.flags.window;
|
|
2552
|
+
let windowedTrend;
|
|
2553
|
+
if (windowArg) {
|
|
2554
|
+
const m = windowArg.match(/^(\d+)([hdmw])$/);
|
|
2555
|
+
if (!m) {
|
|
2556
|
+
output.printError(`--window must match Nh|Nd|Nm|Nw (got ${windowArg})`);
|
|
2557
|
+
return { success: false, exitCode: 1 };
|
|
2558
|
+
}
|
|
2559
|
+
const n = parseInt(m[1], 10);
|
|
2560
|
+
const unitMs = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]] ?? 0;
|
|
2561
|
+
const windowMs = n * unitMs;
|
|
2562
|
+
if (windowMs <= 0) {
|
|
2563
|
+
output.printError(`--window must be positive (got ${windowArg})`);
|
|
2564
|
+
return { success: false, exitCode: 1 };
|
|
2565
|
+
}
|
|
2566
|
+
// Bin perCall by (outcome.ts - earliest.ts) / windowMs.
|
|
2567
|
+
// Sort first so window indices are monotonic and gaps are visible.
|
|
2568
|
+
const sorted = [...perCall].sort((a, b) => a.ts.localeCompare(b.ts));
|
|
2569
|
+
if (sorted.length === 0) {
|
|
2570
|
+
windowedTrend = [];
|
|
2571
|
+
}
|
|
2572
|
+
else {
|
|
2573
|
+
const earliestMs = Date.parse(sorted[0].ts);
|
|
2574
|
+
const bins = new Map();
|
|
2575
|
+
for (const c of sorted) {
|
|
2576
|
+
const idx = Math.floor((Date.parse(c.ts) - earliestMs) / windowMs);
|
|
2577
|
+
if (!bins.has(idx))
|
|
2578
|
+
bins.set(idx, []);
|
|
2579
|
+
bins.get(idx).push(c);
|
|
2580
|
+
}
|
|
2581
|
+
const rows = [];
|
|
2582
|
+
let priorPct = null;
|
|
2583
|
+
for (const idx of [...bins.keys()].sort((a, b) => a - b)) {
|
|
2584
|
+
const items = bins.get(idx);
|
|
2585
|
+
let actual = 0, cf = 0;
|
|
2586
|
+
for (const c of items) {
|
|
2587
|
+
actual += c.actualCost;
|
|
2588
|
+
cf += c.counterfactuals[primaryBaseline]?.cost ?? 0;
|
|
2589
|
+
}
|
|
2590
|
+
const savings = cf - actual;
|
|
2591
|
+
const pct = cf > 0 ? (savings / cf) * 100 : 0;
|
|
2592
|
+
const windowStart = new Date(earliestMs + idx * windowMs).toISOString();
|
|
2593
|
+
const windowEnd = new Date(earliestMs + (idx + 1) * windowMs - 1).toISOString();
|
|
2594
|
+
const deltaVsPriorPct = priorPct === null ? null : round2(pct - priorPct);
|
|
2595
|
+
priorPct = pct;
|
|
2596
|
+
rows.push({
|
|
2597
|
+
windowStart, windowEnd, n: items.length,
|
|
2598
|
+
actualUsd: round6(actual),
|
|
2599
|
+
counterfactualUsd: round6(cf),
|
|
2600
|
+
savingsUsd: round6(savings),
|
|
2601
|
+
savingsPct: round2(pct),
|
|
2602
|
+
deltaVsPriorPct,
|
|
2603
|
+
});
|
|
2604
|
+
}
|
|
2605
|
+
windowedTrend = rows;
|
|
2606
|
+
}
|
|
2607
|
+
payload.windowedTrend = windowedTrend;
|
|
2608
|
+
payload.windowConfig = { duration: windowArg, primaryBaseline };
|
|
2609
|
+
}
|
|
2610
|
+
// iter 50 — drift alert. When --alert-on-drop-pct is set AND --window
|
|
2611
|
+
// produced ≥ 2 windows, compare the most-recent window's savings% to
|
|
2612
|
+
// the mean of prior windows'. If it dropped by > threshold, fail
|
|
2613
|
+
// (exit 1) so monitoring catches it. Independent of fmt — alert
|
|
2614
|
+
// state goes into the payload AND drives the exit code.
|
|
2615
|
+
const alertDropPctArg = (ctx.flags['alert-on-drop-pct'] ?? ctx.flags.alertOnDropPct);
|
|
2616
|
+
let alertTriggered = false;
|
|
2617
|
+
let alertReason = null;
|
|
2618
|
+
if (alertDropPctArg !== undefined && alertDropPctArg !== null && alertDropPctArg !== '') {
|
|
2619
|
+
const dropThreshold = typeof alertDropPctArg === 'string' ? parseFloat(alertDropPctArg) : alertDropPctArg;
|
|
2620
|
+
if (!isFinite(dropThreshold) || dropThreshold <= 0) {
|
|
2621
|
+
output.printError(`--alert-on-drop-pct must be a positive number (got ${alertDropPctArg})`);
|
|
2622
|
+
return { success: false, exitCode: 1 };
|
|
2623
|
+
}
|
|
2624
|
+
if (!windowedTrend || windowedTrend.length < 2) {
|
|
2625
|
+
alertReason = `not enough windows for drift detection (need ≥ 2, got ${windowedTrend?.length ?? 0}) — alert skipped`;
|
|
2626
|
+
}
|
|
2627
|
+
else {
|
|
2628
|
+
const latest = windowedTrend[windowedTrend.length - 1];
|
|
2629
|
+
const prior = windowedTrend.slice(0, -1);
|
|
2630
|
+
const priorMean = prior.reduce((s, w) => s + w.savingsPct, 0) / prior.length;
|
|
2631
|
+
const dropPct = priorMean - latest.savingsPct;
|
|
2632
|
+
if (dropPct > dropThreshold) {
|
|
2633
|
+
alertTriggered = true;
|
|
2634
|
+
alertReason = `latest window savings ${latest.savingsPct.toFixed(2)}% is ${dropPct.toFixed(2)} points BELOW prior windows' mean ${priorMean.toFixed(2)}% (threshold ${dropThreshold})`;
|
|
2635
|
+
}
|
|
2636
|
+
else {
|
|
2637
|
+
alertReason = `latest window savings ${latest.savingsPct.toFixed(2)}% within ${dropThreshold} points of prior mean ${priorMean.toFixed(2)}% — OK`;
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
payload.alert = {
|
|
2641
|
+
triggered: alertTriggered,
|
|
2642
|
+
reason: alertReason,
|
|
2643
|
+
dropThreshold,
|
|
2644
|
+
};
|
|
2645
|
+
}
|
|
2646
|
+
if (fmt === 'json') {
|
|
2647
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
2648
|
+
return alertTriggered
|
|
2649
|
+
? { success: false, exitCode: 1, data: payload }
|
|
2650
|
+
: { success: true, data: payload };
|
|
2651
|
+
}
|
|
2652
|
+
output.writeln();
|
|
2653
|
+
output.writeln(output.bold('Cost-savings analysis (ADR-149 iter 32)'));
|
|
2654
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
2655
|
+
output.writeln(` Input: ${inPath}`);
|
|
2656
|
+
if (since)
|
|
2657
|
+
output.writeln(` Time window: since ${since}`);
|
|
2658
|
+
if (taskHashFilter)
|
|
2659
|
+
output.writeln(` Task hash: ${taskHashFilter} (filtered to this task)`);
|
|
2660
|
+
output.writeln(` Paired calls: ${perCall.length} (dropped: no-cost=${droppedNoOutcomeCost}, no-decision=${droppedNoDecision}, no-tokens=${droppedNoTokens})`);
|
|
2661
|
+
output.writeln('');
|
|
2662
|
+
if (perCall.length === 0) {
|
|
2663
|
+
output.writeln(output.dim(' No cost-bearing paired rows. Enable trajectory recording AND make sure'));
|
|
2664
|
+
output.writeln(output.dim(' outcome rows include `tokens` (iter 31 wired this through agent-execute-core).'));
|
|
2665
|
+
output.writeln('');
|
|
2666
|
+
return { success: true, data: payload };
|
|
2667
|
+
}
|
|
2668
|
+
output.writeln(output.bold(' Headline (actual = $' + totalActual.toFixed(6) + '):'));
|
|
2669
|
+
output.writeln(' baseline counterfactual savings %');
|
|
2670
|
+
for (const [b, agg] of Object.entries(baselineAggs)) {
|
|
2671
|
+
const savingsStr = agg.totalUsd >= 0 ? '$' + agg.totalUsd.toFixed(6) : '-$' + Math.abs(agg.totalUsd).toFixed(6);
|
|
2672
|
+
const colored = agg.totalUsd >= 0 ? output.success(savingsStr) : output.warning(savingsStr);
|
|
2673
|
+
output.writeln(` ${b.padEnd(18)} $${agg.counterfactualUsd.toFixed(6).padEnd(14)} ${colored.padEnd(20)} ${agg.savingsPct.toString().padStart(6)}%`);
|
|
2674
|
+
}
|
|
2675
|
+
output.writeln('');
|
|
2676
|
+
output.writeln(output.dim(` Primary baseline for per-tier and top-savings views: "${primaryBaseline}"`));
|
|
2677
|
+
output.writeln('');
|
|
2678
|
+
const primaryAgg = baselineAggs[primaryBaseline];
|
|
2679
|
+
output.writeln(output.bold(' By tier (' + primaryBaseline + '):'));
|
|
2680
|
+
output.writeln(' tier n actual counterfactual savings %');
|
|
2681
|
+
for (const [k, v] of Object.entries(primaryAgg.byTier)) {
|
|
2682
|
+
output.writeln(` ${k.padEnd(8)} ${String(v.n).padStart(3)} $${v.actualUsd.toFixed(6).padEnd(14)} $${v.counterfactualUsd.toFixed(6).padEnd(14)} $${v.savingsUsd.toFixed(6).padEnd(14)} ${v.savingsPct.toString().padStart(6)}%`);
|
|
2683
|
+
}
|
|
2684
|
+
output.writeln('');
|
|
2685
|
+
if (topSavings.length > 0) {
|
|
2686
|
+
output.writeln(output.bold(` Top ${topSavings.length} largest individual savings (${primaryBaseline}):`));
|
|
2687
|
+
output.writeln(' ts actual → counterfactual saved');
|
|
2688
|
+
for (const t of topSavings) {
|
|
2689
|
+
const cfModel = t.counterfactuals[primaryBaseline]?.model ?? '—';
|
|
2690
|
+
const sv = t.counterfactuals[primaryBaseline]?.savings ?? 0;
|
|
2691
|
+
output.writeln(` ${t.ts.slice(0, 19)} ${t.actualModel.padEnd(40)} → ${cfModel.padEnd(40)} $${sv.toFixed(6)}`);
|
|
2692
|
+
}
|
|
2693
|
+
output.writeln('');
|
|
2694
|
+
}
|
|
2695
|
+
// iter 34 — windowed trend table for drift detection.
|
|
2696
|
+
if (windowedTrend && windowedTrend.length > 0) {
|
|
2697
|
+
output.writeln(output.bold(` Windowed trend (${windowArg} bins, baseline=${primaryBaseline}):`));
|
|
2698
|
+
output.writeln(' window start n actual counterfactual savings % Δ% vs prior');
|
|
2699
|
+
for (const w of windowedTrend) {
|
|
2700
|
+
const arrow = w.deltaVsPriorPct === null ? ''
|
|
2701
|
+
: w.deltaVsPriorPct > 0 ? output.success(`↑ +${w.deltaVsPriorPct.toFixed(2)}`)
|
|
2702
|
+
: w.deltaVsPriorPct < 0 ? output.warning(`↓ ${w.deltaVsPriorPct.toFixed(2)}`)
|
|
2703
|
+
: '·';
|
|
2704
|
+
output.writeln(` ${w.windowStart.slice(0, 19)} ${String(w.n).padStart(3)} $${w.actualUsd.toFixed(6).padEnd(11)} $${w.counterfactualUsd.toFixed(6).padEnd(14)} $${w.savingsUsd.toFixed(6).padEnd(11)} ${w.savingsPct.toString().padStart(6)}% ${arrow}`);
|
|
2705
|
+
}
|
|
2706
|
+
output.writeln('');
|
|
2707
|
+
output.writeln(output.dim(' Δ% vs prior: change in savings % from the prior window. Large negative'));
|
|
2708
|
+
output.writeln(output.dim(' deltas suggest router degradation, workload shift, or calibration drift.'));
|
|
2709
|
+
output.writeln('');
|
|
2710
|
+
}
|
|
2711
|
+
// iter 50 — alert footer when --alert-on-drop-pct was used.
|
|
2712
|
+
if (alertReason !== null) {
|
|
2713
|
+
if (alertTriggered) {
|
|
2714
|
+
output.writeln(output.warning(` ⚠ ALERT: ${alertReason}`));
|
|
2715
|
+
}
|
|
2716
|
+
else {
|
|
2717
|
+
output.writeln(output.dim(` ${alertReason}`));
|
|
2718
|
+
}
|
|
2719
|
+
output.writeln('');
|
|
2720
|
+
}
|
|
2721
|
+
return alertTriggered
|
|
2722
|
+
? { success: false, exitCode: 1, data: payload }
|
|
2723
|
+
: { success: true, data: payload };
|
|
2724
|
+
},
|
|
2725
|
+
};
|
|
2726
|
+
// ADR-149 iter 36 — operational observability for the trajectory JSONL itself.
|
|
2727
|
+
// Iter 17 added the recorder with rotation. Iter 28/30/32 consume the data.
|
|
2728
|
+
// Nothing previously surfaced "is logging healthy?" — size vs cap, rotation
|
|
2729
|
+
// count, parse success rate, pair-join rate, time range. SREs need this view.
|
|
2730
|
+
const routerTrajectoryHealthCommand = {
|
|
2731
|
+
name: 'trajectory-health',
|
|
2732
|
+
description: 'Show health of the routing-decision JSONL log: size, rotations, parse rate, pair-join rate (ADR-149 iter 36)',
|
|
2733
|
+
options: [
|
|
2734
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/...)' },
|
|
2735
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
2736
|
+
],
|
|
2737
|
+
examples: [
|
|
2738
|
+
{ command: 'claude-flow neural router trajectory-health', description: 'Snapshot of trajectory log health' },
|
|
2739
|
+
{ command: 'claude-flow neural router trajectory-health --format json | jq .pairJoinRatePct', description: 'Pipe-friendly pair-join rate for dashboards' },
|
|
2740
|
+
],
|
|
2741
|
+
action: async (ctx) => {
|
|
2742
|
+
const fs = await import('node:fs');
|
|
2743
|
+
const path = await import('node:path');
|
|
2744
|
+
const inPath = ctx.flags.in
|
|
2745
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
2746
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
2747
|
+
const fmt = ctx.flags.format || 'table';
|
|
2748
|
+
// Recorder config (mirrors router-trajectory.ts defaults).
|
|
2749
|
+
const recorderEnabled = process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY === '1';
|
|
2750
|
+
const maxSizeBytes = parseInt(process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXSIZE ?? `${10 * 1024 * 1024}`, 10) | 0;
|
|
2751
|
+
const maxRotations = Math.max(0, parseInt(process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXROTATIONS ?? '3', 10) || 3);
|
|
2752
|
+
if (!fs.existsSync(inPath)) {
|
|
2753
|
+
const payload = {
|
|
2754
|
+
recorderEnabled,
|
|
2755
|
+
input: inPath,
|
|
2756
|
+
exists: false,
|
|
2757
|
+
message: 'No trajectory file at the configured path.',
|
|
2758
|
+
hint: recorderEnabled
|
|
2759
|
+
? 'Recorder is enabled — file should appear after the next routing decision.'
|
|
2760
|
+
: 'Recorder is OFF. Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 to enable.',
|
|
2761
|
+
};
|
|
2762
|
+
if (fmt === 'json')
|
|
2763
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
2764
|
+
else {
|
|
2765
|
+
output.writeln('');
|
|
2766
|
+
output.writeln(output.bold('Trajectory health'));
|
|
2767
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
2768
|
+
output.writeln(` Path: ${inPath}`);
|
|
2769
|
+
output.writeln(` Recorder gate: ${recorderEnabled ? output.success('ON') : output.warning('OFF (CLAUDE_FLOW_ROUTER_TRAJECTORY=1 to enable)')}`);
|
|
2770
|
+
output.writeln(` Status: ${output.warning('file does not exist')}`);
|
|
2771
|
+
output.writeln(` Hint: ${payload.hint}`);
|
|
2772
|
+
output.writeln('');
|
|
2773
|
+
}
|
|
2774
|
+
return { success: true, data: payload };
|
|
2775
|
+
}
|
|
2776
|
+
const stat = fs.statSync(inPath);
|
|
2777
|
+
const sizeBytes = stat.size;
|
|
2778
|
+
const sizePct = maxSizeBytes > 0 ? Math.round((sizeBytes / maxSizeBytes) * 1000) / 10 : 0;
|
|
2779
|
+
// Count .bak rotation files.
|
|
2780
|
+
const rotationFiles = [];
|
|
2781
|
+
for (let i = 1; i <= maxRotations; i++) {
|
|
2782
|
+
const p = `${inPath}.${i}`;
|
|
2783
|
+
if (fs.existsSync(p)) {
|
|
2784
|
+
const s = fs.statSync(p);
|
|
2785
|
+
rotationFiles.push({ index: i, path: p, bytes: s.size, mtime: s.mtime.toISOString() });
|
|
2786
|
+
}
|
|
2787
|
+
}
|
|
2788
|
+
const lines = fs.readFileSync(inPath, 'utf8').split('\n').filter(l => l.trim().length > 0);
|
|
2789
|
+
let decisions = 0, outcomes = 0, malformed = 0, otherType = 0;
|
|
2790
|
+
let oldestTs = null;
|
|
2791
|
+
let newestTs = null;
|
|
2792
|
+
const decisionHashes = new Set();
|
|
2793
|
+
const outcomeHashes = new Set();
|
|
2794
|
+
for (const l of lines) {
|
|
2795
|
+
try {
|
|
2796
|
+
const r = JSON.parse(l);
|
|
2797
|
+
if (r.type === 'decision') {
|
|
2798
|
+
decisions++;
|
|
2799
|
+
if (r.task_hash)
|
|
2800
|
+
decisionHashes.add(r.task_hash);
|
|
2801
|
+
}
|
|
2802
|
+
else if (r.type === 'outcome') {
|
|
2803
|
+
outcomes++;
|
|
2804
|
+
if (r.task_hash)
|
|
2805
|
+
outcomeHashes.add(r.task_hash);
|
|
2806
|
+
}
|
|
2807
|
+
else {
|
|
2808
|
+
otherType++;
|
|
2809
|
+
}
|
|
2810
|
+
if (r.ts) {
|
|
2811
|
+
if (oldestTs === null || r.ts < oldestTs)
|
|
2812
|
+
oldestTs = r.ts;
|
|
2813
|
+
if (newestTs === null || r.ts > newestTs)
|
|
2814
|
+
newestTs = r.ts;
|
|
2815
|
+
}
|
|
2816
|
+
}
|
|
2817
|
+
catch {
|
|
2818
|
+
malformed++;
|
|
2819
|
+
}
|
|
2820
|
+
}
|
|
2821
|
+
// Pair-join rate: % of decisions that have a matching outcome.
|
|
2822
|
+
let pairedHashes = 0;
|
|
2823
|
+
for (const h of decisionHashes)
|
|
2824
|
+
if (outcomeHashes.has(h))
|
|
2825
|
+
pairedHashes++;
|
|
2826
|
+
const pairJoinRatePct = decisionHashes.size > 0
|
|
2827
|
+
? Math.round((pairedHashes / decisionHashes.size) * 10000) / 100
|
|
2828
|
+
: 0;
|
|
2829
|
+
const parseSuccessPct = lines.length > 0
|
|
2830
|
+
? Math.round(((lines.length - malformed) / lines.length) * 10000) / 100
|
|
2831
|
+
: 100;
|
|
2832
|
+
const payload = {
|
|
2833
|
+
recorderEnabled,
|
|
2834
|
+
input: inPath,
|
|
2835
|
+
exists: true,
|
|
2836
|
+
file: {
|
|
2837
|
+
sizeBytes,
|
|
2838
|
+
sizePct,
|
|
2839
|
+
maxSizeBytes,
|
|
2840
|
+
maxRotations,
|
|
2841
|
+
rotationsOnDisk: rotationFiles.length,
|
|
2842
|
+
rotations: rotationFiles,
|
|
2843
|
+
mtime: stat.mtime.toISOString(),
|
|
2844
|
+
},
|
|
2845
|
+
rows: {
|
|
2846
|
+
total: lines.length,
|
|
2847
|
+
decisions, outcomes, otherType, malformed,
|
|
2848
|
+
},
|
|
2849
|
+
pairing: {
|
|
2850
|
+
uniqueDecisionHashes: decisionHashes.size,
|
|
2851
|
+
uniqueOutcomeHashes: outcomeHashes.size,
|
|
2852
|
+
pairedHashes,
|
|
2853
|
+
pairJoinRatePct,
|
|
2854
|
+
parseSuccessPct,
|
|
2855
|
+
},
|
|
2856
|
+
timeRange: {
|
|
2857
|
+
oldestTs, newestTs,
|
|
2858
|
+
spanHours: oldestTs && newestTs ? Math.round(((Date.parse(newestTs) - Date.parse(oldestTs)) / 3600_000) * 10) / 10 : 0,
|
|
2859
|
+
},
|
|
2860
|
+
};
|
|
2861
|
+
if (fmt === 'json') {
|
|
2862
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
2863
|
+
return { success: true, data: payload };
|
|
2864
|
+
}
|
|
2865
|
+
output.writeln();
|
|
2866
|
+
output.writeln(output.bold('Trajectory health (ADR-149 iter 36)'));
|
|
2867
|
+
output.writeln(output.dim('─'.repeat(60)));
|
|
2868
|
+
output.writeln(` Path: ${inPath}`);
|
|
2869
|
+
output.writeln(` Recorder gate: ${recorderEnabled ? output.success('ON') : output.warning('OFF')}`);
|
|
2870
|
+
output.writeln('');
|
|
2871
|
+
output.writeln(output.bold(' File:'));
|
|
2872
|
+
const sizeMb = (sizeBytes / 1_048_576).toFixed(3);
|
|
2873
|
+
const capMb = (maxSizeBytes / 1_048_576).toFixed(1);
|
|
2874
|
+
const sizeWarn = sizePct >= 80 ? output.warning(`(${sizePct}% of cap)`) : output.dim(`(${sizePct}% of cap)`);
|
|
2875
|
+
output.writeln(` size: ${sizeBytes} bytes (${sizeMb} MB) of ${capMb} MB max ${sizeWarn}`);
|
|
2876
|
+
output.writeln(` last write: ${stat.mtime.toISOString()}`);
|
|
2877
|
+
output.writeln(` rotations: ${rotationFiles.length} of ${maxRotations} max .bak files on disk`);
|
|
2878
|
+
for (const r of rotationFiles) {
|
|
2879
|
+
output.writeln(` .${r.index}: ${r.bytes} bytes ${r.mtime}`);
|
|
2880
|
+
}
|
|
2881
|
+
output.writeln('');
|
|
2882
|
+
output.writeln(output.bold(' Rows:'));
|
|
2883
|
+
output.writeln(` total: ${lines.length}`);
|
|
2884
|
+
output.writeln(` decisions: ${decisions}`);
|
|
2885
|
+
output.writeln(` outcomes: ${outcomes}`);
|
|
2886
|
+
if (otherType > 0)
|
|
2887
|
+
output.writeln(` other type: ${otherType}`);
|
|
2888
|
+
output.writeln(` malformed: ${malformed} (parse success ${parseSuccessPct}%)`);
|
|
2889
|
+
output.writeln('');
|
|
2890
|
+
output.writeln(output.bold(' Pairing (decision ↔ outcome join by task_hash):'));
|
|
2891
|
+
output.writeln(` unique decision hashes: ${decisionHashes.size}`);
|
|
2892
|
+
output.writeln(` unique outcome hashes: ${outcomeHashes.size}`);
|
|
2893
|
+
output.writeln(` paired: ${pairedHashes} (${pairJoinRatePct}%)`);
|
|
2894
|
+
if (pairJoinRatePct < 50 && decisionHashes.size > 5) {
|
|
2895
|
+
output.writeln(output.warning(` ⚠ pair-join rate < 50% — outcome rows may not be wired through (iter 17/31).`));
|
|
2896
|
+
}
|
|
2897
|
+
output.writeln('');
|
|
2898
|
+
if (oldestTs && newestTs) {
|
|
2899
|
+
output.writeln(output.bold(' Time range:'));
|
|
2900
|
+
output.writeln(` oldest: ${oldestTs}`);
|
|
2901
|
+
output.writeln(` newest: ${newestTs}`);
|
|
2902
|
+
output.writeln(` span: ${payload.timeRange.spanHours} hours`);
|
|
2903
|
+
output.writeln('');
|
|
2904
|
+
}
|
|
2905
|
+
return { success: true, data: payload };
|
|
2906
|
+
},
|
|
2907
|
+
};
|
|
2908
|
+
// ADR-149 iter 54 — consolidated env-var inspection. The router has ~15
|
|
2909
|
+
// CLAUDE_FLOW_ROUTER_* env vars accumulated across iters 12-53. Operators
|
|
2910
|
+
// need a single command that lists each with its current value (or default),
|
|
2911
|
+
// effect, and which iter introduced it. Color-coded: green = override set,
|
|
2912
|
+
// dim = default.
|
|
2913
|
+
const routerConfigCommand = {
|
|
2914
|
+
name: 'config',
|
|
2915
|
+
description: 'List all CLAUDE_FLOW_ROUTER_* env vars with current values + effects (ADR-149 iter 54)',
|
|
2916
|
+
options: [
|
|
2917
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
2918
|
+
{ name: 'only-overrides', type: 'boolean', description: 'Only show env vars that are explicitly set (hide defaults)', default: false },
|
|
2919
|
+
],
|
|
2920
|
+
examples: [
|
|
2921
|
+
{ command: 'claude-flow neural router config', description: 'All router env vars with current values' },
|
|
2922
|
+
{ command: 'claude-flow neural router config --only-overrides', description: 'Just what the operator has set' },
|
|
2923
|
+
{ command: 'claude-flow neural router config --format json | jq', description: 'Audit / diff against another deployment' },
|
|
2924
|
+
],
|
|
2925
|
+
action: async (ctx) => {
|
|
2926
|
+
const fmt = ctx.flags.format || 'table';
|
|
2927
|
+
const onlyOverrides = Boolean(ctx.flags['only-overrides'] ?? ctx.flags.onlyOverrides);
|
|
2928
|
+
const rows = [
|
|
2929
|
+
// Core gate (iter 0)
|
|
2930
|
+
{ name: 'CLAUDE_FLOW_ROUTER_NEURAL', iter: 0, defaultValue: 'unset (0)', currentValue: process.env.CLAUDE_FLOW_ROUTER_NEURAL ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_NEURAL, effect: 'Gate. =1 enables neural router; otherwise pure-bandit heuristic.' },
|
|
2931
|
+
{ name: 'CLAUDE_FLOW_ROUTER_MODEL_PATH', iter: 0, defaultValue: 'unset', currentValue: process.env.CLAUDE_FLOW_ROUTER_MODEL_PATH ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_MODEL_PATH, effect: 'Override the KRR artifact path. Defaults to bundled.' },
|
|
2932
|
+
{ name: 'CLAUDE_FLOW_ROUTER_QUALITY_BAR', iter: 0, defaultValue: '0.50', currentValue: process.env.CLAUDE_FLOW_ROUTER_QUALITY_BAR ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_QUALITY_BAR, effect: 'Cost-optimal mode: minimum predicted quality to pick a candidate.' },
|
|
2933
|
+
// Trajectory (iter 17)
|
|
2934
|
+
{ name: 'CLAUDE_FLOW_ROUTER_TRAJECTORY', iter: 17, defaultValue: 'unset (0)', currentValue: process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY, effect: 'Gate. =1 writes decision+outcome rows to .swarm/model-router-trajectories.jsonl.' },
|
|
2935
|
+
{ name: 'CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH', iter: 17, defaultValue: '.swarm/model-router-trajectories.jsonl', currentValue: process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH, effect: 'Override the trajectory JSONL path.' },
|
|
2936
|
+
{ name: 'CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXSIZE', iter: 17, defaultValue: '10485760 (10MB)', currentValue: process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXSIZE ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXSIZE, effect: 'Bytes before rotation. Set 0 to disable rotation.' },
|
|
2937
|
+
{ name: 'CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXROTATIONS', iter: 17, defaultValue: '3', currentValue: process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXROTATIONS ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_MAXROTATIONS, effect: 'Max .bak files to keep when rotating.' },
|
|
2938
|
+
{ name: 'CLAUDE_FLOW_ROUTER_TRAJECTORY_TASKLEN', iter: 17, defaultValue: '500', currentValue: process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_TASKLEN ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_TASKLEN, effect: 'Max chars of task text to persist per row (truncated above this).' },
|
|
2939
|
+
// Latency budget (iter 12)
|
|
2940
|
+
{ name: 'CLAUDE_FLOW_ROUTER_LATENCY_BUDGET_MS', iter: 12, defaultValue: '0 (unbounded)', currentValue: process.env.CLAUDE_FLOW_ROUTER_LATENCY_BUDGET_MS ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_LATENCY_BUDGET_MS, effect: 'Drop candidates whose measured p50 latency > this many ms before selection.' },
|
|
2941
|
+
// Per-modelId bandit (iter 14)
|
|
2942
|
+
{ name: 'CLAUDE_FLOW_ROUTER_BANDIT_PER_MODEL', iter: 14, defaultValue: 'unset (0)', currentValue: process.env.CLAUDE_FLOW_ROUTER_BANDIT_PER_MODEL ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_BANDIT_PER_MODEL, effect: 'Gate. =1 enables per-modelId Thompson sampling perturbation of neural prediction.' },
|
|
2943
|
+
// k-NN backend (iter 0)
|
|
2944
|
+
{ name: 'CLAUDE_FLOW_ROUTER_KNN_K', iter: 0, defaultValue: '5', currentValue: process.env.CLAUDE_FLOW_ROUTER_KNN_K ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_KNN_K, effect: 'k for the k-NN backend when KRR is not loadable.' },
|
|
2945
|
+
{ name: 'CLAUDE_FLOW_ROUTER_SEED_CORPUS', iter: 0, defaultValue: 'bundled seed-rows.json', currentValue: process.env.CLAUDE_FLOW_ROUTER_SEED_CORPUS ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_SEED_CORPUS, effect: 'Override the DRACO seed corpus path.' },
|
|
2946
|
+
// Calibration (iter 22-25)
|
|
2947
|
+
{ name: 'CLAUDE_FLOW_ROUTER_CALIBRATE', iter: 24, defaultValue: 'unset (default ON)', currentValue: process.env.CLAUDE_FLOW_ROUTER_CALIBRATE ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_CALIBRATE, effect: 'Isotonic calibration of KRR predictions. =0 opts out (recovers raw KRR). Default-on since iter 24 (OOS validated).' },
|
|
2948
|
+
{ name: 'CLAUDE_FLOW_ROUTER_CALIBRATOR_PATH', iter: 22, defaultValue: 'bundled seed-router.calibrator.json', currentValue: process.env.CLAUDE_FLOW_ROUTER_CALIBRATOR_PATH ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_CALIBRATOR_PATH, effect: 'Override the unified calibrator path. Per-tier files (low/med/high) load from the same dir.' },
|
|
2949
|
+
// Cost ceiling (iter 29)
|
|
2950
|
+
{ name: 'CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK', iter: 29, defaultValue: '0 (disabled)', currentValue: process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK, effect: 'Orthogonal selector mode: pick BEST quality among candidates ≤ ceiling $/Mtok blended.' },
|
|
2951
|
+
// A/B mode (iter 5/37)
|
|
2952
|
+
{ name: 'CLAUDE_FLOW_ROUTER_AB', iter: 5, defaultValue: 'unset (0)', currentValue: process.env.CLAUDE_FLOW_ROUTER_AB ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_AB, effect: 'Legacy all-on A/B mode. Records bandit_pick + hybrid_pick on every decision.' },
|
|
2953
|
+
{ name: 'CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE', iter: 37, defaultValue: '0 (disabled)', currentValue: process.env.CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE, effect: 'Sampled A/B mode. 0..1 fraction of decisions to A/B (deterministic by task_hash). Overrides legacy AB=1.' },
|
|
2954
|
+
// Ensemble uncertainty (iter 44)
|
|
2955
|
+
{ name: 'CLAUDE_FLOW_ROUTER_ENSEMBLE_UNCERTAINTY_THRESHOLD', iter: 44, defaultValue: '0 (disabled)', currentValue: process.env.CLAUDE_FLOW_ROUTER_ENSEMBLE_UNCERTAINTY_THRESHOLD ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_ENSEMBLE_UNCERTAINTY_THRESHOLD, effect: 'When > 0: if |unified_q - specialist_q| > threshold for picked model, fall back to bandit.' },
|
|
2956
|
+
// Bandit warmup (iter 52/53)
|
|
2957
|
+
{ name: 'CLAUDE_FLOW_ROUTER_BANDIT_WARMUP_RANGE', iter: 52, defaultValue: '8', currentValue: process.env.CLAUDE_FLOW_ROUTER_BANDIT_WARMUP_RANGE ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_BANDIT_WARMUP_RANGE, effect: 'Continuous warmup denominator. Smaller = bandit ramps faster; larger = more conservative.' },
|
|
2958
|
+
{ name: 'CLAUDE_FLOW_ROUTER_BANDIT_FULL_INFLUENCE', iter: 53, defaultValue: 'unset (0)', currentValue: process.env.CLAUDE_FLOW_ROUTER_BANDIT_FULL_INFLUENCE ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_BANDIT_FULL_INFLUENCE, effect: 'Gate. =1 uses asymptotic curve (samples-2)/(samples+WARMUP) — bandit dominates at scale.' },
|
|
2959
|
+
{ name: 'CLAUDE_FLOW_ROUTER_BANDIT_SHRINKAGE_LAMBDA', iter: 57, defaultValue: '4', currentValue: process.env.CLAUDE_FLOW_ROUTER_BANDIT_SHRINKAGE_LAMBDA ?? '', isOverride: !!process.env.CLAUDE_FLOW_ROUTER_BANDIT_SHRINKAGE_LAMBDA, effect: 'Cross-bucket shrinkage strength. λ=0 disables; higher λ = more bias toward marginal anchor for cold cells.' },
|
|
2960
|
+
];
|
|
2961
|
+
const visible = onlyOverrides ? rows.filter(r => r.isOverride) : rows;
|
|
2962
|
+
if (fmt === 'json') {
|
|
2963
|
+
output.writeln(JSON.stringify(visible, null, 2));
|
|
2964
|
+
return { success: true, data: visible };
|
|
2965
|
+
}
|
|
2966
|
+
output.writeln();
|
|
2967
|
+
output.writeln(output.bold('Router config — ADR-149 iter 54 (env-var inventory)'));
|
|
2968
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
2969
|
+
output.writeln(` ${visible.length} of ${rows.length} entries (${rows.filter(r => r.isOverride).length} overridden, ${rows.length - rows.filter(r => r.isOverride).length} default)`);
|
|
2970
|
+
output.writeln('');
|
|
2971
|
+
for (const r of visible) {
|
|
2972
|
+
const value = r.isOverride ? output.success(r.currentValue) : output.dim(`(default: ${r.defaultValue})`);
|
|
2973
|
+
output.writeln(` ${r.name} = ${value}`);
|
|
2974
|
+
output.writeln(` ${output.dim(`iter ${r.iter}`)} ${r.effect}`);
|
|
2975
|
+
output.writeln('');
|
|
2976
|
+
}
|
|
2977
|
+
if (visible.length === 0 && onlyOverrides) {
|
|
2978
|
+
output.writeln(output.dim(' No overrides — all router behavior is at defaults.'));
|
|
2979
|
+
output.writeln('');
|
|
2980
|
+
}
|
|
2981
|
+
return { success: true, data: visible };
|
|
2982
|
+
},
|
|
2983
|
+
};
|
|
2984
|
+
// ADR-149 iter 55 — side-by-side comparison of the two selector modes.
|
|
2985
|
+
// Iter 29 added quality-best-under-budget; iter 30 added `decide` for the
|
|
2986
|
+
// default cost-optimal mode. This subcommand runs BOTH on the same task
|
|
2987
|
+
// so operators can see which mode is right for their workload BEFORE
|
|
2988
|
+
// flipping CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK on for production.
|
|
2989
|
+
const routerCompareModesCommand = {
|
|
2990
|
+
name: 'compare-modes',
|
|
2991
|
+
description: 'Compare selector modes side-by-side for a hypothetical task (cost-optimal vs cost-ceiling) — ADR-149 iter 55',
|
|
2992
|
+
options: [
|
|
2993
|
+
{ name: 'task', short: 't', type: 'string', description: 'Task text (or positional arg)' },
|
|
2994
|
+
{ name: 'ceiling', type: 'number', description: 'Cost-ceiling $/Mtok for iter 29 mode (default 20)', default: '20' },
|
|
2995
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
2996
|
+
],
|
|
2997
|
+
examples: [
|
|
2998
|
+
{ command: 'claude-flow neural router compare-modes "refactor strategy pattern"', description: 'Compare both modes for a task' },
|
|
2999
|
+
{ command: 'claude-flow neural router compare-modes -t "..." --ceiling 5', description: 'Cost-ceiling at $5 blended' },
|
|
3000
|
+
],
|
|
3001
|
+
action: async (ctx) => {
|
|
3002
|
+
const task = ctx.flags.task ?? (ctx.args && ctx.args[0]) ?? null;
|
|
3003
|
+
if (!task) {
|
|
3004
|
+
output.printError('Provide a task: --task "..." or as positional arg');
|
|
3005
|
+
return { success: false, exitCode: 1 };
|
|
3006
|
+
}
|
|
3007
|
+
const ceiling = parseFloat(ctx.flags.ceiling || '20') || 20;
|
|
3008
|
+
const fmt = ctx.flags.format || 'table';
|
|
3009
|
+
const { embedTaskWithCache } = await import('../ruvector/task-embedder.js');
|
|
3010
|
+
const { tryCostOptimalRoute, __resetNeuralRouterForTests } = await import('../ruvector/neural-router.js');
|
|
3011
|
+
const { analyzeTaskComplexity } = await import('../ruvector/model-router.js');
|
|
3012
|
+
const complexity = analyzeTaskComplexity(task);
|
|
3013
|
+
const bucket = complexity.score < 0.34 ? 'low' : complexity.score < 0.67 ? 'med' : 'high';
|
|
3014
|
+
let embedding;
|
|
3015
|
+
try {
|
|
3016
|
+
embedding = await embedTaskWithCache(task);
|
|
3017
|
+
}
|
|
3018
|
+
catch { /* */ }
|
|
3019
|
+
if (!embedding) {
|
|
3020
|
+
output.printError('Embedder unavailable — cannot run neural routing for comparison');
|
|
3021
|
+
return { success: false, exitCode: 1 };
|
|
3022
|
+
}
|
|
3023
|
+
// Mode 1: cost-optimal (default — no ceiling)
|
|
3024
|
+
process.env.CLAUDE_FLOW_ROUTER_NEURAL = '1';
|
|
3025
|
+
delete process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK;
|
|
3026
|
+
__resetNeuralRouterForTests();
|
|
3027
|
+
const costOptimal = await tryCostOptimalRoute(embedding, { complexityBucket: bucket });
|
|
3028
|
+
// Mode 2: cost-ceiling
|
|
3029
|
+
process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK = String(ceiling);
|
|
3030
|
+
__resetNeuralRouterForTests();
|
|
3031
|
+
const costCeiling = await tryCostOptimalRoute(embedding, { complexityBucket: bucket });
|
|
3032
|
+
// Clean up
|
|
3033
|
+
delete process.env.CLAUDE_FLOW_ROUTER_COST_CEILING_USD_PER_MTOK;
|
|
3034
|
+
__resetNeuralRouterForTests();
|
|
3035
|
+
const payload = {
|
|
3036
|
+
task: task.length > 200 ? task.slice(0, 200) + '…' : task,
|
|
3037
|
+
complexity: complexity.score,
|
|
3038
|
+
bucket,
|
|
3039
|
+
modes: {
|
|
3040
|
+
costOptimal: costOptimal ? {
|
|
3041
|
+
modelId: costOptimal.modelId,
|
|
3042
|
+
predictedQuality: costOptimal.predictedQuality,
|
|
3043
|
+
costPerMTok: costOptimal.alternatives.find(a => a.modelId === costOptimal.modelId)?.costPerMTok ?? null,
|
|
3044
|
+
metBar: costOptimal.metBar,
|
|
3045
|
+
} : null,
|
|
3046
|
+
costCeiling: costCeiling ? {
|
|
3047
|
+
ceilingUsd: ceiling,
|
|
3048
|
+
modelId: costCeiling.modelId,
|
|
3049
|
+
predictedQuality: costCeiling.predictedQuality,
|
|
3050
|
+
costPerMTok: costCeiling.alternatives.find(a => a.modelId === costCeiling.modelId)?.costPerMTok ?? null,
|
|
3051
|
+
metBar: costCeiling.metBar,
|
|
3052
|
+
} : null,
|
|
3053
|
+
},
|
|
3054
|
+
sameModel: costOptimal?.modelId === costCeiling?.modelId,
|
|
3055
|
+
deltaQuality: (costCeiling && costOptimal) ? costCeiling.predictedQuality - costOptimal.predictedQuality : null,
|
|
3056
|
+
deltaCost: (costCeiling && costOptimal)
|
|
3057
|
+
? ((costCeiling.alternatives.find(a => a.modelId === costCeiling.modelId)?.costPerMTok ?? 0) - (costOptimal.alternatives.find(a => a.modelId === costOptimal.modelId)?.costPerMTok ?? 0))
|
|
3058
|
+
: null,
|
|
3059
|
+
};
|
|
3060
|
+
if (fmt === 'json') {
|
|
3061
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3062
|
+
return { success: true, data: payload };
|
|
3063
|
+
}
|
|
3064
|
+
output.writeln();
|
|
3065
|
+
output.writeln(output.bold('Selector mode comparison (ADR-149 iter 55)'));
|
|
3066
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3067
|
+
output.writeln(` Task: "${payload.task}"`);
|
|
3068
|
+
output.writeln(` Complexity: ${complexity.score.toFixed(3)} (bucket: ${bucket})`);
|
|
3069
|
+
output.writeln('');
|
|
3070
|
+
output.writeln(output.bold(' Cost-optimal mode (default — cheapest above qualityBar):'));
|
|
3071
|
+
if (costOptimal) {
|
|
3072
|
+
const co = payload.modes.costOptimal;
|
|
3073
|
+
output.writeln(` picked: ${output.success(co.modelId)}`);
|
|
3074
|
+
output.writeln(` predicted Q: ${co.predictedQuality.toFixed(4)}`);
|
|
3075
|
+
output.writeln(` cost ($/Mtok): $${co.costPerMTok?.toFixed(2)}`);
|
|
3076
|
+
output.writeln(` met quality bar: ${co.metBar ? '✓' : '✗'}`);
|
|
3077
|
+
}
|
|
3078
|
+
else {
|
|
3079
|
+
output.writeln(` ${output.warning('null (neural backend declined — would fall back to bandit)')}`);
|
|
3080
|
+
}
|
|
3081
|
+
output.writeln('');
|
|
3082
|
+
output.writeln(output.bold(` Cost-ceiling mode (iter 29 — best quality ≤ $${ceiling}/Mtok):`));
|
|
3083
|
+
if (costCeiling) {
|
|
3084
|
+
const cc = payload.modes.costCeiling;
|
|
3085
|
+
output.writeln(` picked: ${output.success(cc.modelId)}`);
|
|
3086
|
+
output.writeln(` predicted Q: ${cc.predictedQuality.toFixed(4)}`);
|
|
3087
|
+
output.writeln(` cost ($/Mtok): $${cc.costPerMTok?.toFixed(2)}`);
|
|
3088
|
+
output.writeln(` met quality bar: ${cc.metBar ? '✓' : '✗'}`);
|
|
3089
|
+
}
|
|
3090
|
+
else {
|
|
3091
|
+
output.writeln(` ${output.warning('null')}`);
|
|
3092
|
+
}
|
|
3093
|
+
output.writeln('');
|
|
3094
|
+
if (costOptimal && costCeiling) {
|
|
3095
|
+
if (payload.sameModel) {
|
|
3096
|
+
output.writeln(output.dim(' Both modes picked the same model — selector choice irrelevant for this task.'));
|
|
3097
|
+
}
|
|
3098
|
+
else {
|
|
3099
|
+
const dq = payload.deltaQuality;
|
|
3100
|
+
const dc = payload.deltaCost;
|
|
3101
|
+
const qSign = dq > 0 ? '+' : '';
|
|
3102
|
+
const cSign = dc > 0 ? '+' : '';
|
|
3103
|
+
const qColor = dq > 0 ? output.success(`${qSign}${dq.toFixed(4)}`) : output.warning(`${dq.toFixed(4)}`);
|
|
3104
|
+
const cColor = dc > 0 ? output.warning(`${cSign}$${dc.toFixed(2)}`) : output.success(`$${dc.toFixed(2)}`);
|
|
3105
|
+
output.writeln(` Δ (ceiling − optimal): predicted Q: ${qColor} cost: ${cColor}`);
|
|
3106
|
+
output.writeln('');
|
|
3107
|
+
output.writeln(output.dim(` Cost-ceiling pays extra cost for higher quality (or is forced cheap if ceiling is tight).`));
|
|
3108
|
+
output.writeln(output.dim(` Cost-optimal accepts qualityBar threshold but minimizes spend.`));
|
|
3109
|
+
}
|
|
3110
|
+
}
|
|
3111
|
+
output.writeln('');
|
|
3112
|
+
return { success: true, data: payload };
|
|
3113
|
+
},
|
|
3114
|
+
};
|
|
3115
|
+
// ADR-149 iter 49 — single-command SRE dashboard. The router has 13 subcommands
|
|
3116
|
+
// (iter 48); ops want ONE that says "is everything working AND saving money?".
|
|
3117
|
+
// Aggregates the most-asked signals into one terse screen.
|
|
3118
|
+
const routerStatsSummaryCommand = {
|
|
3119
|
+
name: 'stats-summary',
|
|
3120
|
+
description: 'One-screen SRE dashboard: gate, recent activity, savings, bandit warmest cell (ADR-149 iter 49)',
|
|
3121
|
+
options: [
|
|
3122
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
3123
|
+
],
|
|
3124
|
+
examples: [
|
|
3125
|
+
{ command: 'claude-flow neural router stats-summary', description: 'One-screen health check' },
|
|
3126
|
+
{ command: 'claude-flow neural router stats-summary --format json | jq .', description: 'Pipe to dashboards / alerting' },
|
|
3127
|
+
],
|
|
3128
|
+
action: async (ctx) => {
|
|
3129
|
+
const fs = await import('node:fs');
|
|
3130
|
+
const path = await import('node:path');
|
|
3131
|
+
const fmt = ctx.flags.format || 'table';
|
|
3132
|
+
// 1. Backend gate / status
|
|
3133
|
+
const { neuralRouterStatus } = await import('../ruvector/neural-router.js');
|
|
3134
|
+
const { getModelRouterStats } = await import('../ruvector/model-router.js');
|
|
3135
|
+
const backend = await neuralRouterStatus();
|
|
3136
|
+
const stats = getModelRouterStats();
|
|
3137
|
+
// 2. Trajectory existence + basic counts
|
|
3138
|
+
const trajectoryPath = process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
3139
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
3140
|
+
let trajectory = { exists: false, rows: 0, decisions: 0, outcomes: 0, pairedWithCost: 0, oldestTs: null, newestTs: null };
|
|
3141
|
+
let recent24h = null;
|
|
3142
|
+
let cost7d = null;
|
|
3143
|
+
if (fs.existsSync(trajectoryPath)) {
|
|
3144
|
+
trajectory.exists = true;
|
|
3145
|
+
const { MODEL_PRICES } = await import('../ruvector/model-prices.js');
|
|
3146
|
+
const lines = fs.readFileSync(trajectoryPath, 'utf8').split('\n').filter(l => l.trim().length > 0);
|
|
3147
|
+
trajectory.rows = lines.length;
|
|
3148
|
+
const cutoff24h = Date.now() - 24 * 3600_000;
|
|
3149
|
+
const cutoff7d = Date.now() - 7 * 86400_000;
|
|
3150
|
+
let recDec = 0, recFallback = 0;
|
|
3151
|
+
const decs7d = new Map();
|
|
3152
|
+
const outs7d = new Map();
|
|
3153
|
+
for (const l of lines) {
|
|
3154
|
+
try {
|
|
3155
|
+
const r = JSON.parse(l);
|
|
3156
|
+
const tsMs = Date.parse(r.ts);
|
|
3157
|
+
if (r.type === 'decision') {
|
|
3158
|
+
trajectory.decisions++;
|
|
3159
|
+
if (tsMs >= cutoff24h) {
|
|
3160
|
+
recDec++;
|
|
3161
|
+
if (r.routed_by === 'bandit-fallback')
|
|
3162
|
+
recFallback++;
|
|
3163
|
+
}
|
|
3164
|
+
if (tsMs >= cutoff7d)
|
|
3165
|
+
decs7d.set(r.task_hash, r);
|
|
3166
|
+
}
|
|
3167
|
+
else if (r.type === 'outcome') {
|
|
3168
|
+
trajectory.outcomes++;
|
|
3169
|
+
if (r.cost_usd != null)
|
|
3170
|
+
trajectory.pairedWithCost++;
|
|
3171
|
+
if (tsMs >= cutoff7d)
|
|
3172
|
+
outs7d.set(r.task_hash, r);
|
|
3173
|
+
}
|
|
3174
|
+
if (r.ts) {
|
|
3175
|
+
if (!trajectory.oldestTs || r.ts < trajectory.oldestTs)
|
|
3176
|
+
trajectory.oldestTs = r.ts;
|
|
3177
|
+
if (!trajectory.newestTs || r.ts > trajectory.newestTs)
|
|
3178
|
+
trajectory.newestTs = r.ts;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
catch { /* malformed */ }
|
|
3182
|
+
}
|
|
3183
|
+
recent24h = {
|
|
3184
|
+
decisions: recDec, fallbacks: recFallback,
|
|
3185
|
+
fallbackRatePct: recDec > 0 ? Math.round((recFallback / recDec) * 10000) / 100 : 0,
|
|
3186
|
+
};
|
|
3187
|
+
// 7-day cost via heuristic counterfactual
|
|
3188
|
+
let pairs = 0, actual = 0, cf = 0;
|
|
3189
|
+
for (const [hash, dec] of decs7d) {
|
|
3190
|
+
const out = outs7d.get(hash);
|
|
3191
|
+
if (!out?.cost_usd || !out.tokens)
|
|
3192
|
+
continue;
|
|
3193
|
+
pairs++;
|
|
3194
|
+
actual += out.cost_usd;
|
|
3195
|
+
const tierModel = dec.complexity < 0.34 ? 'haiku' : dec.complexity < 0.67 ? 'sonnet' : 'opus';
|
|
3196
|
+
const cfModel = dec.ab_pair?.bandit_pick ?? tierModel;
|
|
3197
|
+
const p = MODEL_PRICES[cfModel] ?? { in: 1, out: 1 };
|
|
3198
|
+
cf += (out.tokens.input * p.in + out.tokens.output * p.out) / 1_000_000;
|
|
3199
|
+
}
|
|
3200
|
+
if (pairs > 0) {
|
|
3201
|
+
cost7d = {
|
|
3202
|
+
pairs,
|
|
3203
|
+
actualUsd: Math.round(actual * 1_000_000) / 1_000_000,
|
|
3204
|
+
counterfactualUsd: Math.round(cf * 1_000_000) / 1_000_000,
|
|
3205
|
+
savingsUsd: Math.round((cf - actual) * 1_000_000) / 1_000_000,
|
|
3206
|
+
savingsPct: cf > 0 ? Math.round(((cf - actual) / cf) * 10000) / 100 : 0,
|
|
3207
|
+
};
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
let warmestCell = null;
|
|
3211
|
+
const statePath = path.resolve(process.cwd(), '.swarm', 'model-router-state.json');
|
|
3212
|
+
if (fs.existsSync(statePath)) {
|
|
3213
|
+
try {
|
|
3214
|
+
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
3215
|
+
const priors = state.priorsById ?? state.priors ?? {};
|
|
3216
|
+
let bestSamples = 0;
|
|
3217
|
+
for (const bucket of ['low', 'med', 'high']) {
|
|
3218
|
+
const b = priors[bucket];
|
|
3219
|
+
if (!b)
|
|
3220
|
+
continue;
|
|
3221
|
+
for (const [k, p] of Object.entries(b)) {
|
|
3222
|
+
const samples = p.alpha + p.beta - 2;
|
|
3223
|
+
if (samples > bestSamples) {
|
|
3224
|
+
bestSamples = samples;
|
|
3225
|
+
warmestCell = { bucket, key: k, samples, meanQuality: p.alpha / (p.alpha + p.beta) };
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
catch { /* malformed */ }
|
|
3231
|
+
}
|
|
3232
|
+
const payload = {
|
|
3233
|
+
backend: {
|
|
3234
|
+
enabled: backend.enabled,
|
|
3235
|
+
available: backend.available,
|
|
3236
|
+
routedBy: backend.routedBy,
|
|
3237
|
+
reason: backend.reason,
|
|
3238
|
+
},
|
|
3239
|
+
processLocal: {
|
|
3240
|
+
totalDecisions: stats.totalDecisions,
|
|
3241
|
+
modelDistribution: stats.modelDistribution,
|
|
3242
|
+
routedByCounts: stats.routedByCounts,
|
|
3243
|
+
abDisagreementRate: stats.ab.comparisons > 0 ? Math.round(stats.ab.disagreementRate * 10000) / 100 : 0,
|
|
3244
|
+
},
|
|
3245
|
+
trajectory,
|
|
3246
|
+
recent24h,
|
|
3247
|
+
cost7d,
|
|
3248
|
+
warmestBanditCell: warmestCell,
|
|
3249
|
+
};
|
|
3250
|
+
if (fmt === 'json') {
|
|
3251
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3252
|
+
return { success: true, data: payload };
|
|
3253
|
+
}
|
|
3254
|
+
output.writeln();
|
|
3255
|
+
output.writeln(output.bold('Router stats summary — ADR-149 iter 49 (one-screen SRE view)'));
|
|
3256
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3257
|
+
output.writeln('');
|
|
3258
|
+
output.writeln(output.bold(' Backend:'));
|
|
3259
|
+
output.writeln(` gate: ${backend.enabled ? output.success('open (NEURAL=1)') : output.warning('closed')}`);
|
|
3260
|
+
output.writeln(` available: ${backend.available ? output.success('yes') : output.warning('no')}`);
|
|
3261
|
+
output.writeln(` active backend: ${backend.routedBy ?? '—'}`);
|
|
3262
|
+
output.writeln(` reason: ${backend.reason}`);
|
|
3263
|
+
output.writeln('');
|
|
3264
|
+
output.writeln(output.bold(' Process-local (since this server started):'));
|
|
3265
|
+
output.writeln(` decisions: ${stats.totalDecisions}`);
|
|
3266
|
+
output.writeln(` routed_by: heuristic=${stats.routedByCounts.heuristic} hybrid=${stats.routedByCounts.hybrid} bandit-fallback=${stats.routedByCounts['bandit-fallback']}`);
|
|
3267
|
+
if (stats.ab.comparisons > 0) {
|
|
3268
|
+
output.writeln(` A/B: ${stats.ab.comparisons} comparisons, ${stats.ab.disagreements} disagreements (${(stats.ab.disagreementRate * 100).toFixed(1)}%)`);
|
|
3269
|
+
}
|
|
3270
|
+
output.writeln('');
|
|
3271
|
+
output.writeln(output.bold(' Trajectory log:'));
|
|
3272
|
+
if (!trajectory.exists) {
|
|
3273
|
+
output.writeln(` ${output.warning('file does not exist — recorder OFF or no decisions made')}`);
|
|
3274
|
+
}
|
|
3275
|
+
else {
|
|
3276
|
+
output.writeln(` rows: ${trajectory.rows} (${trajectory.decisions} decisions / ${trajectory.outcomes} outcomes / ${trajectory.pairedWithCost} cost-bearing)`);
|
|
3277
|
+
if (trajectory.oldestTs && trajectory.newestTs) {
|
|
3278
|
+
output.writeln(` span: ${trajectory.oldestTs.slice(0, 19)} → ${trajectory.newestTs.slice(0, 19)}`);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
output.writeln('');
|
|
3282
|
+
if (recent24h) {
|
|
3283
|
+
const rateStr = recent24h.fallbackRatePct > 30 ? output.warning(recent24h.fallbackRatePct + '% ⚠')
|
|
3284
|
+
: recent24h.fallbackRatePct > 10 ? recent24h.fallbackRatePct + '%'
|
|
3285
|
+
: output.success(recent24h.fallbackRatePct + '%');
|
|
3286
|
+
output.writeln(output.bold(` Last 24h:`));
|
|
3287
|
+
output.writeln(` decisions: ${recent24h.decisions}`);
|
|
3288
|
+
output.writeln(` fallback rate: ${rateStr} (neural backend → bandit when prediction unusable)`);
|
|
3289
|
+
output.writeln('');
|
|
3290
|
+
}
|
|
3291
|
+
if (cost7d) {
|
|
3292
|
+
const savingsStr = cost7d.savingsUsd >= 0
|
|
3293
|
+
? output.success(`$${cost7d.savingsUsd.toFixed(4)}`)
|
|
3294
|
+
: output.warning(`-$${Math.abs(cost7d.savingsUsd).toFixed(4)}`);
|
|
3295
|
+
output.writeln(output.bold(' Last 7d cost-savings (vs heuristic baseline):'));
|
|
3296
|
+
output.writeln(` paired calls: ${cost7d.pairs}`);
|
|
3297
|
+
output.writeln(` actual: $${cost7d.actualUsd.toFixed(4)}`);
|
|
3298
|
+
output.writeln(` counterfactual: $${cost7d.counterfactualUsd.toFixed(4)}`);
|
|
3299
|
+
output.writeln(` savings: ${savingsStr} (${cost7d.savingsPct}%)`);
|
|
3300
|
+
output.writeln('');
|
|
3301
|
+
}
|
|
3302
|
+
if (warmestCell) {
|
|
3303
|
+
output.writeln(output.bold(' Bandit warmest cell:'));
|
|
3304
|
+
output.writeln(` ${warmestCell.bucket} × ${warmestCell.key} → ${warmestCell.samples} samples, meanQ=${warmestCell.meanQuality.toFixed(3)}`);
|
|
3305
|
+
output.writeln('');
|
|
3306
|
+
}
|
|
3307
|
+
output.writeln(output.dim(' For drill-down: `router decisions`, `router cost-savings`, `router bandit-state`'));
|
|
3308
|
+
output.writeln('');
|
|
3309
|
+
return { success: true, data: payload };
|
|
3310
|
+
},
|
|
3311
|
+
};
|
|
3312
|
+
// ADR-149 iter 48 — bandit-state inspection. The persisted bandit posteriors
|
|
3313
|
+
// (`.swarm/model-router-state.json`) accumulate across restarts but are
|
|
3314
|
+
// otherwise invisible. Surfacing the (bucket × model) prior matrix lets
|
|
3315
|
+
// operators see where bandit learning is thin (cold cells) and where it's
|
|
3316
|
+
// confident (large α+β).
|
|
3317
|
+
const routerBanditStateCommand = {
|
|
3318
|
+
name: 'bandit-state',
|
|
3319
|
+
description: 'Inspect persisted bandit Beta priors per bucket × model (ADR-149 iter 48)',
|
|
3320
|
+
options: [
|
|
3321
|
+
{ name: 'path', type: 'string', description: 'Path to model-router-state.json (default: .swarm/model-router-state.json)' },
|
|
3322
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
3323
|
+
{ name: 'cold-threshold', type: 'number', description: 'Highlight cells with sample count below this (default 4 — matches iter 14 density guard)', default: '4' },
|
|
3324
|
+
],
|
|
3325
|
+
examples: [
|
|
3326
|
+
{ command: 'claude-flow neural router bandit-state', description: 'Show all Beta priors per bucket × tier + per bucket × modelId' },
|
|
3327
|
+
{ command: 'claude-flow neural router bandit-state --format json | jq .priorsById', description: 'Just the per-modelId matrix' },
|
|
3328
|
+
],
|
|
3329
|
+
action: async (ctx) => {
|
|
3330
|
+
const fs = await import('node:fs');
|
|
3331
|
+
const path = await import('node:path');
|
|
3332
|
+
const statePath = ctx.flags.path
|
|
3333
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-state.json');
|
|
3334
|
+
const fmt = ctx.flags.format || 'table';
|
|
3335
|
+
const coldThreshold = parseInt(ctx.flags['cold-threshold'] || '4', 10) || 4;
|
|
3336
|
+
if (!fs.existsSync(statePath)) {
|
|
3337
|
+
const msg = `Bandit state file not found at ${statePath}`;
|
|
3338
|
+
if (fmt === 'json')
|
|
3339
|
+
output.writeln(JSON.stringify({ error: msg, hint: 'State is created on first routing decision. Run any agent_spawn flow with CLAUDE_FLOW_ROUTER_NEURAL=1.' }, null, 2));
|
|
3340
|
+
else {
|
|
3341
|
+
output.printError(msg);
|
|
3342
|
+
output.writeln(output.dim(' State is created on first routing decision. Run any agent_spawn flow.'));
|
|
3343
|
+
}
|
|
3344
|
+
return { success: false, exitCode: 1 };
|
|
3345
|
+
}
|
|
3346
|
+
let state;
|
|
3347
|
+
try {
|
|
3348
|
+
state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
|
3349
|
+
}
|
|
3350
|
+
catch (err) {
|
|
3351
|
+
const msg = `Failed to parse bandit state: ${err instanceof Error ? err.message : String(err)}`;
|
|
3352
|
+
if (fmt === 'json')
|
|
3353
|
+
output.writeln(JSON.stringify({ error: msg }, null, 2));
|
|
3354
|
+
else
|
|
3355
|
+
output.printError(msg);
|
|
3356
|
+
return { success: false, exitCode: 1 };
|
|
3357
|
+
}
|
|
3358
|
+
// Helper: enumerate all cells from a BucketedPriors with derived stats.
|
|
3359
|
+
const cells = (priors) => {
|
|
3360
|
+
const out = [];
|
|
3361
|
+
if (!priors)
|
|
3362
|
+
return out;
|
|
3363
|
+
for (const bucket of ['low', 'med', 'high']) {
|
|
3364
|
+
const b = priors[bucket];
|
|
3365
|
+
if (!b)
|
|
3366
|
+
continue;
|
|
3367
|
+
for (const [k, p] of Object.entries(b)) {
|
|
3368
|
+
const samples = p.alpha + p.beta - 2; // -2 because Beta(1,1) is the uniform prior
|
|
3369
|
+
const meanQuality = p.alpha / (p.alpha + p.beta);
|
|
3370
|
+
out.push({ bucket, key: k, alpha: p.alpha, beta: p.beta, samples, meanQuality, cold: samples < coldThreshold });
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
return out;
|
|
3374
|
+
};
|
|
3375
|
+
const tierCells = cells(state.priors);
|
|
3376
|
+
const idCells = cells(state.priorsById);
|
|
3377
|
+
const coldTierCells = tierCells.filter(c => c.cold);
|
|
3378
|
+
const coldIdCells = idCells.filter(c => c.cold);
|
|
3379
|
+
const payload = {
|
|
3380
|
+
input: statePath,
|
|
3381
|
+
stateVersion: state.version ?? 2,
|
|
3382
|
+
totalDecisions: state.totalDecisions ?? 0,
|
|
3383
|
+
lastUpdated: state.lastUpdated ?? null,
|
|
3384
|
+
coldThreshold,
|
|
3385
|
+
priors: tierCells,
|
|
3386
|
+
priorsById: idCells,
|
|
3387
|
+
summary: {
|
|
3388
|
+
tierCells: tierCells.length,
|
|
3389
|
+
coldTierCells: coldTierCells.length,
|
|
3390
|
+
idCells: idCells.length,
|
|
3391
|
+
coldIdCells: coldIdCells.length,
|
|
3392
|
+
warmestIdCell: idCells.length > 0 ? [...idCells].sort((a, b) => b.samples - a.samples)[0] : null,
|
|
3393
|
+
},
|
|
3394
|
+
};
|
|
3395
|
+
if (fmt === 'json') {
|
|
3396
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3397
|
+
return { success: true, data: payload };
|
|
3398
|
+
}
|
|
3399
|
+
output.writeln();
|
|
3400
|
+
output.writeln(output.bold('Bandit state inspection (ADR-149 iter 48)'));
|
|
3401
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3402
|
+
output.writeln(` Path: ${statePath}`);
|
|
3403
|
+
output.writeln(` Schema version: v${state.version ?? 2} (v2=tier priors only; v3=adds priorsById)`);
|
|
3404
|
+
output.writeln(` Total decisions: ${state.totalDecisions ?? 0}`);
|
|
3405
|
+
if (state.lastUpdated)
|
|
3406
|
+
output.writeln(` Last update: ${state.lastUpdated}`);
|
|
3407
|
+
output.writeln(` Cold threshold: samples < ${coldThreshold} (iter 14 density-guard cutoff)`);
|
|
3408
|
+
output.writeln('');
|
|
3409
|
+
const renderTable = (label, rows) => {
|
|
3410
|
+
if (rows.length === 0) {
|
|
3411
|
+
output.writeln(` ${label}: (empty — no outcomes recorded yet for this layer)`);
|
|
3412
|
+
output.writeln('');
|
|
3413
|
+
return;
|
|
3414
|
+
}
|
|
3415
|
+
output.writeln(output.bold(` ${label}:`));
|
|
3416
|
+
output.writeln(' bucket key α β samples meanQ');
|
|
3417
|
+
for (const c of rows.sort((a, b) => (a.bucket.localeCompare(b.bucket)) || (b.samples - a.samples))) {
|
|
3418
|
+
const coldMark = c.cold ? output.warning(' ❄ cold') : '';
|
|
3419
|
+
output.writeln(` ${c.bucket.padEnd(6)} ${c.key.padEnd(38)} ${c.alpha.toFixed(1).padStart(5)} ${c.beta.toFixed(1).padStart(5)} ${String(c.samples).padStart(7)} ${c.meanQuality.toFixed(3)}${coldMark}`);
|
|
3420
|
+
}
|
|
3421
|
+
output.writeln('');
|
|
3422
|
+
};
|
|
3423
|
+
renderTable('Tier priors (bucket × tier label)', tierCells);
|
|
3424
|
+
renderTable('Per-modelId priors (bucket × concrete modelId, iter 14)', idCells);
|
|
3425
|
+
output.writeln(output.bold(' Summary:'));
|
|
3426
|
+
output.writeln(` tier cells: ${tierCells.length} (cold: ${coldTierCells.length})`);
|
|
3427
|
+
output.writeln(` per-modelId cells: ${idCells.length} (cold: ${coldIdCells.length})`);
|
|
3428
|
+
if (payload.summary.warmestIdCell) {
|
|
3429
|
+
const w = payload.summary.warmestIdCell;
|
|
3430
|
+
output.writeln(` warmest cell: ${w.bucket} × ${w.key} (${w.samples} samples, meanQ=${w.meanQuality.toFixed(3)})`);
|
|
3431
|
+
}
|
|
3432
|
+
output.writeln('');
|
|
3433
|
+
if (coldIdCells.length > 0 && idCells.length > 0) {
|
|
3434
|
+
output.writeln(output.dim(` Cold cells suppress iter 14 per-modelId Thompson perturbation. Until α+β ≥ ${coldThreshold + 2},`));
|
|
3435
|
+
output.writeln(output.dim(' the neural prediction dominates that (bucket, modelId) pair without bandit correction.'));
|
|
3436
|
+
output.writeln('');
|
|
3437
|
+
}
|
|
3438
|
+
return { success: true, data: payload };
|
|
3439
|
+
},
|
|
3440
|
+
};
|
|
3441
|
+
// ADR-149 iter 38 — consumer for iter 37's sampled A/B mode. Aggregates
|
|
3442
|
+
// ab_pair from decision rows into a (bandit_pick × hybrid_pick) confusion
|
|
3443
|
+
// matrix plus disagreement rate. Operators see WHERE the neural prior
|
|
3444
|
+
// moves the bandit's decisions.
|
|
3445
|
+
const routerAbStatsCommand = {
|
|
3446
|
+
name: 'ab-stats',
|
|
3447
|
+
description: 'Aggregate A/B (bandit-vs-hybrid) disagreement from trajectory ab_pair rows (ADR-149 iter 38)',
|
|
3448
|
+
options: [
|
|
3449
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/...)' },
|
|
3450
|
+
{ name: 'since', short: 's', type: 'string', description: 'Time window suffix: 1h, 24h, 7d, 30d' },
|
|
3451
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
3452
|
+
],
|
|
3453
|
+
examples: [
|
|
3454
|
+
{ command: 'claude-flow neural router ab-stats', description: 'All A/B comparisons recorded so far' },
|
|
3455
|
+
{ command: 'claude-flow neural router ab-stats --since 7d --format json', description: 'Last 7 days, pipe-friendly' },
|
|
3456
|
+
{ command: 'CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE=0.05 ... && claude-flow neural router ab-stats', description: 'After running with iter 37 sampling on' },
|
|
3457
|
+
],
|
|
3458
|
+
action: async (ctx) => {
|
|
3459
|
+
const fs = await import('node:fs');
|
|
3460
|
+
const path = await import('node:path');
|
|
3461
|
+
const inPath = ctx.flags.in
|
|
3462
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
3463
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
3464
|
+
const since = ctx.flags.since;
|
|
3465
|
+
const fmt = ctx.flags.format || 'table';
|
|
3466
|
+
if (!fs.existsSync(inPath)) {
|
|
3467
|
+
const msg = `Trajectory file not found at ${inPath}`;
|
|
3468
|
+
if (fmt === 'json')
|
|
3469
|
+
output.writeln(JSON.stringify({ error: msg }, null, 2));
|
|
3470
|
+
else {
|
|
3471
|
+
output.printError(msg);
|
|
3472
|
+
output.writeln(output.dim(' Set CLAUDE_FLOW_ROUTER_TRAJECTORY=1 + CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE=0.05 to start collecting ab_pair data.'));
|
|
3473
|
+
}
|
|
3474
|
+
return { success: false, exitCode: 1 };
|
|
3475
|
+
}
|
|
3476
|
+
// Parse + filter to decision rows that carry ab_pair.
|
|
3477
|
+
let cutoffMs = null;
|
|
3478
|
+
if (since) {
|
|
3479
|
+
const m = since.match(/^(\d+)([hdmw])$/);
|
|
3480
|
+
if (m) {
|
|
3481
|
+
const n = parseInt(m[1], 10);
|
|
3482
|
+
const unitMs = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]] ?? 0;
|
|
3483
|
+
cutoffMs = Date.now() - n * unitMs;
|
|
3484
|
+
}
|
|
3485
|
+
}
|
|
3486
|
+
const rows = fs.readFileSync(inPath, 'utf8').split('\n').filter(l => l.trim().length > 0);
|
|
3487
|
+
let totalDecisions = 0;
|
|
3488
|
+
let malformed = 0;
|
|
3489
|
+
const abRows = [];
|
|
3490
|
+
for (const l of rows) {
|
|
3491
|
+
try {
|
|
3492
|
+
const r = JSON.parse(l);
|
|
3493
|
+
if (r.type !== 'decision')
|
|
3494
|
+
continue;
|
|
3495
|
+
totalDecisions++;
|
|
3496
|
+
if (!r.ab_pair)
|
|
3497
|
+
continue;
|
|
3498
|
+
if (cutoffMs !== null && Date.parse(r.ts) < cutoffMs)
|
|
3499
|
+
continue;
|
|
3500
|
+
abRows.push(r);
|
|
3501
|
+
}
|
|
3502
|
+
catch {
|
|
3503
|
+
malformed++;
|
|
3504
|
+
}
|
|
3505
|
+
}
|
|
3506
|
+
const cells = new Map();
|
|
3507
|
+
let disagree = 0;
|
|
3508
|
+
const banditTotals = {};
|
|
3509
|
+
const hybridTotals = {};
|
|
3510
|
+
for (const r of abRows) {
|
|
3511
|
+
const ap = r.ab_pair;
|
|
3512
|
+
const key = `${ap.bandit_pick}→${ap.hybrid_pick}`;
|
|
3513
|
+
const cell = cells.get(key) ?? { bandit: ap.bandit_pick, hybrid: ap.hybrid_pick, count: 0 };
|
|
3514
|
+
cell.count++;
|
|
3515
|
+
cells.set(key, cell);
|
|
3516
|
+
banditTotals[ap.bandit_pick] = (banditTotals[ap.bandit_pick] ?? 0) + 1;
|
|
3517
|
+
hybridTotals[ap.hybrid_pick] = (hybridTotals[ap.hybrid_pick] ?? 0) + 1;
|
|
3518
|
+
if (ap.disagree)
|
|
3519
|
+
disagree++;
|
|
3520
|
+
}
|
|
3521
|
+
const disagreeRatePct = abRows.length > 0 ? Math.round((disagree / abRows.length) * 10000) / 100 : 0;
|
|
3522
|
+
// Models actually seen, in stable iteration order.
|
|
3523
|
+
const allModels = Array.from(new Set([...Object.keys(banditTotals), ...Object.keys(hybridTotals)])).sort();
|
|
3524
|
+
const matrix = {};
|
|
3525
|
+
for (const b of allModels) {
|
|
3526
|
+
matrix[b] = {};
|
|
3527
|
+
for (const h of allModels) {
|
|
3528
|
+
matrix[b][h] = cells.get(`${b}→${h}`)?.count ?? 0;
|
|
3529
|
+
}
|
|
3530
|
+
}
|
|
3531
|
+
// Disagreement breakdown — off-diagonal cells sorted by count desc.
|
|
3532
|
+
const offDiag = [];
|
|
3533
|
+
for (const c of cells.values()) {
|
|
3534
|
+
if (c.bandit !== c.hybrid) {
|
|
3535
|
+
offDiag.push({ ...c, pctOfDisagrees: disagree > 0 ? Math.round((c.count / disagree) * 10000) / 100 : 0 });
|
|
3536
|
+
}
|
|
3537
|
+
}
|
|
3538
|
+
offDiag.sort((a, b) => b.count - a.count);
|
|
3539
|
+
const payload = {
|
|
3540
|
+
input: inPath,
|
|
3541
|
+
filters: { since },
|
|
3542
|
+
totalDecisions, malformed,
|
|
3543
|
+
abComparisons: abRows.length,
|
|
3544
|
+
disagreements: disagree,
|
|
3545
|
+
disagreementRatePct: disagreeRatePct,
|
|
3546
|
+
models: allModels,
|
|
3547
|
+
banditTotals, hybridTotals,
|
|
3548
|
+
confusionMatrix: matrix,
|
|
3549
|
+
disagreementBreakdown: offDiag,
|
|
3550
|
+
coveragePct: totalDecisions > 0 ? Math.round((abRows.length / totalDecisions) * 10000) / 100 : 0,
|
|
3551
|
+
};
|
|
3552
|
+
if (fmt === 'json') {
|
|
3553
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3554
|
+
return { success: true, data: payload };
|
|
3555
|
+
}
|
|
3556
|
+
output.writeln();
|
|
3557
|
+
output.writeln(output.bold('A/B (bandit vs hybrid) stats — ADR-149 iter 37/38'));
|
|
3558
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3559
|
+
output.writeln(` Input: ${inPath}`);
|
|
3560
|
+
if (since)
|
|
3561
|
+
output.writeln(` Time window: since ${since}`);
|
|
3562
|
+
output.writeln(` Total decisions: ${totalDecisions} (${malformed} malformed)`);
|
|
3563
|
+
output.writeln(` A/B comparisons: ${abRows.length} (${payload.coveragePct}% of decisions had ab_pair)`);
|
|
3564
|
+
output.writeln('');
|
|
3565
|
+
if (abRows.length === 0) {
|
|
3566
|
+
output.writeln(output.dim(' No ab_pair rows found. Enable iter 37 sampling:'));
|
|
3567
|
+
output.writeln(output.dim(' export CLAUDE_FLOW_ROUTER_TRAJECTORY=1'));
|
|
3568
|
+
output.writeln(output.dim(' export CLAUDE_FLOW_ROUTER_AB_SAMPLE_RATE=0.05 # 5% sampling'));
|
|
3569
|
+
output.writeln('');
|
|
3570
|
+
return { success: true, data: payload };
|
|
3571
|
+
}
|
|
3572
|
+
output.writeln(` Disagreements: ${disagree} (${disagreeRatePct}% of A/B comparisons)`);
|
|
3573
|
+
output.writeln('');
|
|
3574
|
+
output.writeln(output.bold(` Confusion matrix (rows = bandit pick, cols = hybrid pick, ${allModels.length} models):`));
|
|
3575
|
+
const headerPad = Math.max(8, ...allModels.map(m => m.length));
|
|
3576
|
+
output.writeln(' ' + 'bandit \\ hybrid'.padEnd(headerPad) + ' ' + allModels.map(m => m.padStart(8)).join(''));
|
|
3577
|
+
for (const b of allModels) {
|
|
3578
|
+
const cells = allModels.map(h => String(matrix[b][h] ?? 0).padStart(8));
|
|
3579
|
+
output.writeln(` ${b.padEnd(headerPad)} ${cells.join('')}`);
|
|
3580
|
+
}
|
|
3581
|
+
output.writeln(output.dim(' (Diagonal cells = agreement, off-diagonal = disagreement.)'));
|
|
3582
|
+
output.writeln('');
|
|
3583
|
+
if (offDiag.length > 0) {
|
|
3584
|
+
output.writeln(output.bold(' Disagreement breakdown (bandit → hybrid):'));
|
|
3585
|
+
output.writeln(' transition'.padEnd(40) + ' count % of disagrees');
|
|
3586
|
+
for (const c of offDiag) {
|
|
3587
|
+
output.writeln(` ${(c.bandit + ' → ' + c.hybrid).padEnd(38)} ${String(c.count).padStart(5)} ${c.pctOfDisagrees.toString().padStart(6)}%`);
|
|
3588
|
+
}
|
|
3589
|
+
output.writeln('');
|
|
3590
|
+
}
|
|
3591
|
+
return { success: true, data: payload };
|
|
3592
|
+
},
|
|
3593
|
+
};
|
|
3594
|
+
// ADR-149 iter 43 — show the canonical price table that drives cost
|
|
3595
|
+
// computations, blended-price routing, and counterfactual baselines.
|
|
3596
|
+
// Operators ask "what does the router think gpt-4.1 costs?" frequently;
|
|
3597
|
+
// previously the answer required reading src/ruvector/model-prices.ts.
|
|
3598
|
+
const routerPricesCommand = {
|
|
3599
|
+
name: 'prices',
|
|
3600
|
+
description: 'Show the per-model price table that drives blended cost + counterfactual computations (ADR-149 iter 43)',
|
|
3601
|
+
options: [
|
|
3602
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
3603
|
+
{ name: 'sort', type: 'string', description: 'Sort by: blended (default), input, output, name', default: 'blended' },
|
|
3604
|
+
],
|
|
3605
|
+
examples: [
|
|
3606
|
+
{ command: 'claude-flow neural router prices', description: 'Show all models sorted by blended price' },
|
|
3607
|
+
{ command: 'claude-flow neural router prices --sort name', description: 'Sort alphabetically' },
|
|
3608
|
+
{ command: 'claude-flow neural router prices --format json | jq \'.[] | select(.id | contains("opus"))\'', description: 'Filter via jq' },
|
|
3609
|
+
],
|
|
3610
|
+
action: async (ctx) => {
|
|
3611
|
+
const { MODEL_PRICES, blendedPrice } = await import('../ruvector/model-prices.js');
|
|
3612
|
+
const fmt = ctx.flags.format || 'table';
|
|
3613
|
+
const sortKey = (ctx.flags.sort || 'blended').toLowerCase();
|
|
3614
|
+
const rows = Object.entries(MODEL_PRICES).map(([id, p]) => ({
|
|
3615
|
+
id,
|
|
3616
|
+
inPerMtok: p.in,
|
|
3617
|
+
outPerMtok: p.out,
|
|
3618
|
+
blendedPerMtok: blendedPrice(id),
|
|
3619
|
+
}));
|
|
3620
|
+
switch (sortKey) {
|
|
3621
|
+
case 'input':
|
|
3622
|
+
rows.sort((a, b) => a.inPerMtok - b.inPerMtok);
|
|
3623
|
+
break;
|
|
3624
|
+
case 'output':
|
|
3625
|
+
rows.sort((a, b) => a.outPerMtok - b.outPerMtok);
|
|
3626
|
+
break;
|
|
3627
|
+
case 'name':
|
|
3628
|
+
rows.sort((a, b) => a.id.localeCompare(b.id));
|
|
3629
|
+
break;
|
|
3630
|
+
case 'blended':
|
|
3631
|
+
default: rows.sort((a, b) => a.blendedPerMtok - b.blendedPerMtok);
|
|
3632
|
+
}
|
|
3633
|
+
if (fmt === 'json') {
|
|
3634
|
+
output.writeln(JSON.stringify(rows, null, 2));
|
|
3635
|
+
return { success: true, data: rows };
|
|
3636
|
+
}
|
|
3637
|
+
output.writeln();
|
|
3638
|
+
output.writeln(output.bold('Model price table (ADR-149 iter 31/43 — single source of truth)'));
|
|
3639
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3640
|
+
output.writeln(` ${rows.length} entries, sorted by ${sortKey}`);
|
|
3641
|
+
output.writeln('');
|
|
3642
|
+
output.writeln(' model id $/Mtok in $/Mtok out blended ($/Mtok)');
|
|
3643
|
+
for (const r of rows) {
|
|
3644
|
+
output.writeln(` ${r.id.padEnd(44)} ${('$' + r.inPerMtok.toFixed(2)).padStart(9)} ${('$' + r.outPerMtok.toFixed(2)).padStart(11)} ${('$' + r.blendedPerMtok.toFixed(2)).padStart(15)}`);
|
|
3645
|
+
}
|
|
3646
|
+
output.writeln('');
|
|
3647
|
+
output.writeln(output.dim(' Blended = $/Mtok_in + 3 × $/Mtok_out (1 input : 3 output ratio for code tasks).'));
|
|
3648
|
+
output.writeln(output.dim(' Unknown model ids fall back to $1/Mtok blended (1×in + 1×out).'));
|
|
3649
|
+
output.writeln('');
|
|
3650
|
+
return { success: true, data: rows };
|
|
3651
|
+
},
|
|
3652
|
+
};
|
|
3653
|
+
// ADR-149 iter 41 — forward-looking budget projection. Iter 32-34 measure
|
|
3654
|
+
// past cost (actual vs counterfactual, per-window drift). This subcommand
|
|
3655
|
+
// extrapolates: given the measured rate and average cost per decision over
|
|
3656
|
+
// a recent window, what will routing cost over the next 7d / 30d / 90d /
|
|
3657
|
+
// 365d? Also projects the heuristic baseline so operators see the savings
|
|
3658
|
+
// trajectory across a quarter or year.
|
|
3659
|
+
const routerCostProjectionCommand = {
|
|
3660
|
+
name: 'cost-projection',
|
|
3661
|
+
description: 'Project monthly/quarterly cost from measured rate (ADR-149 iter 41)',
|
|
3662
|
+
options: [
|
|
3663
|
+
{ name: 'in', short: 'i', type: 'string', description: 'Trajectory JSONL path (default: $CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH or .swarm/...)' },
|
|
3664
|
+
{ name: 'window', short: 'w', type: 'string', description: 'Measurement window to extrapolate FROM (default 7d). Format: 1h, 24h, 7d, 30d' },
|
|
3665
|
+
{ name: 'horizons', type: 'string', description: 'Projection horizons (CSV of duration suffixes). Default: 7d,30d,90d,365d' },
|
|
3666
|
+
{ name: 'format', short: 'f', type: 'string', description: 'Output format: table, json', default: 'table' },
|
|
3667
|
+
],
|
|
3668
|
+
examples: [
|
|
3669
|
+
{ command: 'claude-flow neural router cost-projection', description: 'Project from last 7d of data → 7d/30d/90d/365d horizons' },
|
|
3670
|
+
{ command: 'claude-flow neural router cost-projection --window 24h --horizons 7d,30d', description: 'Project from last day only' },
|
|
3671
|
+
{ command: 'claude-flow neural router cost-projection --format json | jq .horizons[1].projectedSavingsUsd', description: '30-day savings projection for dashboards' },
|
|
3672
|
+
],
|
|
3673
|
+
action: async (ctx) => {
|
|
3674
|
+
const fs = await import('node:fs');
|
|
3675
|
+
const path = await import('node:path');
|
|
3676
|
+
const { MODEL_PRICES } = await import('../ruvector/model-prices.js');
|
|
3677
|
+
const inPath = ctx.flags.in
|
|
3678
|
+
?? process.env.CLAUDE_FLOW_ROUTER_TRAJECTORY_PATH
|
|
3679
|
+
?? path.resolve(process.cwd(), '.swarm', 'model-router-trajectories.jsonl');
|
|
3680
|
+
const windowSpec = ctx.flags.window ?? '7d';
|
|
3681
|
+
const horizonSpecs = (ctx.flags.horizons ?? '7d,30d,90d,365d').split(',').map(s => s.trim());
|
|
3682
|
+
const fmt = ctx.flags.format || 'table';
|
|
3683
|
+
const parseDuration = (s) => {
|
|
3684
|
+
const m = s.match(/^(\d+)([hdmw])$/);
|
|
3685
|
+
if (!m)
|
|
3686
|
+
return null;
|
|
3687
|
+
const n = parseInt(m[1], 10);
|
|
3688
|
+
const unitMs = { m: 60_000, h: 3_600_000, d: 86_400_000, w: 7 * 86_400_000 }[m[2]] ?? 0;
|
|
3689
|
+
return n * unitMs;
|
|
3690
|
+
};
|
|
3691
|
+
const windowMs = parseDuration(windowSpec);
|
|
3692
|
+
if (!windowMs || windowMs <= 0) {
|
|
3693
|
+
output.printError(`--window must be Nh/Nd/Nm/Nw (got ${windowSpec})`);
|
|
3694
|
+
return { success: false, exitCode: 1 };
|
|
3695
|
+
}
|
|
3696
|
+
if (!fs.existsSync(inPath)) {
|
|
3697
|
+
const msg = `Trajectory file not found at ${inPath}`;
|
|
3698
|
+
if (fmt === 'json')
|
|
3699
|
+
output.writeln(JSON.stringify({ error: msg }, null, 2));
|
|
3700
|
+
else
|
|
3701
|
+
output.printError(msg);
|
|
3702
|
+
return { success: false, exitCode: 1 };
|
|
3703
|
+
}
|
|
3704
|
+
// iter 66 — outcomes now ARRAY (was Map). Same fix as iter 62/63/65:
|
|
3705
|
+
// a Map keyed by task_hash collapses multiple runs of the same task to
|
|
3706
|
+
// the LATEST outcome only, biasing the rate downward (pairCount =
|
|
3707
|
+
// unique-tasks, not unique-calls). Production projections were
|
|
3708
|
+
// systematically too small for any workload with recurring tasks.
|
|
3709
|
+
const decisions = new Map();
|
|
3710
|
+
const outcomes = [];
|
|
3711
|
+
let malformed = 0;
|
|
3712
|
+
const cutoffMs = Date.now() - windowMs;
|
|
3713
|
+
for (const l of fs.readFileSync(inPath, 'utf8').split('\n')) {
|
|
3714
|
+
if (!l.trim())
|
|
3715
|
+
continue;
|
|
3716
|
+
try {
|
|
3717
|
+
const r = JSON.parse(l);
|
|
3718
|
+
if (Date.parse(r.ts) < cutoffMs)
|
|
3719
|
+
continue;
|
|
3720
|
+
if (r.type === 'decision')
|
|
3721
|
+
decisions.set(r.task_hash, r);
|
|
3722
|
+
else if (r.type === 'outcome')
|
|
3723
|
+
outcomes.push(r);
|
|
3724
|
+
}
|
|
3725
|
+
catch {
|
|
3726
|
+
malformed++;
|
|
3727
|
+
}
|
|
3728
|
+
}
|
|
3729
|
+
// Pair + compute window totals. Iterate OUTCOMES (not deduped decisions);
|
|
3730
|
+
// each outcome row contributes its own cost + tokens to the rate.
|
|
3731
|
+
let pairCount = 0;
|
|
3732
|
+
let actualUsd = 0;
|
|
3733
|
+
let counterfactualUsd = 0; // heuristic = tier-by-complexity baseline (iter 32 default)
|
|
3734
|
+
let totalInputTokens = 0;
|
|
3735
|
+
let totalOutputTokens = 0;
|
|
3736
|
+
for (const out of outcomes) {
|
|
3737
|
+
if (!out?.cost_usd || !out.tokens)
|
|
3738
|
+
continue;
|
|
3739
|
+
const dec = decisions.get(out.task_hash);
|
|
3740
|
+
if (!dec)
|
|
3741
|
+
continue;
|
|
3742
|
+
pairCount++;
|
|
3743
|
+
actualUsd += out.cost_usd;
|
|
3744
|
+
totalInputTokens += out.tokens.input;
|
|
3745
|
+
totalOutputTokens += out.tokens.output;
|
|
3746
|
+
// Heuristic counterfactual: tier-by-complexity (matches iter 32 default).
|
|
3747
|
+
const tierModel = dec.complexity < 0.34 ? 'haiku'
|
|
3748
|
+
: dec.complexity < 0.67 ? 'sonnet' : 'opus';
|
|
3749
|
+
const cfModel = dec.ab_pair?.bandit_pick ?? tierModel;
|
|
3750
|
+
const p = MODEL_PRICES[cfModel] ?? { in: 1, out: 1 };
|
|
3751
|
+
counterfactualUsd += (out.tokens.input * p.in + out.tokens.output * p.out) / 1_000_000;
|
|
3752
|
+
}
|
|
3753
|
+
if (pairCount === 0) {
|
|
3754
|
+
const msg = `No paired cost-bearing rows in the last ${windowSpec}. Cannot project.`;
|
|
3755
|
+
const payload = { error: msg, windowSpec, hint: 'Enable CLAUDE_FLOW_ROUTER_TRAJECTORY=1 and run some routed agent calls first.' };
|
|
3756
|
+
if (fmt === 'json')
|
|
3757
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3758
|
+
else {
|
|
3759
|
+
output.printError(msg);
|
|
3760
|
+
output.writeln(output.dim(` ${payload.hint}`));
|
|
3761
|
+
}
|
|
3762
|
+
return { success: true, data: payload };
|
|
3763
|
+
}
|
|
3764
|
+
// Per-second rate from this window's pair count.
|
|
3765
|
+
const callsPerSecond = pairCount / (windowMs / 1000);
|
|
3766
|
+
const avgActualPerCall = actualUsd / pairCount;
|
|
3767
|
+
const avgCounterfactualPerCall = counterfactualUsd / pairCount;
|
|
3768
|
+
// Extrapolate to horizons. Costs scale linearly with calls; calls scale
|
|
3769
|
+
// linearly with elapsed time at the measured rate.
|
|
3770
|
+
const horizons = horizonSpecs.map(spec => {
|
|
3771
|
+
const ms = parseDuration(spec);
|
|
3772
|
+
if (!ms)
|
|
3773
|
+
return { spec, error: 'invalid duration' };
|
|
3774
|
+
const projectedCalls = Math.round(callsPerSecond * (ms / 1000));
|
|
3775
|
+
const projectedActualUsd = avgActualPerCall * projectedCalls;
|
|
3776
|
+
const projectedCounterfactualUsd = avgCounterfactualPerCall * projectedCalls;
|
|
3777
|
+
const projectedSavingsUsd = projectedCounterfactualUsd - projectedActualUsd;
|
|
3778
|
+
const projectedSavingsPct = projectedCounterfactualUsd > 0
|
|
3779
|
+
? (projectedSavingsUsd / projectedCounterfactualUsd) * 100 : 0;
|
|
3780
|
+
return {
|
|
3781
|
+
spec, durationMs: ms,
|
|
3782
|
+
projectedCalls,
|
|
3783
|
+
projectedActualUsd: Math.round(projectedActualUsd * 1_000_000) / 1_000_000,
|
|
3784
|
+
projectedCounterfactualUsd: Math.round(projectedCounterfactualUsd * 1_000_000) / 1_000_000,
|
|
3785
|
+
projectedSavingsUsd: Math.round(projectedSavingsUsd * 1_000_000) / 1_000_000,
|
|
3786
|
+
projectedSavingsPct: Math.round(projectedSavingsPct * 100) / 100,
|
|
3787
|
+
};
|
|
3788
|
+
});
|
|
3789
|
+
const payload = {
|
|
3790
|
+
input: inPath,
|
|
3791
|
+
window: windowSpec,
|
|
3792
|
+
malformed,
|
|
3793
|
+
measurement: {
|
|
3794
|
+
pairs: pairCount,
|
|
3795
|
+
actualUsd: Math.round(actualUsd * 1_000_000) / 1_000_000,
|
|
3796
|
+
counterfactualUsd: Math.round(counterfactualUsd * 1_000_000) / 1_000_000,
|
|
3797
|
+
savingsUsd: Math.round((counterfactualUsd - actualUsd) * 1_000_000) / 1_000_000,
|
|
3798
|
+
callsPerSecond: Math.round(callsPerSecond * 1_000_000) / 1_000_000,
|
|
3799
|
+
callsPerDay: Math.round(callsPerSecond * 86400 * 100) / 100,
|
|
3800
|
+
avgActualPerCall: Math.round(avgActualPerCall * 1_000_000) / 1_000_000,
|
|
3801
|
+
avgCounterfactualPerCall: Math.round(avgCounterfactualPerCall * 1_000_000) / 1_000_000,
|
|
3802
|
+
avgInputTokensPerCall: Math.round(totalInputTokens / pairCount),
|
|
3803
|
+
avgOutputTokensPerCall: Math.round(totalOutputTokens / pairCount),
|
|
3804
|
+
},
|
|
3805
|
+
horizons,
|
|
3806
|
+
};
|
|
3807
|
+
if (fmt === 'json') {
|
|
3808
|
+
output.writeln(JSON.stringify(payload, null, 2));
|
|
3809
|
+
return { success: true, data: payload };
|
|
3810
|
+
}
|
|
3811
|
+
output.writeln();
|
|
3812
|
+
output.writeln(output.bold('Cost projection (ADR-149 iter 41)'));
|
|
3813
|
+
output.writeln(output.dim('─'.repeat(72)));
|
|
3814
|
+
output.writeln(` Input: ${inPath}`);
|
|
3815
|
+
output.writeln(` Measurement window: last ${windowSpec} (${pairCount} paired calls)`);
|
|
3816
|
+
output.writeln('');
|
|
3817
|
+
output.writeln(output.bold(' Measured rate:'));
|
|
3818
|
+
output.writeln(` Calls/day: ${payload.measurement.callsPerDay}`);
|
|
3819
|
+
output.writeln(` Avg actual cost/call: $${payload.measurement.avgActualPerCall.toFixed(6)}`);
|
|
3820
|
+
output.writeln(` Avg counterfactual/call: $${payload.measurement.avgCounterfactualPerCall.toFixed(6)} (heuristic: cheap→haiku, mid→sonnet, strong→opus)`);
|
|
3821
|
+
output.writeln(` Avg tokens/call: ${payload.measurement.avgInputTokensPerCall} in / ${payload.measurement.avgOutputTokensPerCall} out`);
|
|
3822
|
+
output.writeln('');
|
|
3823
|
+
output.writeln(output.bold(' Projections (linear extrapolation from measured rate):'));
|
|
3824
|
+
output.writeln(' horizon projected calls actual $ counterfactual $ savings $ %');
|
|
3825
|
+
for (const h of horizons) {
|
|
3826
|
+
if ('error' in h) {
|
|
3827
|
+
output.writeln(` ${h.spec.padEnd(7)} invalid duration`);
|
|
3828
|
+
continue;
|
|
3829
|
+
}
|
|
3830
|
+
const savingsStr = h.projectedSavingsUsd >= 0
|
|
3831
|
+
? output.success(`$${h.projectedSavingsUsd.toFixed(2)}`)
|
|
3832
|
+
: output.warning(`-$${Math.abs(h.projectedSavingsUsd).toFixed(2)}`);
|
|
3833
|
+
output.writeln(` ${h.spec.padEnd(7)} ${String(h.projectedCalls).padStart(15)} $${h.projectedActualUsd.toFixed(2).padStart(12)} $${h.projectedCounterfactualUsd.toFixed(2).padStart(14)} ${savingsStr.padEnd(14)} ${h.projectedSavingsPct.toFixed(2).padStart(6)}%`);
|
|
3834
|
+
}
|
|
3835
|
+
output.writeln('');
|
|
3836
|
+
output.writeln(output.dim(' Assumes the next horizon\'s workload mix and rate matches the measurement window.'));
|
|
3837
|
+
output.writeln(output.dim(' Use iter 34 (--window) to check if recent windows are drifting before trusting these.'));
|
|
3838
|
+
output.writeln('');
|
|
3839
|
+
return { success: true, data: payload };
|
|
3840
|
+
},
|
|
3841
|
+
};
|
|
3842
|
+
const routerCommand = {
|
|
3843
|
+
name: 'router',
|
|
3844
|
+
description: 'Cost-optimal neural router lifecycle (ADR-148/149): status, models, prices, config, train, train-from-trajectories, decide, compare-modes, decisions, cost-savings, cost-projection, trajectory-health, ab-stats, bandit-state, stats-summary, reload',
|
|
3845
|
+
subcommands: [routerStatusCommand, routerModelsCommand, routerPricesCommand, routerConfigCommand, routerTrainCommand, routerTrainFromTrajectoriesCommand, routerDecideCommand, routerCompareModesCommand, routerDecisionsCommand, routerCostSavingsCommand, routerCostProjectionCommand, routerTrajectoryHealthCommand, routerAbStatsCommand, routerBanditStateCommand, routerStatsSummaryCommand, routerReloadCommand],
|
|
3846
|
+
examples: [
|
|
3847
|
+
{ command: 'claude-flow neural router status', description: 'Show router state, gate, counters' },
|
|
3848
|
+
{ command: 'claude-flow neural router models', description: 'List candidate registry with measured stats (ADR-149)' },
|
|
3849
|
+
{ command: 'claude-flow neural router prices', description: 'Show the canonical $/Mtok price table (iter 43)' },
|
|
3850
|
+
{ command: 'claude-flow neural router config', description: 'Inventory all CLAUDE_FLOW_ROUTER_* env vars (iter 54)' },
|
|
3851
|
+
{ command: 'claude-flow neural router train -o ./router.krr.json', description: 'Train a KRR artifact' },
|
|
3852
|
+
{ command: 'claude-flow neural router train-from-trajectories -w production-rows.json', description: 'Pair production JSONL into a training corpus (iter 18)' },
|
|
3853
|
+
{ command: 'claude-flow neural router decide "fix typo in cache.ts"', description: 'Inspect decision for a hypothetical task (iter 30)' },
|
|
3854
|
+
{ command: 'claude-flow neural router compare-modes "task" --ceiling 5', description: 'Compare cost-optimal vs cost-ceiling side-by-side (iter 55)' },
|
|
3855
|
+
{ command: 'claude-flow neural router decisions --since 24h', description: 'Query recorded decisions (iter 28)' },
|
|
3856
|
+
{ command: 'claude-flow neural router cost-savings --since 7d', description: 'Actual vs heuristic-counterfactual spend (iter 32)' },
|
|
3857
|
+
{ command: 'claude-flow neural router trajectory-health', description: 'JSONL log health: size, rotations, parse + pair-join rate (iter 36)' },
|
|
3858
|
+
{ command: 'claude-flow neural router ab-stats', description: 'A/B disagreement matrix from sampled ab_pair (iter 37/38)' },
|
|
3859
|
+
{ command: 'claude-flow neural router bandit-state', description: 'Persisted Beta priors per (bucket × model) (iter 48)' },
|
|
3860
|
+
{ command: 'claude-flow neural router stats-summary', description: 'One-screen SRE dashboard (iter 49)' },
|
|
3861
|
+
{ command: 'claude-flow neural router cost-projection', description: 'Project monthly/quarterly spend from measured rate (iter 41)' },
|
|
3862
|
+
{ command: 'claude-flow neural router reload', description: 'Clear in-process backend cache' },
|
|
3863
|
+
],
|
|
3864
|
+
action: async () => {
|
|
3865
|
+
output.writeln('Use a subcommand: status | models | prices | config | train | train-from-trajectories | decide | compare-modes | decisions | cost-savings | cost-projection | trajectory-health | ab-stats | bandit-state | stats-summary | reload');
|
|
3866
|
+
return { success: true };
|
|
3867
|
+
},
|
|
3868
|
+
};
|
|
1519
3869
|
// Main neural command
|
|
1520
3870
|
export const neuralCommand = {
|
|
1521
3871
|
name: 'neural',
|
|
1522
3872
|
description: 'Neural pattern training, MoE, Flash Attention, pattern learning',
|
|
1523
|
-
subcommands: [trainCommand, statusCommand, patternsCommand, predictCommand, optimizeCommand, benchmarkCommand, listCommand, exportCommand, importCommand],
|
|
3873
|
+
subcommands: [trainCommand, statusCommand, patternsCommand, predictCommand, optimizeCommand, benchmarkCommand, listCommand, exportCommand, importCommand, routerCommand],
|
|
1524
3874
|
examples: [
|
|
1525
3875
|
{ command: 'claude-flow neural status', description: 'Check neural system status' },
|
|
1526
3876
|
{ command: 'claude-flow neural train -p coordination', description: 'Train coordination patterns' },
|