agent-nuvira 2.7.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/agents/writer-tool-calling.d.ts +14 -0
- package/dist/agents/agents/writer-tool-calling.d.ts.map +1 -1
- package/dist/agents/agents/writer-tool-calling.js +20 -0
- package/dist/agents/agents/writer-tool-calling.js.map +1 -1
- package/dist/agents/agents/writer.d.ts.map +1 -1
- package/dist/agents/agents/writer.js +15 -4
- package/dist/agents/agents/writer.js.map +1 -1
- package/dist/agents/edit-module.d.ts +7 -0
- package/dist/agents/edit-module.d.ts.map +1 -1
- package/dist/agents/edit-module.js +21 -6
- package/dist/agents/edit-module.js.map +1 -1
- package/dist/agents/orchestrator.d.ts +40 -0
- package/dist/agents/orchestrator.d.ts.map +1 -1
- package/dist/agents/orchestrator.js +116 -27
- package/dist/agents/orchestrator.js.map +1 -1
- package/dist/agents/tool-calling-agent.d.ts +19 -0
- package/dist/agents/tool-calling-agent.d.ts.map +1 -1
- package/dist/agents/tool-calling-agent.js +42 -2
- package/dist/agents/tool-calling-agent.js.map +1 -1
- package/dist/cli/chat.d.ts +55 -1
- package/dist/cli/chat.d.ts.map +1 -1
- package/dist/cli/chat.js +239 -63
- package/dist/cli/chat.js.map +1 -1
- package/dist/cli/cli-program.d.ts +6 -0
- package/dist/cli/cli-program.d.ts.map +1 -0
- package/dist/cli/cli-program.js +222 -0
- package/dist/cli/cli-program.js.map +1 -0
- package/dist/cli/edit.d.ts.map +1 -1
- package/dist/cli/edit.js +18 -3
- package/dist/cli/edit.js.map +1 -1
- package/dist/cli/execute.d.ts.map +1 -1
- package/dist/cli/execute.js +29 -5
- package/dist/cli/execute.js.map +1 -1
- package/dist/cli/failover-runner.d.ts +13 -0
- package/dist/cli/failover-runner.d.ts.map +1 -1
- package/dist/cli/failover-runner.js +11 -0
- package/dist/cli/failover-runner.js.map +1 -1
- package/dist/cli/gateway.d.ts +16 -0
- package/dist/cli/gateway.d.ts.map +1 -1
- package/dist/cli/gateway.js +108 -2
- package/dist/cli/gateway.js.map +1 -1
- package/dist/cli/loop-executor.d.ts.map +1 -1
- package/dist/cli/loop-executor.js +245 -12
- package/dist/cli/loop-executor.js.map +1 -1
- package/dist/cli/plan.d.ts.map +1 -1
- package/dist/cli/plan.js +20 -6
- package/dist/cli/plan.js.map +1 -1
- package/dist/cli/process-control.d.ts +10 -0
- package/dist/cli/process-control.d.ts.map +1 -1
- package/dist/cli/process-control.js +73 -7
- package/dist/cli/process-control.js.map +1 -1
- package/dist/cli/router.d.ts +16 -5
- package/dist/cli/router.d.ts.map +1 -1
- package/dist/cli/router.js +31 -206
- package/dist/cli/router.js.map +1 -1
- package/dist/config/paths.d.ts +23 -0
- package/dist/config/paths.d.ts.map +1 -1
- package/dist/config/paths.js +30 -0
- package/dist/config/paths.js.map +1 -1
- package/dist/config/provider-env.d.ts +37 -0
- package/dist/config/provider-env.d.ts.map +1 -0
- package/dist/config/provider-env.js +86 -0
- package/dist/config/provider-env.js.map +1 -0
- package/dist/context/history.d.ts.map +1 -1
- package/dist/context/history.js +24 -8
- package/dist/context/history.js.map +1 -1
- package/dist/context/session-recall.d.ts +30 -0
- package/dist/context/session-recall.d.ts.map +1 -1
- package/dist/context/session-recall.js +61 -0
- package/dist/context/session-recall.js.map +1 -1
- package/dist/file.js +0 -1
- package/dist/file.js.map +1 -1
- package/dist/gateway/adapters.d.ts +10 -0
- package/dist/gateway/adapters.d.ts.map +1 -1
- package/dist/gateway/adapters.js +4 -1
- package/dist/gateway/adapters.js.map +1 -1
- package/dist/gateway/dedup.d.ts +105 -0
- package/dist/gateway/dedup.d.ts.map +1 -0
- package/dist/gateway/dedup.js +160 -0
- package/dist/gateway/dedup.js.map +1 -0
- package/dist/gateway/heartbeat.d.ts +99 -0
- package/dist/gateway/heartbeat.d.ts.map +1 -0
- package/dist/gateway/heartbeat.js +122 -0
- package/dist/gateway/heartbeat.js.map +1 -0
- package/dist/gateway/inbox.d.ts +14 -1
- package/dist/gateway/inbox.d.ts.map +1 -1
- package/dist/gateway/inbox.js.map +1 -1
- package/dist/gateway/platform-config.d.ts +5 -0
- package/dist/gateway/platform-config.d.ts.map +1 -1
- package/dist/gateway/platform-config.js +8 -13
- package/dist/gateway/platform-config.js.map +1 -1
- package/dist/gateway/registry.d.ts +47 -0
- package/dist/gateway/registry.d.ts.map +1 -1
- package/dist/gateway/registry.js +227 -21
- package/dist/gateway/registry.js.map +1 -1
- package/dist/gateway/whatsapp/baileys-bridge.d.ts +11 -1
- package/dist/gateway/whatsapp/baileys-bridge.d.ts.map +1 -1
- package/dist/gateway/whatsapp/baileys-bridge.js +64 -1
- package/dist/gateway/whatsapp/baileys-bridge.js.map +1 -1
- package/dist/gateway/whatsapp/bridge.d.ts +4 -2
- package/dist/gateway/whatsapp/bridge.d.ts.map +1 -1
- package/dist/gateway/whatsapp/bridge.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/inference/factory.d.ts +16 -0
- package/dist/inference/factory.d.ts.map +1 -1
- package/dist/inference/factory.js +42 -0
- package/dist/inference/factory.js.map +1 -1
- package/dist/inference/gemini-adapter.d.ts.map +1 -1
- package/dist/inference/gemini-adapter.js +3 -0
- package/dist/inference/gemini-adapter.js.map +1 -1
- package/dist/inference/interface.d.ts +12 -0
- package/dist/inference/interface.d.ts.map +1 -1
- package/dist/inference/local-adapter.d.ts +2 -0
- package/dist/inference/local-adapter.d.ts.map +1 -1
- package/dist/inference/local-adapter.js +36 -5
- package/dist/inference/local-adapter.js.map +1 -1
- package/dist/inference/native-tools.d.ts +8 -0
- package/dist/inference/native-tools.d.ts.map +1 -1
- package/dist/inference/native-tools.js +14 -3
- package/dist/inference/native-tools.js.map +1 -1
- package/dist/inference/tool-call-utils.d.ts +66 -7
- package/dist/inference/tool-call-utils.d.ts.map +1 -1
- package/dist/inference/tool-call-utils.js +134 -13
- package/dist/inference/tool-call-utils.js.map +1 -1
- package/dist/learning/auto-router.d.ts.map +1 -1
- package/dist/learning/auto-router.js +90 -5
- package/dist/learning/auto-router.js.map +1 -1
- package/dist/learning/context-budget.d.ts +133 -0
- package/dist/learning/context-budget.d.ts.map +1 -0
- package/dist/learning/context-budget.js +196 -0
- package/dist/learning/context-budget.js.map +1 -0
- package/dist/learning/cost-tracker.d.ts.map +1 -1
- package/dist/learning/cost-tracker.js +20 -8
- package/dist/learning/cost-tracker.js.map +1 -1
- package/dist/learning/eval-framework.d.ts.map +1 -1
- package/dist/learning/eval-framework.js +6 -2
- package/dist/learning/eval-framework.js.map +1 -1
- package/dist/learning/failure-bookkeeping.d.ts +19 -0
- package/dist/learning/failure-bookkeeping.d.ts.map +1 -1
- package/dist/learning/failure-bookkeeping.js +67 -2
- package/dist/learning/failure-bookkeeping.js.map +1 -1
- package/dist/learning/feedback.d.ts.map +1 -1
- package/dist/learning/feedback.js +19 -8
- package/dist/learning/feedback.js.map +1 -1
- package/dist/learning/model-harness.d.ts +76 -0
- package/dist/learning/model-harness.d.ts.map +1 -0
- package/dist/learning/model-harness.js +119 -0
- package/dist/learning/model-harness.js.map +1 -0
- package/dist/learning/model-selection.d.ts +15 -0
- package/dist/learning/model-selection.d.ts.map +1 -1
- package/dist/learning/model-selection.js +27 -1
- package/dist/learning/model-selection.js.map +1 -1
- package/dist/learning/provider-fallback.d.ts +12 -0
- package/dist/learning/provider-fallback.d.ts.map +1 -1
- package/dist/learning/provider-fallback.js +74 -0
- package/dist/learning/provider-fallback.js.map +1 -1
- package/dist/learning/provider-revival.d.ts +106 -0
- package/dist/learning/provider-revival.d.ts.map +1 -0
- package/dist/learning/provider-revival.js +154 -0
- package/dist/learning/provider-revival.js.map +1 -0
- package/dist/learning/resilient-call.d.ts.map +1 -1
- package/dist/learning/resilient-call.js +48 -3
- package/dist/learning/resilient-call.js.map +1 -1
- package/dist/nlu/conversation-gate.d.ts +37 -0
- package/dist/nlu/conversation-gate.d.ts.map +1 -1
- package/dist/nlu/conversation-gate.js +49 -0
- package/dist/nlu/conversation-gate.js.map +1 -1
- package/dist/nlu/schema.d.ts +2 -2
- package/dist/observability/dag-bridge.d.ts +69 -0
- package/dist/observability/dag-bridge.d.ts.map +1 -0
- package/dist/observability/dag-bridge.js +56 -0
- package/dist/observability/dag-bridge.js.map +1 -0
- package/dist/observability/event-bus.d.ts +5 -10
- package/dist/observability/event-bus.d.ts.map +1 -1
- package/dist/observability/event-bus.js +10 -42
- package/dist/observability/event-bus.js.map +1 -1
- package/dist/skills/execution-audit.d.ts.map +1 -1
- package/dist/skills/execution-audit.js +17 -5
- package/dist/skills/execution-audit.js.map +1 -1
- package/dist/skills/sandbox-executor.d.ts +12 -1
- package/dist/skills/sandbox-executor.d.ts.map +1 -1
- package/dist/skills/sandbox-executor.js +28 -6
- package/dist/skills/sandbox-executor.js.map +1 -1
- package/dist/skills/secret-capture.d.ts +45 -1
- package/dist/skills/secret-capture.d.ts.map +1 -1
- package/dist/skills/secret-capture.js +82 -19
- package/dist/skills/secret-capture.js.map +1 -1
- package/dist/skills/skill-env-inventory.d.ts +79 -0
- package/dist/skills/skill-env-inventory.d.ts.map +1 -0
- package/dist/skills/skill-env-inventory.js +167 -0
- package/dist/skills/skill-env-inventory.js.map +1 -0
- package/dist/skills/skill-executor.d.ts +2 -0
- package/dist/skills/skill-executor.d.ts.map +1 -1
- package/dist/skills/skill-executor.js +19 -24
- package/dist/skills/skill-executor.js.map +1 -1
- package/dist/tools/ask-user.d.ts +14 -1
- package/dist/tools/ask-user.d.ts.map +1 -1
- package/dist/tools/ask-user.js +25 -1
- package/dist/tools/ask-user.js.map +1 -1
- package/dist/tools/followup-utils.d.ts +64 -0
- package/dist/tools/followup-utils.d.ts.map +1 -0
- package/dist/tools/followup-utils.js +158 -0
- package/dist/tools/followup-utils.js.map +1 -0
- package/dist/tools/memory-tools.d.ts +35 -0
- package/dist/tools/memory-tools.d.ts.map +1 -1
- package/dist/tools/memory-tools.js +76 -0
- package/dist/tools/memory-tools.js.map +1 -1
- package/dist/tools/pipeline-tool.d.ts.map +1 -1
- package/dist/tools/pipeline-tool.js +8 -3
- package/dist/tools/pipeline-tool.js.map +1 -1
- package/dist/tools/registry.d.ts +25 -12
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +31 -6
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/skill-tool.d.ts.map +1 -1
- package/dist/tools/skill-tool.js +52 -5
- package/dist/tools/skill-tool.js.map +1 -1
- package/dist/tools/tool-loop.d.ts +63 -1
- package/dist/tools/tool-loop.d.ts.map +1 -1
- package/dist/tools/tool-loop.js +305 -66
- package/dist/tools/tool-loop.js.map +1 -1
- package/dist/tools/toolsets.d.ts +19 -3
- package/dist/tools/toolsets.d.ts.map +1 -1
- package/dist/tools/toolsets.js +21 -5
- package/dist/tools/toolsets.js.map +1 -1
- package/dist/utils/env.d.ts.map +1 -1
- package/dist/utils/env.js +9 -13
- package/dist/utils/env.js.map +1 -1
- package/dist/web-dashboard/chat-console.d.ts +13 -1
- package/dist/web-dashboard/chat-console.d.ts.map +1 -1
- package/dist/web-dashboard/chat-console.js +16 -2
- package/dist/web-dashboard/chat-console.js.map +1 -1
- package/dist/web-dashboard/hub-data.d.ts +2 -0
- package/dist/web-dashboard/hub-data.d.ts.map +1 -1
- package/dist/web-dashboard/hub-data.js +4 -0
- package/dist/web-dashboard/hub-data.js.map +1 -1
- package/dist/web-dashboard/server.d.ts +6 -0
- package/dist/web-dashboard/server.d.ts.map +1 -1
- package/dist/web-dashboard/server.js +192 -3
- package/dist/web-dashboard/server.js.map +1 -1
- package/dist/web-dashboard/src/types.d.ts +31 -1
- package/dist/web-dashboard/src/types.d.ts.map +1 -1
- package/package.json +11 -1
- package/src/web-dashboard/public/assets/index-56yPnN_m.js +207 -0
- package/src/web-dashboard/public/assets/index-56yPnN_m.js.map +1 -0
- package/src/web-dashboard/public/assets/{index-C507EUWf.css → index-Cyd6tIew.css} +1 -1
- package/src/web-dashboard/public/index.html +2 -2
- package/dist/example.d.ts +0 -1
- package/dist/example.d.ts.map +0 -1
- package/dist/example.js +0 -3
- package/dist/example.js.map +0 -1
- package/dist/fresh.d.ts +0 -2
- package/dist/fresh.d.ts.map +0 -1
- package/dist/fresh.js +0 -2
- package/dist/fresh.js.map +0 -1
- package/dist/mcp/mcp-dashboard-oauth.d.ts +0 -94
- package/dist/mcp/mcp-dashboard-oauth.d.ts.map +0 -1
- package/dist/mcp/mcp-dashboard-oauth.js +0 -140
- package/dist/mcp/mcp-dashboard-oauth.js.map +0 -1
- package/dist/memory/background-sync.d.ts +0 -107
- package/dist/memory/background-sync.d.ts.map +0 -1
- package/dist/memory/background-sync.js +0 -242
- package/dist/memory/background-sync.js.map +0 -1
- package/dist/memory/cross-session.d.ts +0 -68
- package/dist/memory/cross-session.d.ts.map +0 -1
- package/dist/memory/cross-session.js +0 -153
- package/dist/memory/cross-session.js.map +0 -1
- package/dist/memory/drift-detector.d.ts +0 -76
- package/dist/memory/drift-detector.d.ts.map +0 -1
- package/dist/memory/drift-detector.js +0 -216
- package/dist/memory/drift-detector.js.map +0 -1
- package/dist/memory/enhanced-manager.d.ts +0 -105
- package/dist/memory/enhanced-manager.d.ts.map +0 -1
- package/dist/memory/enhanced-manager.js +0 -280
- package/dist/memory/enhanced-manager.js.map +0 -1
- package/dist/memory/session-extraction.d.ts +0 -93
- package/dist/memory/session-extraction.d.ts.map +0 -1
- package/dist/memory/session-extraction.js +0 -290
- package/dist/memory/session-extraction.js.map +0 -1
- package/dist/memory/sqlite-store.d.ts +0 -208
- package/dist/memory/sqlite-store.d.ts.map +0 -1
- package/dist/memory/sqlite-store.js +0 -556
- package/dist/memory/sqlite-store.js.map +0 -1
- package/dist/skills/daytona-executor.d.ts +0 -78
- package/dist/skills/daytona-executor.d.ts.map +0 -1
- package/dist/skills/daytona-executor.js +0 -256
- package/dist/skills/daytona-executor.js.map +0 -1
- package/dist/skills/modal-executor.d.ts +0 -81
- package/dist/skills/modal-executor.d.ts.map +0 -1
- package/dist/skills/modal-executor.js +0 -285
- package/dist/skills/modal-executor.js.map +0 -1
- package/dist/sync/inventory.d.ts +0 -1
- package/dist/sync/inventory.d.ts.map +0 -1
- package/dist/sync/inventory.js +0 -3
- package/dist/sync/inventory.js.map +0 -1
- package/dist/sync/validate.d.ts +0 -1
- package/dist/sync/validate.d.ts.map +0 -1
- package/dist/sync/validate.js +0 -3
- package/dist/sync/validate.js.map +0 -1
- package/dist/test.d.ts +0 -2
- package/dist/test.d.ts.map +0 -1
- package/dist/test.js +0 -3
- package/dist/test.js.map +0 -1
- package/dist/tools/browser-tool.d.ts +0 -206
- package/dist/tools/browser-tool.d.ts.map +0 -1
- package/dist/tools/browser-tool.js +0 -726
- package/dist/tools/browser-tool.js.map +0 -1
- package/dist/tools/delegate-tool.d.ts +0 -109
- package/dist/tools/delegate-tool.d.ts.map +0 -1
- package/dist/tools/delegate-tool.js +0 -185
- package/dist/tools/delegate-tool.js.map +0 -1
- package/dist/tools/delegation-state.d.ts +0 -87
- package/dist/tools/delegation-state.d.ts.map +0 -1
- package/dist/tools/delegation-state.js +0 -278
- package/dist/tools/delegation-state.js.map +0 -1
- package/dist/tools/desktop-ui.d.ts +0 -148
- package/dist/tools/desktop-ui.d.ts.map +0 -1
- package/dist/tools/desktop-ui.js +0 -486
- package/dist/tools/desktop-ui.js.map +0 -1
- package/dist/tools/image-video-tool.d.ts +0 -88
- package/dist/tools/image-video-tool.d.ts.map +0 -1
- package/dist/tools/image-video-tool.js +0 -276
- package/dist/tools/image-video-tool.js.map +0 -1
- package/dist/tools/kanban-cron-tools.d.ts +0 -92
- package/dist/tools/kanban-cron-tools.d.ts.map +0 -1
- package/dist/tools/kanban-cron-tools.js +0 -197
- package/dist/tools/kanban-cron-tools.js.map +0 -1
- package/dist/tools/messaging-tool.d.ts +0 -127
- package/dist/tools/messaging-tool.d.ts.map +0 -1
- package/dist/tools/messaging-tool.js +0 -314
- package/dist/tools/messaging-tool.js.map +0 -1
- package/dist/tools/modality/generic-caller.d.ts +0 -43
- package/dist/tools/modality/generic-caller.d.ts.map +0 -1
- package/dist/tools/modality/generic-caller.js +0 -254
- package/dist/tools/modality/generic-caller.js.map +0 -1
- package/dist/tools/modality/modality-catalog.d.ts +0 -134
- package/dist/tools/modality/modality-catalog.d.ts.map +0 -1
- package/dist/tools/modality/modality-catalog.js +0 -445
- package/dist/tools/modality/modality-catalog.js.map +0 -1
- package/dist/tools/modality/tool-router.d.ts +0 -81
- package/dist/tools/modality/tool-router.d.ts.map +0 -1
- package/dist/tools/modality/tool-router.js +0 -84
- package/dist/tools/modality/tool-router.js.map +0 -1
- package/dist/tools/security-tools.d.ts +0 -107
- package/dist/tools/security-tools.d.ts.map +0 -1
- package/dist/tools/security-tools.js +0 -264
- package/dist/tools/security-tools.js.map +0 -1
- package/dist/tools/ssh-tool.d.ts +0 -130
- package/dist/tools/ssh-tool.d.ts.map +0 -1
- package/dist/tools/ssh-tool.js +0 -328
- package/dist/tools/ssh-tool.js.map +0 -1
- package/dist/tools/tts-tool.d.ts +0 -91
- package/dist/tools/tts-tool.d.ts.map +0 -1
- package/dist/tools/tts-tool.js +0 -406
- package/dist/tools/tts-tool.js.map +0 -1
- package/dist/utils/shell-detection.d.ts +0 -124
- package/dist/utils/shell-detection.d.ts.map +0 -1
- package/dist/utils/shell-detection.js +0 -305
- package/dist/utils/shell-detection.js.map +0 -1
- package/dist/utils/windows-paths.d.ts +0 -185
- package/dist/utils/windows-paths.d.ts.map +0 -1
- package/dist/utils/windows-paths.js +0 -472
- package/dist/utils/windows-paths.js.map +0 -1
- package/src/web-dashboard/public/assets/index-C4frng1Q.js +0 -207
- package/src/web-dashboard/public/assets/index-C4frng1Q.js.map +0 -1
|
@@ -80,6 +80,80 @@ export function classifyFallbackError(err) {
|
|
|
80
80
|
export function isRetryableError(errorType) {
|
|
81
81
|
return errorType !== 'auth';
|
|
82
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* Deterministic REQUEST-SHAPE rejections — our request was malformed, or used a
|
|
85
|
+
* feature the endpoint does not accept. The same request fails identically for
|
|
86
|
+
* every model on that provider, forever, so the fault is in the HARNESS (our
|
|
87
|
+
* prompt/adapter/tool-call encoding), never in the model's ability.
|
|
88
|
+
*
|
|
89
|
+
* Verified live: Gemini answers a multi-turn tool call with
|
|
90
|
+
* `400 INVALID_ARGUMENT — Function call is missing a thought_signature in
|
|
91
|
+
* functionCall parts`. That killed a step-2 tool call in a real run, and the
|
|
92
|
+
* generic failure path then booked it as a Gemini provider failure — parking a
|
|
93
|
+
* healthy provider, decaying health, and teaching the bandit that the model was
|
|
94
|
+
* weak when the bug would have broken ANY model on that transport.
|
|
95
|
+
*
|
|
96
|
+
* Deliberately NARROW. Over-matching here is worse than under-matching: a false
|
|
97
|
+
* positive silently stops a genuinely broken provider from ever being
|
|
98
|
+
* deprioritized. Every pattern must name a request-encoding defect, not a
|
|
99
|
+
* generic 4xx (`400 model not found` is a MODEL problem, handled elsewhere).
|
|
100
|
+
*/
|
|
101
|
+
const HARNESS_FAULT_PATTERNS = [
|
|
102
|
+
// Gemini's per-part signature requirement (the confirmed live case).
|
|
103
|
+
/thought_signature|thoughtSignature/i,
|
|
104
|
+
// The same defect stated generically by other OpenAI-compatible endpoints.
|
|
105
|
+
/function call is missing/i,
|
|
106
|
+
// Explicit request-shape rejection codes.
|
|
107
|
+
/invalid_request_error/i,
|
|
108
|
+
/\binvalid_argument\b/i,
|
|
109
|
+
// The endpoint refuses the tool-calling surface itself.
|
|
110
|
+
/unsupported\s+(?:tool|function|content|part)\b/i,
|
|
111
|
+
// Our tool ARGUMENTS did not satisfy the declared schema.
|
|
112
|
+
/failed to parse (?:the )?(?:tool|arguments|function)/i,
|
|
113
|
+
/(?:tool|function) (?:arguments|call) (?:is|are) not valid json/i,
|
|
114
|
+
];
|
|
115
|
+
/**
|
|
116
|
+
* Error types a SHORT same-provider retry can plausibly fix.
|
|
117
|
+
*
|
|
118
|
+
* `isRetryableError` above answers a different question — "is it worth trying a
|
|
119
|
+
* DIFFERENT provider?" — and the failover walk assumes a replacement exists. With
|
|
120
|
+
* one provider configured (the common single-key setup) there is nothing to fail
|
|
121
|
+
* over TO, so a transient spike became a dead run.
|
|
122
|
+
*
|
|
123
|
+
* Verified live: Gemini answered `503 ... currently experiencing high demand`
|
|
124
|
+
* during a real 4-step task. There was no same-provider retry, so the 503 became
|
|
125
|
+
* a hard failure, the circuit breaker parked the provider for 120s, the
|
|
126
|
+
* context-gatherer degraded to "0 relevant files", and the writer then edited
|
|
127
|
+
* BLIND. One capacity spike at a shared endpoint turned into a wrong answer.
|
|
128
|
+
*
|
|
129
|
+
* Server/network/timeout are the only classes a retry can fix: auth needs a new
|
|
130
|
+
* key, rate-limit needs the reset window, and unknown/model faults are not
|
|
131
|
+
* load-related.
|
|
132
|
+
*/
|
|
133
|
+
const TRANSIENT_RETRY_TYPES = new Set([
|
|
134
|
+
'server',
|
|
135
|
+
'network',
|
|
136
|
+
'timeout',
|
|
137
|
+
]);
|
|
138
|
+
/** Should this error be retried against the SAME provider after a short backoff? */
|
|
139
|
+
export function isTransientForRetry(err) {
|
|
140
|
+
return TRANSIENT_RETRY_TYPES.has(classifyFallbackError(err));
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Is this failure a harness fault (our request shape) rather than a provider or
|
|
144
|
+
* model failure?
|
|
145
|
+
*
|
|
146
|
+
* Callers use this to decide ATTRIBUTION, never retryability: a harness fault is
|
|
147
|
+
* provider-specific, so failing over to a different provider is still the right
|
|
148
|
+
* move — but the provider's health, the model's health score and the bandit
|
|
149
|
+
* prior must all stay untouched.
|
|
150
|
+
*/
|
|
151
|
+
export function isHarnessFault(err) {
|
|
152
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
153
|
+
if (!message)
|
|
154
|
+
return false;
|
|
155
|
+
return HARNESS_FAULT_PATTERNS.some((re) => re.test(message));
|
|
156
|
+
}
|
|
83
157
|
/**
|
|
84
158
|
* Floor for hint-derived quota parks — never re-admit faster than this, so a
|
|
85
159
|
* 429 with a 1s reset hint can't hot-loop the router back into the same
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-fallback.js","sourceRoot":"","sources":["../../src/learning/provider-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAI1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AA2CtF,+EAA+E;AAE/E,+EAA+E;AAC/E,+EAA+E;AAC/E,mEAAmE;AACnE,MAAM,uBAAuB,GAA2B;IACtD,OAAO,EAAE,IAAI;IACb,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;AAEF,kEAAkE;AAClE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AACvC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAErC,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3K,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IACE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;QACrB,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACnC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAChC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;QAC7B,KAAK,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QAC7C,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACpC,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACnJ,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/N,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAA4B;IAC3D,OAAO,SAAS,KAAK,MAAM,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,qEAAqE;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC7D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,qFAAqF,CAAC,CAAC;IAClH,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,iDAAiD;IACjD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChE,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACvE,CAAC;IACD,wBAAwB;IACxB,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC1D,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,oEAAoE;IACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC;IAClG,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC;IACjE,CAAC;IACD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC1E,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,CAAC;IACD,uCAAuC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAC9E,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,MAAM,MAAM,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,UAAU,CAAC;QAC9C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QAC5D,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAC1C,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAA6D,CAAC;QAChF,IAAI,OAAO,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YACjD,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACtC,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxC,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvC,IAAI,CAAC,GAAG,CAAC;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC;gBAC7B,CAAC;gBACD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACpD,CAAC;YACD,uEAAuE;YACvE,yEAAyE;YACzE,0EAA0E;YAC1E,KAAK,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAChG,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBACxB,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;wBAAE,SAAS;oBACjC,IAAI,CAAC,GAAG,IAAI;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,sBAAsB;oBACrD,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,sBAAsB,CAAC,aAAsB;IAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7C,OAAO,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,YAAoB,EACpB,KAAyB,EACzB,GAAY,EACZ,SAA6B,EAC7B,MAAe,EACf,SAAkB;IAElB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,SAAS,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrC,oEAAoE;QACpE,qEAAqE;QACrE,yEAAyE;QACzE,4DAA4D;QAC5D,MAAM,aAAa,GACjB,yCAAyC,CAAC,IAAI,CAAC,KAAK,CAAC;YACrD,oEAAoE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnF,IAAI,aAAa,EAAE,CAAC;YAClB,QAAQ,CAAC,eAAe,CAAC,YAAY,EAAE,KAAK,IAAI,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,oEAAoE;YACpE,oEAAoE;YACpE,6BAA6B;YAC7B,QAAQ,CAAC,UAAU,CACjB,YAAY,EACZ,KAAK,IAAI,SAAS,EAClB,KAAK,EACL,IAAI,EACJ,GAAG,EACH,SAAS,EACT,SAAS,EACT,SAAS,EACT,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAC5E,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CACnC,YAAoB,EACpB,KAAyB,EACzB,MAAe,EACf,SAAkB;IAElB,IAAI,CAAC;QACH,gBAAgB,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,KAAK,IAAI,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;QAC5H,yEAAyE;QACzE,2EAA2E;QAC3E,4CAA4C;QAC5C,aAAa,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,4DAA4D;IAC9D,CAAC;AACH,CAAC;AAED,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAgB;IACnB,aAAa,CAAgB;IAC7B,cAAc,CAAyB;IACvC,eAAe,GAAqC,IAAI,GAAG,EAAE,CAAC;IAC9D,aAAa,GAAoE,IAAI,GAAG,EAAE,CAAC;IAC3F,mBAAmB,GAAoB,IAAI,CAAC;IAEpD,YACE,aAA4B,EAC5B,SAA2C;QAE3C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,SAAS,EAAE,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,eAAwB;QACvC,0EAA0E;QAC1E,+DAA+D;QAC/D,MAAM,UAAU,GACd,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS;YAC/B,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,gCAAgC;QAChC,IAAI,eAAe,EAAE,CAAC;YACpB,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACpC,CAAC;QAED,+DAA+D;QAC/D,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,2BAA2B;QAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9C,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,sEAAsE;QACtE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,IAAI,eAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,0EAA0E;QAC1E,4EAA4E;QAC5E,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,sEAAsE;QACtE,0EAA0E;QAC1E,qCAAqC;QACrC,EAAE;QACF,kEAAkE;QAClE,wEAAwE;QACxE,2EAA2E;QAC3E,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,IAAI,CAAC;YACH,eAAe,GAAG,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACxB,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACvC,IAAI,EAAE,IAAI,GAAG,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;gBACnG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,CAAC,KAAK,eAAe,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mDAAmD,CAAC,CAAC;gBAC5E,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,YAAoB;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC;QAE9B,cAAc;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QAED,IAAI,CAAC;YACH,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,YAA4B,CAAC,CAAC;YACpF,MAAM,QAAQ,GAAG,eAAe,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,IAAI,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC/E,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,mCAAmC,YAAY,KAAK,GAAG,EAAE,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,YAAoB;QACjD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,YAA4B,CAAC,CAAC;YACtF,OAAO,MAAM,EAAE,KAAK,IAAI,SAAS,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CACpB,eAAmC,EACnC,MAA8E,EAC9E,OAA8C;QAE9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;QACpD,MAAM,YAAY,GAAkE,EAAE,CAAC;QAEvF,+BAA+B;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAE9C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,2EAA2E;YAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,eAAe,IAAI,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1F,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,aAAa,OAAO,iDAAiD,CAAC,CAAC;YACzF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACjD,OAAO;oBACL,QAAQ;oBACR,QAAQ,EAAE,OAAO;oBACjB,QAAQ,EAAE,CAAC;oBACX,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;oBAChE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACtC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,CAAC,CAAC,iCAAiC;YAC9C,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7D,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,oCAAoC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9F,SAAS;YACX,CAAC;YAED,gEAAgE;YAChE,4DAA4D;YAC5D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBAEjC,oDAAoD;gBACpD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAEhC,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,EAAE,KAAK,IAAI,MAAM,iBAAiB,EAAE,KAAK,QAAQ,KAAK,CAAC,CAAC;gBAErF,mEAAmE;gBACnE,sEAAsE;gBACtE,sEAAsE;gBACtE,iEAAiE;gBACjE,yBAAyB;gBACzB,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAEvF,OAAO;oBACL,QAAQ;oBACR,QAAQ,EAAE,EAAE;oBACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;oBACf,YAAY,EAAE,CAAC,GAAG,YAAY,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;oBAC3D,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACtC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAClE,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE7C,mCAAmC;gBACnC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAEvB,sEAAsE;gBACtE,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,mEAAmE;gBACnE,uEAAuE;gBACvE,6DAA6D;gBAC7D,uEAAuE;gBACvE,qEAAqE;gBACrE,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAEvG,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAE7E,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;oBACjC,2DAA2D;oBAC3D,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,SAAS,4BAA4B,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1F,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,qCAAqC;gBACrC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;oBACvD,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,GAAG,CAAC,cAAc;wBACpD,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1E,CAAC;gBACJ,CAAC;gBAED,4DAA4D;gBAC5D,MAAM,KAAK,GAAG,SAAS,KAAK,YAAY;oBACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,GAAG,CAAC;oBACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC;gBAErC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,KAAK,IAAI,CAAC,CAAC;gBACvG,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,0CAA0C;QAC1C,MAAM,IAAI,KAAK,CACb,qCAAqC,YAAY,CAAC,MAAM,aAAa;YACrE,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChG,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,YAAoB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEnD,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,WAAW,GAAG,kBAAkB,EAAE,CAAC;YAC3D,+CAA+C;YAC/C,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;QAC1B,CAAC;QAED,0CAA0C;QAC1C,IAAI,KAAK,CAAC,QAAQ,IAAI,4BAA4B,EAAE,CAAC;YACnD,KAAK,CAAC,aAAa,GAAG,GAAG,GAAG,oBAAoB,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,mCAAmC,YAAY,uBAAuB,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC;YAClH,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,8BAA8B;QACpD,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,YAAqB;QACvC,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,uBAAuB;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAA6E,EAAE,CAAC;QAE5F,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAA0C;QACrD,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,SAAS,EAAE,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,sBAAsB;QAC5B,IAAI,IAAI,CAAC,mBAAmB,KAAK,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAClC,CAAC;CACF;AAED,+EAA+E;AAE/E,IAAI,gBAAgB,GAA4B,IAAI,CAAC;AAErD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,aAA6B,EAC7B,SAA2C;IAE3C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;QACtG,CAAC;QACD,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACpE,CAAC;SAAM,IAAI,SAAS,EAAE,CAAC;QACrB,kEAAkE;QAClE,gBAAgB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,gBAAgB,GAAG,IAAI,CAAC;AAC1B,CAAC"}
|
|
1
|
+
{"version":3,"file":"provider-fallback.js","sourceRoot":"","sources":["../../src/learning/provider-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAI1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AA2CtF,+EAA+E;AAE/E,+EAA+E;AAC/E,+EAA+E;AAC/E,mEAAmE;AACnE,MAAM,uBAAuB,GAA2B;IACtD,OAAO,EAAE,IAAI;IACb,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,IAAI;CACnB,CAAC;AAEF,kEAAkE;AAClE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AACvC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAErC,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3K,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IACE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;QACrB,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACnC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAChC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC5B,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;QAC7B,KAAK,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QAC7C,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACpC,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACnJ,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/N,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAA4B;IAC3D,OAAO,SAAS,KAAK,MAAM,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,sBAAsB,GAAa;IACvC,qEAAqE;IACrE,qCAAqC;IACrC,2EAA2E;IAC3E,2BAA2B;IAC3B,0CAA0C;IAC1C,wBAAwB;IACxB,uBAAuB;IACvB,wDAAwD;IACxD,iDAAiD;IACjD,0DAA0D;IAC1D,uDAAuD;IACvD,iEAAiE;CAClE,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,qBAAqB,GAAmC,IAAI,GAAG,CAAoB;IACvF,QAAQ;IACR,SAAS;IACT,SAAS;CACV,CAAC,CAAC;AAEH,oFAAoF;AACpF,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,OAAO,qBAAqB,CAAC,GAAG,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,qEAAqE;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC7D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,qFAAqF,CAAC,CAAC;IAClH,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,iDAAiD;IACjD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChE,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACvE,CAAC;IACD,wBAAwB;IACxB,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC1D,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,oEAAoE;IACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC;IAClG,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC;IACjE,CAAC;IACD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC1E,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,CAAC;IACD,uCAAuC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAC9E,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,MAAM,MAAM,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,UAAU,CAAC;QAC9C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QAC5D,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAC1C,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAA6D,CAAC;QAChF,IAAI,OAAO,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;YACjD,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACtC,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxC,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACvC,IAAI,CAAC,GAAG,CAAC;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC;gBAC7B,CAAC;gBACD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACpD,CAAC;YACD,uEAAuE;YACvE,yEAAyE;YACzE,0EAA0E;YAC1E,KAAK,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAChG,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBACxB,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;wBAAE,SAAS;oBACjC,IAAI,CAAC,GAAG,IAAI;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,sBAAsB;oBACrD,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;wBAAE,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,sBAAsB,CAAC,aAAsB;IAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7C,OAAO,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,YAAoB,EACpB,KAAyB,EACzB,GAAY,EACZ,SAA6B,EAC7B,MAAe,EACf,SAAkB;IAElB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,SAAS,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrC,oEAAoE;QACpE,qEAAqE;QACrE,yEAAyE;QACzE,4DAA4D;QAC5D,MAAM,aAAa,GACjB,yCAAyC,CAAC,IAAI,CAAC,KAAK,CAAC;YACrD,oEAAoE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnF,IAAI,aAAa,EAAE,CAAC;YAClB,QAAQ,CAAC,eAAe,CAAC,YAAY,EAAE,KAAK,IAAI,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,oEAAoE;YACpE,oEAAoE;YACpE,6BAA6B;YAC7B,QAAQ,CAAC,UAAU,CACjB,YAAY,EACZ,KAAK,IAAI,SAAS,EAClB,KAAK,EACL,IAAI,EACJ,GAAG,EACH,SAAS,EACT,SAAS,EACT,SAAS,EACT,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAC5E,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CACnC,YAAoB,EACpB,KAAyB,EACzB,MAAe,EACf,SAAkB;IAElB,IAAI,CAAC;QACH,gBAAgB,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,KAAK,IAAI,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;QAC5H,yEAAyE;QACzE,2EAA2E;QAC3E,4CAA4C;QAC5C,aAAa,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,4DAA4D;IAC9D,CAAC;AACH,CAAC;AAED,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAgB;IACnB,aAAa,CAAgB;IAC7B,cAAc,CAAyB;IACvC,eAAe,GAAqC,IAAI,GAAG,EAAE,CAAC;IAC9D,aAAa,GAAoE,IAAI,GAAG,EAAE,CAAC;IAC3F,mBAAmB,GAAoB,IAAI,CAAC;IAEpD,YACE,aAA4B,EAC5B,SAA2C;QAE3C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,SAAS,EAAE,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,eAAwB;QACvC,0EAA0E;QAC1E,+DAA+D;QAC/D,MAAM,UAAU,GACd,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS;YAC/B,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,gCAAgC;QAChC,IAAI,eAAe,EAAE,CAAC;YACpB,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACpC,CAAC;QAED,+DAA+D;QAC/D,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,2BAA2B;QAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9C,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,sEAAsE;QACtE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAEhC,IAAI,eAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,0EAA0E;QAC1E,4EAA4E;QAC5E,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,sEAAsE;QACtE,0EAA0E;QAC1E,qCAAqC;QACrC,EAAE;QACF,kEAAkE;QAClE,wEAAwE;QACxE,2EAA2E;QAC3E,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,IAAI,CAAC;YACH,eAAe,GAAG,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACxB,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACvC,IAAI,EAAE,IAAI,GAAG,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;gBACnG,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,CAAC,KAAK,eAAe,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mDAAmD,CAAC,CAAC;gBAC5E,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,YAAoB;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC;QAE9B,cAAc;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QAED,IAAI,CAAC;YACH,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,YAA4B,CAAC,CAAC;YACpF,MAAM,QAAQ,GAAG,eAAe,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,IAAI,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC/E,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,mCAAmC,YAAY,KAAK,GAAG,EAAE,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,YAAoB;QACjD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,YAA4B,CAAC,CAAC;YACtF,OAAO,MAAM,EAAE,KAAK,IAAI,SAAS,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CACpB,eAAmC,EACnC,MAA8E,EAC9E,OAA8C;QAE9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;QACpD,MAAM,YAAY,GAAkE,EAAE,CAAC;QAEvF,+BAA+B;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAE9C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,2EAA2E;YAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,eAAe,IAAI,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC1F,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,aAAa,OAAO,iDAAiD,CAAC,CAAC;YACzF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACjD,OAAO;oBACL,QAAQ;oBACR,QAAQ,EAAE,OAAO;oBACjB,QAAQ,EAAE,CAAC;oBACX,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;oBAChE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACtC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,CAAC,CAAC,iCAAiC;YAC9C,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7D,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,oCAAoC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9F,SAAS;YACX,CAAC;YAED,gEAAgE;YAChE,4DAA4D;YAC5D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBAEjC,oDAAoD;gBACpD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAEhC,MAAM,CAAC,OAAO,CAAC,OAAO,OAAO,EAAE,KAAK,IAAI,MAAM,iBAAiB,EAAE,KAAK,QAAQ,KAAK,CAAC,CAAC;gBAErF,mEAAmE;gBACnE,sEAAsE;gBACtE,sEAAsE;gBACtE,iEAAiE;gBACjE,yBAAyB;gBACzB,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAEvF,OAAO;oBACL,QAAQ;oBACR,QAAQ,EAAE,EAAE;oBACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;oBACf,YAAY,EAAE,CAAC,GAAG,YAAY,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;oBAC3D,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACtC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAClE,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE7C,mCAAmC;gBACnC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAEvB,sEAAsE;gBACtE,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,mEAAmE;gBACnE,uEAAuE;gBACvE,6DAA6D;gBAC7D,uEAAuE;gBACvE,qEAAqE;gBACrE,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAEvG,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAE7E,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;oBACjC,2DAA2D;oBAC3D,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,SAAS,4BAA4B,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1F,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,qCAAqC;gBACrC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;oBACvD,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,GAAG,CAAC,cAAc;wBACpD,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1E,CAAC;gBACJ,CAAC;gBAED,4DAA4D;gBAC5D,MAAM,KAAK,GAAG,SAAS,KAAK,YAAY;oBACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,GAAG,CAAC;oBACtC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC;gBAErC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,SAAS,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,KAAK,IAAI,CAAC,CAAC;gBACvG,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,0CAA0C;QAC1C,MAAM,IAAI,KAAK,CACb,qCAAqC,YAAY,CAAC,MAAM,aAAa;YACrE,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChG,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,YAAoB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEnD,IAAI,CAAC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,WAAW,GAAG,kBAAkB,EAAE,CAAC;YAC3D,+CAA+C;YAC/C,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;QAC1B,CAAC;QAED,0CAA0C;QAC1C,IAAI,KAAK,CAAC,QAAQ,IAAI,4BAA4B,EAAE,CAAC;YACnD,KAAK,CAAC,aAAa,GAAG,GAAG,GAAG,oBAAoB,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,mCAAmC,YAAY,uBAAuB,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC;YAClH,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,8BAA8B;QACpD,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,YAAqB;QACvC,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,uBAAuB;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,MAAM,GAA6E,EAAE,CAAC;QAE5F,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAA0C;QACrD,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,SAAS,EAAE,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,sBAAsB;QAC5B,IAAI,IAAI,CAAC,mBAAmB,KAAK,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAClC,CAAC;CACF;AAED,+EAA+E;AAE/E,IAAI,gBAAgB,GAA4B,IAAI,CAAC;AAErD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,aAA6B,EAC7B,SAA2C;IAE3C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;QACtG,CAAC;QACD,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACpE,CAAC;SAAM,IAAI,SAAS,EAAE,CAAC;QACrB,kEAAkE;QAClE,gBAAgB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,gBAAgB,GAAG,IAAI,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider revival — the "one more round of check" before giving up on a
|
|
3
|
+
* provider that failed transiently.
|
|
4
|
+
*
|
|
5
|
+
* THE BUG THIS FIXES (verified): a provider that fails transiently (server /
|
|
6
|
+
* network / timeout) is put in a short session exclusion AND, separately, marked
|
|
7
|
+
* in a `transientFailed` set that carries the documented promise — "awaiting a
|
|
8
|
+
* quick on-demand spot-check before re-admission (never re-pick without proof)".
|
|
9
|
+
* That promise was kept in exactly ONE place: `chat.ts`'s session loop. Every
|
|
10
|
+
* other entry path allocated the marker and never read it:
|
|
11
|
+
*
|
|
12
|
+
* path allocates reads (revives)
|
|
13
|
+
* chat.ts ✅ ✅
|
|
14
|
+
* orchestrator.ts ✅ ✅ (covers execute.ts's pipeline path)
|
|
15
|
+
* resilient-call.ts ✅ ✅
|
|
16
|
+
* edit.ts ✅ ✅ (via the runSingleShotAuto `revive` hook)
|
|
17
|
+
* execute.ts ✅ ✅ (the follow-up path that bypasses the
|
|
18
|
+
* orchestrator sweeps its own session)
|
|
19
|
+
* plan.ts ✅ ✅ (via the runSingleShotAuto `revive` hook)
|
|
20
|
+
*
|
|
21
|
+
* Every allocating path now reads what it arms. When this was not true, an
|
|
22
|
+
* exhausted provider stayed
|
|
23
|
+
* excluded for the rest of the session even after it recovered, and the run
|
|
24
|
+
* either degraded or died — while the machinery to prove recovery in seconds sat
|
|
25
|
+
* unused. Live evidence: a pipeline run hit a Gemini 503, the breaker parked the
|
|
26
|
+
* provider, context-gatherer fell back to "0 relevant files", and the writer then
|
|
27
|
+
* edited blind. A single capacity spike at one endpoint became a wrong answer.
|
|
28
|
+
*
|
|
29
|
+
* WHY A STORE ADAPTER: the callers keep their session state in two different
|
|
30
|
+
* shapes — `FailureSessionState` (chat/orchestrator/edit/plan/execute) and
|
|
31
|
+
* resilient-call's own `{ expiresAt, kind }` maps. Rather than duplicate the
|
|
32
|
+
* probe logic per shape (the duplication that caused this bug), the probe works
|
|
33
|
+
* against this small port and each caller adapts.
|
|
34
|
+
*
|
|
35
|
+
* Cycle safety: `auto-router.js` is imported LAZILY (only when a caller does not
|
|
36
|
+
* supply its own model resolver), so this module can be imported from the
|
|
37
|
+
* router's own dependencies without creating a static cycle.
|
|
38
|
+
*/
|
|
39
|
+
import { spotCheckModel } from '../inference/model-probe.js';
|
|
40
|
+
import { type FailureSessionState } from './failure-bookkeeping.js';
|
|
41
|
+
import type { ConfigManager } from '../config/manager.js';
|
|
42
|
+
/** The outcome shape `spotCheckModel` returns, without importing its name. */
|
|
43
|
+
type ProbeOutcome = Awaited<ReturnType<typeof spotCheckModel>>;
|
|
44
|
+
/**
|
|
45
|
+
* The port a caller adapts its failure state to. Deliberately tiny: the sweep
|
|
46
|
+
* needs to enumerate candidates, ask whether an exclusion is still active, and
|
|
47
|
+
* either clear or re-arm one.
|
|
48
|
+
*/
|
|
49
|
+
export interface RevivalStore {
|
|
50
|
+
/** Providers currently marked as transiently failed. */
|
|
51
|
+
transientProviders(): Iterable<string>;
|
|
52
|
+
/** Is this provider's exclusion still in force at `now`? */
|
|
53
|
+
isExclusionActive(provider: string, now: number): boolean;
|
|
54
|
+
/** Re-admit the provider (clear its exclusion + its transient marker). */
|
|
55
|
+
clearProvider(provider: string): void;
|
|
56
|
+
/** Keep it excluded for another transient window. */
|
|
57
|
+
reArmProvider(provider: string, until: number): void;
|
|
58
|
+
/** The concrete model to probe for this provider. */
|
|
59
|
+
resolveProbeModel?(provider: string): string | undefined;
|
|
60
|
+
}
|
|
61
|
+
export interface RevivalSweepResult {
|
|
62
|
+
/** Providers actually probed (network calls made). */
|
|
63
|
+
probed: string[];
|
|
64
|
+
/** Providers now re-admitted — either proven alive or no longer blocked. */
|
|
65
|
+
revived: string[];
|
|
66
|
+
/** Providers that failed the probe and keep their exclusion. */
|
|
67
|
+
stillDown: string[];
|
|
68
|
+
}
|
|
69
|
+
export interface RevivalSweepOptions {
|
|
70
|
+
/**
|
|
71
|
+
* Max providers probed per sweep. Each probe is a 1-token generation, so this
|
|
72
|
+
* bounds the latency a sweep can add to a foreground turn. Default 3.
|
|
73
|
+
*/
|
|
74
|
+
maxProbes?: number;
|
|
75
|
+
/** Agent type used when resolving a probe model (default 'chat'). */
|
|
76
|
+
agentType?: string;
|
|
77
|
+
/** Injectable probe (tests). Defaults to the real 1-token spot-check. */
|
|
78
|
+
probe?: (provider: string, model: string, configManager: ConfigManager) => Promise<ProbeOutcome>;
|
|
79
|
+
/** Injectable clock (tests). */
|
|
80
|
+
now?: () => number;
|
|
81
|
+
/** Visibility hook — a revival is worth showing in the console/trace. */
|
|
82
|
+
onEvent?: (line: string) => void;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Adapt a raw (exclusions, transient markers) pair to the revival port. This is
|
|
86
|
+
* the shape every CLI path holds, including chat, whose two collections are
|
|
87
|
+
* instance fields rather than a `FailureSessionState` object.
|
|
88
|
+
*/
|
|
89
|
+
export declare function collectionRevivalStore(sessionFailedProviders: Map<string, number>, sessionTransientFailedProviders: Set<string>): RevivalStore;
|
|
90
|
+
/** Adapt the shared `FailureSessionState` to the revival port. */
|
|
91
|
+
export declare function sessionRevivalStore(session: FailureSessionState): RevivalStore;
|
|
92
|
+
/**
|
|
93
|
+
* Re-verify transiently-failed providers and re-admit the ones that recovered.
|
|
94
|
+
*
|
|
95
|
+
* Contract (identical to the chat path this was extracted from, so behaviour does
|
|
96
|
+
* not change there):
|
|
97
|
+
* - a provider whose exclusion is still ACTIVE is left alone;
|
|
98
|
+
* - a provider the registry no longer blocks needs no probe — it is re-admitted;
|
|
99
|
+
* - otherwise a 1-token spot-check decides: verified/skipped → re-admit,
|
|
100
|
+
* anything else → re-arm the exclusion for another transient window.
|
|
101
|
+
*
|
|
102
|
+
* Never throws: revival is an optimisation and must never break a call.
|
|
103
|
+
*/
|
|
104
|
+
export declare function sweepTransientFailures(store: RevivalStore, configManager: ConfigManager, options?: RevivalSweepOptions): Promise<RevivalSweepResult>;
|
|
105
|
+
export {};
|
|
106
|
+
//# sourceMappingURL=provider-revival.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider-revival.d.ts","sourceRoot":"","sources":["../../src/learning/provider-revival.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAkC,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,8EAA8E;AAC9E,KAAK,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,wDAAwD;IACxD,kBAAkB,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvC,4DAA4D;IAC5D,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1D,0EAA0E;IAC1E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,qDAAqD;IACrD,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrD,qDAAqD;IACrD,iBAAiB,CAAC,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC1D;AAED,MAAM,WAAW,kBAAkB;IACjC,sDAAsD;IACtD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,gEAAgE;IAChE,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;IACjG,gCAAgC;IAChC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,yEAAyE;IACzE,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAID;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,sBAAsB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC3C,+BAA+B,EAAE,GAAG,CAAC,MAAM,CAAC,GAC3C,YAAY,CAgBd;AAED,kEAAkE;AAClE,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,mBAAmB,GAAG,YAAY,CAE9E;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,YAAY,EACnB,aAAa,EAAE,aAAa,EAC5B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA+D7B"}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider revival — the "one more round of check" before giving up on a
|
|
3
|
+
* provider that failed transiently.
|
|
4
|
+
*
|
|
5
|
+
* THE BUG THIS FIXES (verified): a provider that fails transiently (server /
|
|
6
|
+
* network / timeout) is put in a short session exclusion AND, separately, marked
|
|
7
|
+
* in a `transientFailed` set that carries the documented promise — "awaiting a
|
|
8
|
+
* quick on-demand spot-check before re-admission (never re-pick without proof)".
|
|
9
|
+
* That promise was kept in exactly ONE place: `chat.ts`'s session loop. Every
|
|
10
|
+
* other entry path allocated the marker and never read it:
|
|
11
|
+
*
|
|
12
|
+
* path allocates reads (revives)
|
|
13
|
+
* chat.ts ✅ ✅
|
|
14
|
+
* orchestrator.ts ✅ ✅ (covers execute.ts's pipeline path)
|
|
15
|
+
* resilient-call.ts ✅ ✅
|
|
16
|
+
* edit.ts ✅ ✅ (via the runSingleShotAuto `revive` hook)
|
|
17
|
+
* execute.ts ✅ ✅ (the follow-up path that bypasses the
|
|
18
|
+
* orchestrator sweeps its own session)
|
|
19
|
+
* plan.ts ✅ ✅ (via the runSingleShotAuto `revive` hook)
|
|
20
|
+
*
|
|
21
|
+
* Every allocating path now reads what it arms. When this was not true, an
|
|
22
|
+
* exhausted provider stayed
|
|
23
|
+
* excluded for the rest of the session even after it recovered, and the run
|
|
24
|
+
* either degraded or died — while the machinery to prove recovery in seconds sat
|
|
25
|
+
* unused. Live evidence: a pipeline run hit a Gemini 503, the breaker parked the
|
|
26
|
+
* provider, context-gatherer fell back to "0 relevant files", and the writer then
|
|
27
|
+
* edited blind. A single capacity spike at one endpoint became a wrong answer.
|
|
28
|
+
*
|
|
29
|
+
* WHY A STORE ADAPTER: the callers keep their session state in two different
|
|
30
|
+
* shapes — `FailureSessionState` (chat/orchestrator/edit/plan/execute) and
|
|
31
|
+
* resilient-call's own `{ expiresAt, kind }` maps. Rather than duplicate the
|
|
32
|
+
* probe logic per shape (the duplication that caused this bug), the probe works
|
|
33
|
+
* against this small port and each caller adapts.
|
|
34
|
+
*
|
|
35
|
+
* Cycle safety: `auto-router.js` is imported LAZILY (only when a caller does not
|
|
36
|
+
* supply its own model resolver), so this module can be imported from the
|
|
37
|
+
* router's own dependencies without creating a static cycle.
|
|
38
|
+
*/
|
|
39
|
+
import { spotCheckModel } from '../inference/model-probe.js';
|
|
40
|
+
import { getModelRegistry } from './model-registry.js';
|
|
41
|
+
import { TRANSIENT_FAILURE_EXCLUSION_MS } from './failure-bookkeeping.js';
|
|
42
|
+
import { logger } from '../utils/logger.js';
|
|
43
|
+
const DEFAULT_MAX_PROBES = 3;
|
|
44
|
+
/**
|
|
45
|
+
* Adapt a raw (exclusions, transient markers) pair to the revival port. This is
|
|
46
|
+
* the shape every CLI path holds, including chat, whose two collections are
|
|
47
|
+
* instance fields rather than a `FailureSessionState` object.
|
|
48
|
+
*/
|
|
49
|
+
export function collectionRevivalStore(sessionFailedProviders, sessionTransientFailedProviders) {
|
|
50
|
+
return {
|
|
51
|
+
transientProviders: () => sessionTransientFailedProviders,
|
|
52
|
+
isExclusionActive: (provider, now) => {
|
|
53
|
+
const expiresAt = sessionFailedProviders.get(provider);
|
|
54
|
+
return expiresAt !== undefined && expiresAt > now;
|
|
55
|
+
},
|
|
56
|
+
clearProvider: (provider) => {
|
|
57
|
+
sessionTransientFailedProviders.delete(provider);
|
|
58
|
+
sessionFailedProviders.delete(provider);
|
|
59
|
+
},
|
|
60
|
+
reArmProvider: (provider, until) => {
|
|
61
|
+
sessionFailedProviders.set(provider, until);
|
|
62
|
+
sessionTransientFailedProviders.add(provider);
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Adapt the shared `FailureSessionState` to the revival port. */
|
|
67
|
+
export function sessionRevivalStore(session) {
|
|
68
|
+
return collectionRevivalStore(session.sessionFailedProviders, session.sessionTransientFailedProviders);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Re-verify transiently-failed providers and re-admit the ones that recovered.
|
|
72
|
+
*
|
|
73
|
+
* Contract (identical to the chat path this was extracted from, so behaviour does
|
|
74
|
+
* not change there):
|
|
75
|
+
* - a provider whose exclusion is still ACTIVE is left alone;
|
|
76
|
+
* - a provider the registry no longer blocks needs no probe — it is re-admitted;
|
|
77
|
+
* - otherwise a 1-token spot-check decides: verified/skipped → re-admit,
|
|
78
|
+
* anything else → re-arm the exclusion for another transient window.
|
|
79
|
+
*
|
|
80
|
+
* Never throws: revival is an optimisation and must never break a call.
|
|
81
|
+
*/
|
|
82
|
+
export async function sweepTransientFailures(store, configManager, options = {}) {
|
|
83
|
+
const result = { probed: [], revived: [], stillDown: [] };
|
|
84
|
+
const now = options.now ?? Date.now;
|
|
85
|
+
const probe = options.probe ?? spotCheckModel;
|
|
86
|
+
const maxProbes = options.maxProbes ?? DEFAULT_MAX_PROBES;
|
|
87
|
+
const agentType = options.agentType ?? 'chat';
|
|
88
|
+
try {
|
|
89
|
+
// SNAPSHOT: clearProvider/reArmProvider mutate the set, and Set iteration can
|
|
90
|
+
// revisit a re-added key — which would double-probe it.
|
|
91
|
+
const candidates = [...store.transientProviders()];
|
|
92
|
+
if (candidates.length === 0)
|
|
93
|
+
return result;
|
|
94
|
+
for (const provider of candidates) {
|
|
95
|
+
if (result.probed.length >= maxProbes)
|
|
96
|
+
break;
|
|
97
|
+
const at = now();
|
|
98
|
+
if (store.isExclusionActive(provider, at))
|
|
99
|
+
continue;
|
|
100
|
+
// Optimistic: assume admitted, then re-arm if the probe says otherwise.
|
|
101
|
+
// (A throw anywhere below leaves the provider admitted, exactly like the
|
|
102
|
+
// chat path did — an unreachable registry must not freeze a provider out.)
|
|
103
|
+
store.clearProvider(provider);
|
|
104
|
+
const registry = getModelRegistry();
|
|
105
|
+
if (!registry.getBlockedProviders().includes(provider)) {
|
|
106
|
+
// Nothing to prove: the registry already considers it healthy (a
|
|
107
|
+
// time-based park lapsed, or telemetry healed it).
|
|
108
|
+
result.revived.push(provider);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
let desired;
|
|
112
|
+
try {
|
|
113
|
+
desired = store.resolveProbeModel?.(provider) ?? (await defaultProbeModel(provider, agentType, configManager));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
desired = undefined;
|
|
117
|
+
}
|
|
118
|
+
// No model to probe → the registry block is not about a specific model we
|
|
119
|
+
// can verify; admit it rather than stranding the provider forever.
|
|
120
|
+
if (!desired) {
|
|
121
|
+
result.revived.push(provider);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
result.probed.push(provider);
|
|
125
|
+
const outcome = await probe(provider, desired, configManager);
|
|
126
|
+
// 'skipped' = the model was verified recently (inside the spot-check
|
|
127
|
+
// throttle) — that is healthy, so it counts as a pass.
|
|
128
|
+
if (outcome === 'verified' || outcome === 'skipped') {
|
|
129
|
+
result.revived.push(provider);
|
|
130
|
+
options.onEvent?.(` ♻️ ${provider} is back — re-admitted after a spot-check`);
|
|
131
|
+
logger.info(` ♻️ ${provider} recovered (spot-check ${outcome}) — re-admitted to routing`);
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
store.reArmProvider(provider, at + TRANSIENT_FAILURE_EXCLUSION_MS);
|
|
135
|
+
result.stillDown.push(provider);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
// Best-effort — a sweep failure must never break the call that triggered it.
|
|
141
|
+
}
|
|
142
|
+
return result;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Default probe model: the router's own pick for this provider, so the sweep
|
|
146
|
+
* verifies the model that routing would actually serve. Lazy import keeps this
|
|
147
|
+
* module free of a static dependency on the router.
|
|
148
|
+
*/
|
|
149
|
+
async function defaultProbeModel(provider, agentType, configManager) {
|
|
150
|
+
const { getAutoRouter } = await import('./auto-router.js');
|
|
151
|
+
const model = getAutoRouter().resolveModel(provider, agentType, configManager);
|
|
152
|
+
return model && model !== 'default' ? model : undefined;
|
|
153
|
+
}
|
|
154
|
+
//# sourceMappingURL=provider-revival.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider-revival.js","sourceRoot":"","sources":["../../src/learning/provider-revival.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,8BAA8B,EAA4B,MAAM,0BAA0B,CAAC;AACpG,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAiD5C,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,sBAA2C,EAC3C,+BAA4C;IAE5C,OAAO;QACL,kBAAkB,EAAE,GAAG,EAAE,CAAC,+BAA+B;QACzD,iBAAiB,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YACnC,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvD,OAAO,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,GAAG,CAAC;QACpD,CAAC;QACD,aAAa,EAAE,CAAC,QAAQ,EAAE,EAAE;YAC1B,+BAA+B,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACjD,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;QACD,aAAa,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YACjC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC5C,+BAA+B,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,mBAAmB,CAAC,OAA4B;IAC9D,OAAO,sBAAsB,CAAC,OAAO,CAAC,sBAAsB,EAAE,OAAO,CAAC,+BAA+B,CAAC,CAAC;AACzG,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAmB,EACnB,aAA4B,EAC5B,UAA+B,EAAE;IAEjC,MAAM,MAAM,GAAuB,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC9E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,cAAc,CAAC;IAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAE9C,IAAI,CAAC;QACH,8EAA8E;QAC9E,wDAAwD;QACxD,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACnD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS;gBAAE,MAAM;YAE7C,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YACjB,IAAI,KAAK,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAAE,SAAS;YAEpD,wEAAwE;YACxE,yEAAyE;YACzE,2EAA2E;YAC3E,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,iEAAiE;gBACjE,mDAAmD;gBACnD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,IAAI,OAA2B,CAAC;YAChC,IAAI,CAAC;gBACH,OAAO,GAAG,KAAK,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;YACjH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;YACD,0EAA0E;YAC1E,mEAAmE;YACnE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;YAC9D,qEAAqE;YACrE,uDAAuD;YACvD,IAAI,OAAO,KAAK,UAAU,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBACpD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9B,OAAO,CAAC,OAAO,EAAE,CAAC,SAAS,QAAQ,2CAA2C,CAAC,CAAC;gBAChF,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,0BAA0B,OAAO,4BAA4B,CAAC,CAAC;YAC9F,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,8BAA8B,CAAC,CAAC;gBACnE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6EAA6E;IAC/E,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAC9B,QAAgB,EAChB,SAAiB,EACjB,aAA4B;IAE5B,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/E,OAAO,KAAK,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resilient-call.d.ts","sourceRoot":"","sources":["../../src/learning/resilient-call.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,EAAiB,KAAK,eAAe,EAAuB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAqB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"resilient-call.d.ts","sourceRoot":"","sources":["../../src/learning/resilient-call.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,EAAiB,KAAK,eAAe,EAAuB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAqB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAc7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAIpD,oDAAoD;AACpD,KAAK,WAAW,GAAG,MAAM,GAAG,YAAY,GAAG,SAAS,GAAG,SAAS,GAAG,iBAAiB,GAAG,SAAS,CAAC;AAEjG,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,kEAAkE;IAClE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,iEAAiE;IACjE,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,6CAA6C;AAC7C,MAAM,WAAW,oBAAoB;IACnC,qFAAqF;IACrF,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,wCAAwC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gDAAgD;IAChD,IAAI,EAAE;QACJ,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAgFD;;;;;GAKG;AACH,UAAU,iBAAiB;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,WAAW,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E;AAsED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,oBAAoB,GAC5B,SAAS,CA6SX;AA2DD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,eAAe,GAAG,IAAI,EAChC,IAAI,GAAE,mBAAwB,GAC7B,iBAAiB,EAAE,CA0HrB;AAaD,gFAAgF;AAChF,KAAK,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,qEAAqE;IACrE,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,+DAA+D;IAC/D,mBAAmB,CAAC,EAAE,YAAY,CAAC;IACnC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,gEAAgE;IAChE,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,6BAA6B,CAC3C,IAAI,GAAE,wBAA6B,GAClC,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAW/C"}
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
* - Tools: ctx.callLLM is already resilient (inherited from orchestrator)
|
|
27
27
|
*/
|
|
28
28
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
29
|
-
import { homedir } from 'node:os';
|
|
30
29
|
import { join } from 'node:path';
|
|
30
|
+
import { resolveNuviraConfigDir, resolveNuviraDataPath } from '../config/paths.js';
|
|
31
31
|
import { getAutoRouter } from './auto-router.js';
|
|
32
32
|
import { analyzeComplexity } from './hybrid-router.js';
|
|
33
33
|
import { buildAutoResolveOptions } from './resolve-options.js';
|
|
@@ -37,6 +37,7 @@ import { resolveWorkingModel } from '../inference/model-validator.js';
|
|
|
37
37
|
import { getDefaultModel } from '../inference/provider-catalog.js';
|
|
38
38
|
import { getModelRegistry } from './model-registry.js';
|
|
39
39
|
import { recordActionFailure } from './failure-bookkeeping.js';
|
|
40
|
+
import { sweepTransientFailures } from './provider-revival.js';
|
|
40
41
|
import { getProviderFallback, recordRegistrySuccess } from './provider-fallback.js';
|
|
41
42
|
import { ProviderFactory } from '../inference/factory.js';
|
|
42
43
|
import { recordRoutingDecision } from './routing-history.js';
|
|
@@ -90,7 +91,11 @@ function exclusionDuration(kind) {
|
|
|
90
91
|
}
|
|
91
92
|
function loadPersistedFailures() {
|
|
92
93
|
try {
|
|
93
|
-
|
|
94
|
+
// Resolved through the active config dir (`$NUVIRA_CONFIG_DIR` aware) — a
|
|
95
|
+
// hardcoded `~/.nuvira` here made an isolated process inherit (and mutate)
|
|
96
|
+
// the real profile's exclusions, so a cooldown earned by a live run
|
|
97
|
+
// silently suppressed models inside tests and sandboxes.
|
|
98
|
+
const path = resolveNuviraDataPath(FAILURE_PERSIST_PATH);
|
|
94
99
|
if (!existsSync(path))
|
|
95
100
|
return {};
|
|
96
101
|
const raw = readFileSync(path, 'utf-8');
|
|
@@ -115,7 +120,7 @@ function loadPersistedFailures() {
|
|
|
115
120
|
*/
|
|
116
121
|
function persistFailure(provider, kind, model) {
|
|
117
122
|
try {
|
|
118
|
-
const dir =
|
|
123
|
+
const dir = resolveNuviraConfigDir();
|
|
119
124
|
if (!existsSync(dir))
|
|
120
125
|
mkdirSync(dir, { recursive: true });
|
|
121
126
|
const path = join(dir, FAILURE_PERSIST_PATH);
|
|
@@ -166,6 +171,13 @@ export function createResilientCallLLM(configManager, options) {
|
|
|
166
171
|
currentModel: '',
|
|
167
172
|
exhausted: false,
|
|
168
173
|
};
|
|
174
|
+
// Providers excluded by a TRANSIENT failure (server/network/timeout) during
|
|
175
|
+
// this session. Tracked separately from `state.sessionFailed` because a
|
|
176
|
+
// transient exclusion is PROVISIONAL: it is only upheld until a spot-check
|
|
177
|
+
// proves the provider is still down. The state previously allocated this
|
|
178
|
+
// marker for bookkeeping and never read it, so a provider that recovered kept
|
|
179
|
+
// being skipped for the entire task.
|
|
180
|
+
const transientProviders = new Set();
|
|
169
181
|
// Load cross-pipeline failures
|
|
170
182
|
const persistedFailures = options.crossPipelineMemory !== false ? loadPersistedFailures() : {};
|
|
171
183
|
// Initial routing decision
|
|
@@ -195,6 +207,30 @@ export function createResilientCallLLM(configManager, options) {
|
|
|
195
207
|
if (state.exhausted) {
|
|
196
208
|
throw new Error(`All LLM providers exhausted. No more candidates available for: ${options.task.description}`);
|
|
197
209
|
}
|
|
210
|
+
// ── Re-verify before re-admit ──────────────────────────────────────────
|
|
211
|
+
// Before choosing a candidate, give any provider excluded by a TRANSIENT
|
|
212
|
+
// failure the one-more-round the state always intended: a 1-token
|
|
213
|
+
// spot-check. Verified → back in the pool for THIS call; still down → its
|
|
214
|
+
// exclusion is re-armed. Best-effort and bounded (the sweep stops at 3
|
|
215
|
+
// probes and skips still-active exclusions without a network call).
|
|
216
|
+
try {
|
|
217
|
+
await sweepTransientFailures({
|
|
218
|
+
transientProviders: () => transientProviders,
|
|
219
|
+
isExclusionActive: (provider, now) => expiryAt(state.sessionFailed.get(provider)) > now,
|
|
220
|
+
clearProvider: (provider) => {
|
|
221
|
+
transientProviders.delete(provider);
|
|
222
|
+
state.sessionFailed.delete(provider);
|
|
223
|
+
},
|
|
224
|
+
reArmProvider: (provider, until) => {
|
|
225
|
+
state.sessionFailed.set(provider, { expiresAt: until, kind: 'unknown' });
|
|
226
|
+
transientProviders.add(provider);
|
|
227
|
+
},
|
|
228
|
+
resolveProbeModel: (provider) => autoRouter.resolveModel(provider, options.task.agentType, configManager),
|
|
229
|
+
}, configManager, { agentType: options.task.agentType });
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
// Best-effort — revival must never break the call.
|
|
233
|
+
}
|
|
198
234
|
// Try current provider first, then walk all candidates
|
|
199
235
|
const candidatesToTry = [
|
|
200
236
|
{ provider: state.currentProvider, model: state.currentModel, score: 1.0 },
|
|
@@ -325,6 +361,15 @@ export function createResilientCallLLM(configManager, options) {
|
|
|
325
361
|
kind,
|
|
326
362
|
});
|
|
327
363
|
}
|
|
364
|
+
// TRANSIENT failures are PROVISIONAL: mark the provider as awaiting a
|
|
365
|
+
// spot-check so a later call re-verifies it instead of skipping it for
|
|
366
|
+
// the rest of the task. auth = the key is dead (never revives on its
|
|
367
|
+
// own), rate-limit = the provider's own reset window governs, and
|
|
368
|
+
// model-not-found is definitive for that model — none of those are
|
|
369
|
+
// helped by a probe, so they are deliberately NOT marked.
|
|
370
|
+
if (kind === 'network' || kind === 'timeout' || kind === 'unknown') {
|
|
371
|
+
transientProviders.add(candidate.provider);
|
|
372
|
+
}
|
|
328
373
|
// Persist for cross-pipeline memory
|
|
329
374
|
if (options.crossPipelineMemory !== false) {
|
|
330
375
|
persistFailure(candidate.provider, kind, modelScoped ? failedModel : undefined);
|