@jsonstudio/rcc 0.90.876 → 0.90.1270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -6
- package/configsamples/provider-default/ali-coding-plan/config.v2.json +6 -1
- package/configsamples/provider-default/antigravity/config.v2.json +6 -1
- package/configsamples/provider-default/ark-coding-plan/config.v2.json +7 -2
- package/configsamples/provider-default/crs/config.v2.json +6 -1
- package/configsamples/provider-default/deepseek-web/config.v2.json +6 -1
- package/configsamples/provider-default/gemini/config.v2.json +6 -1
- package/configsamples/provider-default/gemini-cli/config.v2.json +6 -1
- package/configsamples/provider-default/gemini-native/config.v2.json +6 -1
- package/configsamples/provider-default/glm/config.v2.json +7 -2
- package/configsamples/provider-default/glm-anthropic/config.v2.json +6 -1
- package/configsamples/provider-default/kimi/config.v2.json +7 -2
- package/configsamples/provider-default/lmstudio/config.v2.json +6 -1
- package/configsamples/provider-default/lmstudio-proxy/config.v2.json +6 -1
- package/configsamples/provider-default/manifest.json +0 -1
- package/configsamples/provider-default/meituan/config.v2.json +6 -1
- package/configsamples/provider-default/mimo/config.v2.json +7 -2
- package/configsamples/provider-default/modelscope/config.v2.json +7 -2
- package/configsamples/provider-default/my-openai/config.v2.json +6 -1
- package/configsamples/provider-default/nvidia/config.v2.json +7 -2
- package/configsamples/provider-default/opencode-zen-free/config.v2.json +6 -1
- package/configsamples/provider-default/openrouter/config.v2.json +6 -1
- package/configsamples/provider-default/qwen/config.v2.json +11 -1
- package/configsamples/provider-default/tab/config.v2.json +6 -1
- package/configsamples/provider-default/tabglm/config.v2.json +7 -2
- package/dist/build-info.js +2 -2
- package/dist/build-info.js.map +1 -1
- package/dist/cli/commands/camoufox.js +44 -3
- package/dist/cli/commands/camoufox.js.map +1 -1
- package/dist/cli/commands/config.js +2 -2
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/heartbeat.js +82 -27
- package/dist/cli/commands/heartbeat.js.map +1 -1
- package/dist/cli/commands/init.js +1 -2
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/launcher/utils.js +37 -24
- package/dist/cli/commands/launcher/utils.js.map +1 -1
- package/dist/cli/commands/launcher-kernel.js +6 -3
- package/dist/cli/commands/launcher-kernel.js.map +1 -1
- package/dist/cli/commands/restart.js +102 -31
- package/dist/cli/commands/restart.js.map +1 -1
- package/dist/cli/commands/start-types.d.ts +1 -0
- package/dist/cli/commands/start-utils.d.ts +1 -0
- package/dist/cli/commands/start-utils.js +3 -0
- package/dist/cli/commands/start-utils.js.map +1 -1
- package/dist/cli/commands/start.js +217 -51
- package/dist/cli/commands/start.js.map +1 -1
- package/dist/cli/commands/status.js +48 -9
- package/dist/cli/commands/status.js.map +1 -1
- package/dist/cli/config/bootstrap-provider-templates.js +1 -1
- package/dist/cli/config/bootstrap-provider-templates.js.map +1 -1
- package/dist/cli/config/init-provider-catalog.js +3 -50
- package/dist/cli/config/init-provider-catalog.js.map +1 -1
- package/dist/cli/config/init-v2-builder.js +0 -1
- package/dist/cli/config/init-v2-builder.js.map +1 -1
- package/dist/cli/guardian/client.js +40 -18
- package/dist/cli/guardian/client.js.map +1 -1
- package/dist/cli/server/port-utils.d.ts +5 -0
- package/dist/cli/server/port-utils.js +45 -31
- package/dist/cli/server/port-utils.js.map +1 -1
- package/dist/commands/camoufox-fp.js +1 -1
- package/dist/commands/camoufox-fp.js.map +1 -1
- package/dist/commands/oauth.js +48 -64
- package/dist/commands/oauth.js.map +1 -1
- package/dist/commands/provider-update-maintenance.js +4 -4
- package/dist/commands/provider-update-maintenance.js.map +1 -1
- package/dist/commands/token-daemon.js +38 -10
- package/dist/commands/token-daemon.js.map +1 -1
- package/dist/commands/validate.js +31 -3
- package/dist/commands/validate.js.map +1 -1
- package/dist/config/provider-v2-loader.d.ts +5 -2
- package/dist/config/provider-v2-loader.js +80 -26
- package/dist/config/provider-v2-loader.js.map +1 -1
- package/dist/config/routecodex-config-loader.d.ts +1 -0
- package/dist/config/routecodex-config-loader.js +18 -207
- package/dist/config/routecodex-config-loader.js.map +1 -1
- package/dist/config/virtual-router-builder.d.ts +3 -2
- package/dist/config/virtual-router-builder.js +4 -214
- package/dist/config/virtual-router-builder.js.map +1 -1
- package/dist/constants/index.d.ts +2 -3
- package/dist/constants/index.js +2 -4
- package/dist/constants/index.js.map +1 -1
- package/dist/error-handling/route-error-hub.js +1 -0
- package/dist/error-handling/route-error-hub.js.map +1 -1
- package/dist/index.js +98 -21
- package/dist/index.js.map +1 -1
- package/dist/manager/modules/quota/antigravity-quota-manager.d.ts +1 -1
- package/dist/manager/modules/quota/antigravity-quota-manager.js +21 -12
- package/dist/manager/modules/quota/antigravity-quota-manager.js.map +1 -1
- package/dist/manager/modules/quota/antigravity-quota-runtime.d.ts +1 -1
- package/dist/manager/modules/quota/antigravity-quota-runtime.js +32 -45
- package/dist/manager/modules/quota/antigravity-quota-runtime.js.map +1 -1
- package/dist/manager/modules/quota/provider-key-normalization.js +10 -1
- package/dist/manager/modules/quota/provider-key-normalization.js.map +1 -1
- package/dist/manager/modules/quota/provider-quota-daemon.d.ts +2 -1
- package/dist/manager/modules/quota/provider-quota-daemon.error-helpers.d.ts +1 -1
- package/dist/manager/modules/quota/provider-quota-daemon.error-helpers.js +22 -12
- package/dist/manager/modules/quota/provider-quota-daemon.error-helpers.js.map +1 -1
- package/dist/manager/modules/quota/provider-quota-daemon.events.js +69 -33
- package/dist/manager/modules/quota/provider-quota-daemon.events.js.map +1 -1
- package/dist/manager/modules/quota/provider-quota-daemon.js +47 -35
- package/dist/manager/modules/quota/provider-quota-daemon.js.map +1 -1
- package/dist/manager/storage/file-store.js +3 -0
- package/dist/manager/storage/file-store.js.map +1 -1
- package/dist/modules/llmswitch/bridge/antigravity-signature.js +58 -20
- package/dist/modules/llmswitch/bridge/antigravity-signature.js.map +1 -1
- package/dist/modules/llmswitch/bridge/index.d.ts +1 -1
- package/dist/modules/llmswitch/bridge/index.js +1 -1
- package/dist/modules/llmswitch/bridge/index.js.map +1 -1
- package/dist/modules/llmswitch/bridge/runtime-integrations.d.ts +22 -16
- package/dist/modules/llmswitch/bridge/runtime-integrations.js +89 -30
- package/dist/modules/llmswitch/bridge/runtime-integrations.js.map +1 -1
- package/dist/modules/llmswitch/bridge/snapshot-recorder-runtime.js +10 -9
- package/dist/modules/llmswitch/bridge/snapshot-recorder-runtime.js.map +1 -1
- package/dist/modules/llmswitch/bridge/snapshot-recorder.js +3 -0
- package/dist/modules/llmswitch/bridge/snapshot-recorder.js.map +1 -1
- package/dist/modules/llmswitch/bridge/state-integrations.d.ts +1 -0
- package/dist/modules/llmswitch/bridge/state-integrations.js +263 -51
- package/dist/modules/llmswitch/bridge/state-integrations.js.map +1 -1
- package/dist/modules/llmswitch/bridge.d.ts +2 -2
- package/dist/modules/llmswitch/bridge.js +2 -2
- package/dist/modules/llmswitch/bridge.js.map +1 -1
- package/dist/provider-sdk/provider-add-template.d.ts +1 -1
- package/dist/provider-sdk/provider-add-template.js.map +1 -1
- package/dist/provider-sdk/provider-runtime-inference.js +48 -13
- package/dist/provider-sdk/provider-runtime-inference.js.map +1 -1
- package/dist/providers/auth/deepseek-account-token-acquirer.d.ts +24 -0
- package/dist/providers/auth/deepseek-account-token-acquirer.js +42 -13
- package/dist/providers/auth/deepseek-account-token-acquirer.js.map +1 -1
- package/dist/providers/auth/oauth-auth.js +12 -5
- package/dist/providers/auth/oauth-auth.js.map +1 -1
- package/dist/providers/auth/oauth-lifecycle/path-resolver.d.ts +0 -1
- package/dist/providers/auth/oauth-lifecycle/path-resolver.js +10 -9
- package/dist/providers/auth/oauth-lifecycle/path-resolver.js.map +1 -1
- package/dist/providers/auth/oauth-lifecycle/token-helpers.js +0 -1
- package/dist/providers/auth/oauth-lifecycle/token-helpers.js.map +1 -1
- package/dist/providers/auth/oauth-lifecycle/token-io.js +18 -8
- package/dist/providers/auth/oauth-lifecycle/token-io.js.map +1 -1
- package/dist/providers/auth/oauth-lifecycle.d.ts +5 -0
- package/dist/providers/auth/oauth-lifecycle.js +370 -353
- package/dist/providers/auth/oauth-lifecycle.js.map +1 -1
- package/dist/providers/auth/oauth-repair-env.js +0 -26
- package/dist/providers/auth/oauth-repair-env.js.map +1 -1
- package/dist/providers/auth/qwen-userinfo-helper.d.ts +11 -0
- package/dist/providers/auth/qwen-userinfo-helper.js +85 -13
- package/dist/providers/auth/qwen-userinfo-helper.js.map +1 -1
- package/dist/providers/auth/token-refresh/token-state.js +1 -4
- package/dist/providers/auth/token-refresh/token-state.js.map +1 -1
- package/dist/providers/auth/token-scanner/index.d.ts +1 -1
- package/dist/providers/auth/token-scanner/index.js +2 -2
- package/dist/providers/auth/token-storage/token-file-resolver.js +0 -3
- package/dist/providers/auth/token-storage/token-file-resolver.js.map +1 -1
- package/dist/providers/auth/token-storage/token-persistence.js +10 -3
- package/dist/providers/auth/token-storage/token-persistence.js.map +1 -1
- package/dist/providers/auth/tokenfile-auth.d.ts +0 -1
- package/dist/providers/auth/tokenfile-auth.js +23 -30
- package/dist/providers/auth/tokenfile-auth.js.map +1 -1
- package/dist/providers/core/api/provider-config.d.ts +1 -1
- package/dist/providers/core/api/provider-types.d.ts +2 -1
- package/dist/providers/core/config/camoufox-actions.js +23 -15
- package/dist/providers/core/config/camoufox-actions.js.map +1 -1
- package/dist/providers/core/config/camoufox-launcher.js +235 -121
- package/dist/providers/core/config/camoufox-launcher.js.map +1 -1
- package/dist/providers/core/config/oauth-flows.js +23 -1
- package/dist/providers/core/config/oauth-flows.js.map +1 -1
- package/dist/providers/core/config/provider-oauth-configs.js +2 -93
- package/dist/providers/core/config/provider-oauth-configs.js.map +1 -1
- package/dist/providers/core/config/service-profiles.d.ts +0 -1
- package/dist/providers/core/config/service-profiles.js +23 -66
- package/dist/providers/core/config/service-profiles.js.map +1 -1
- package/dist/providers/core/runtime/base-provider-runtime-helpers.js +4 -1
- package/dist/providers/core/runtime/base-provider-runtime-helpers.js.map +1 -1
- package/dist/providers/core/runtime/base-provider.js +21 -11
- package/dist/providers/core/runtime/base-provider.js.map +1 -1
- package/dist/providers/core/runtime/deepseek-http-provider-helpers.d.ts +1 -0
- package/dist/providers/core/runtime/deepseek-http-provider-helpers.js +28 -0
- package/dist/providers/core/runtime/deepseek-http-provider-helpers.js.map +1 -1
- package/dist/providers/core/runtime/deepseek-http-provider.d.ts +0 -2
- package/dist/providers/core/runtime/deepseek-http-provider.js +3 -33
- package/dist/providers/core/runtime/deepseek-http-provider.js.map +1 -1
- package/dist/providers/core/runtime/http-request-executor.d.ts +1 -0
- package/dist/providers/core/runtime/http-request-executor.js +131 -47
- package/dist/providers/core/runtime/http-request-executor.js.map +1 -1
- package/dist/providers/core/runtime/http-transport-provider.d.ts +5 -5
- package/dist/providers/core/runtime/http-transport-provider.js +97 -38
- package/dist/providers/core/runtime/http-transport-provider.js.map +1 -1
- package/dist/providers/core/runtime/openai-responses-sdk-transport.js +2 -1
- package/dist/providers/core/runtime/openai-responses-sdk-transport.js.map +1 -1
- package/dist/providers/core/runtime/provider-error-classifier.js +19 -126
- package/dist/providers/core/runtime/provider-error-classifier.js.map +1 -1
- package/dist/providers/core/runtime/provider-factory-helpers.d.ts +1 -0
- package/dist/providers/core/runtime/provider-factory-helpers.js +8 -12
- package/dist/providers/core/runtime/provider-factory-helpers.js.map +1 -1
- package/dist/providers/core/runtime/provider-factory.js +2 -9
- package/dist/providers/core/runtime/provider-factory.js.map +1 -1
- package/dist/providers/core/runtime/provider-failure-policy.d.ts +135 -0
- package/dist/providers/core/runtime/provider-failure-policy.js +685 -0
- package/dist/providers/core/runtime/provider-failure-policy.js.map +1 -0
- package/dist/providers/core/runtime/provider-family-profile-utils.d.ts +0 -10
- package/dist/providers/core/runtime/provider-family-profile-utils.js +0 -28
- package/dist/providers/core/runtime/provider-family-profile-utils.js.map +1 -1
- package/dist/providers/core/runtime/provider-http-executor-utils.js +40 -1
- package/dist/providers/core/runtime/provider-http-executor-utils.js.map +1 -1
- package/dist/providers/core/runtime/provider-request-header-orchestrator.d.ts +0 -2
- package/dist/providers/core/runtime/provider-request-header-orchestrator.js +35 -8
- package/dist/providers/core/runtime/provider-request-header-orchestrator.js.map +1 -1
- package/dist/providers/core/runtime/provider-response-postprocessor.js +3 -23
- package/dist/providers/core/runtime/provider-response-postprocessor.js.map +1 -1
- package/dist/providers/core/runtime/provider-runtime-utils.js +4 -1
- package/dist/providers/core/runtime/provider-runtime-utils.js.map +1 -1
- package/dist/providers/core/runtime/provider-startup-tasks.js +18 -2
- package/dist/providers/core/runtime/provider-startup-tasks.js.map +1 -1
- package/dist/providers/core/runtime/responses-provider-helpers.d.ts +1 -0
- package/dist/providers/core/runtime/responses-provider-helpers.js +11 -12
- package/dist/providers/core/runtime/responses-provider-helpers.js.map +1 -1
- package/dist/providers/core/runtime/responses-provider.js +15 -10
- package/dist/providers/core/runtime/responses-provider.js.map +1 -1
- package/dist/providers/core/runtime/transport/auth-provider-factory.d.ts +0 -1
- package/dist/providers/core/runtime/transport/auth-provider-factory.js +1 -15
- package/dist/providers/core/runtime/transport/auth-provider-factory.js.map +1 -1
- package/dist/providers/core/runtime/transport/index.d.ts +0 -1
- package/dist/providers/core/runtime/transport/index.js +0 -1
- package/dist/providers/core/runtime/transport/index.js.map +1 -1
- package/dist/providers/core/runtime/transport/oauth-recovery-handler.d.ts +2 -0
- package/dist/providers/core/runtime/transport/oauth-recovery-handler.js +76 -5
- package/dist/providers/core/runtime/transport/oauth-recovery-handler.js.map +1 -1
- package/dist/providers/core/runtime/transport/request-header-builder.d.ts +0 -1
- package/dist/providers/core/runtime/transport/request-header-builder.js +1 -7
- package/dist/providers/core/runtime/transport/request-header-builder.js.map +1 -1
- package/dist/providers/core/runtime/transport/runtime-detector.d.ts +0 -2
- package/dist/providers/core/runtime/transport/runtime-detector.js +0 -20
- package/dist/providers/core/runtime/transport/runtime-detector.js.map +1 -1
- package/dist/providers/core/runtime/transport/session-header-utils.d.ts +6 -0
- package/dist/providers/core/runtime/transport/session-header-utils.js +61 -2
- package/dist/providers/core/runtime/transport/session-header-utils.js.map +1 -1
- package/dist/providers/core/runtime/vercel-ai-sdk/anthropic-sdk-request-exec.js +2 -1
- package/dist/providers/core/runtime/vercel-ai-sdk/anthropic-sdk-request-exec.js.map +1 -1
- package/dist/providers/core/runtime/vercel-ai-sdk/openai-sdk-transport.js +2 -1
- package/dist/providers/core/runtime/vercel-ai-sdk/openai-sdk-transport.js.map +1 -1
- package/dist/providers/core/strategies/oauth-auth-code-flow.d.ts +1 -4
- package/dist/providers/core/strategies/oauth-auth-code-flow.js +26 -84
- package/dist/providers/core/strategies/oauth-auth-code-flow.js.map +1 -1
- package/dist/providers/core/strategies/oauth-device-flow.d.ts +2 -0
- package/dist/providers/core/strategies/oauth-device-flow.js +43 -8
- package/dist/providers/core/strategies/oauth-device-flow.js.map +1 -1
- package/dist/providers/core/utils/http-client.d.ts +7 -5
- package/dist/providers/core/utils/http-client.js +108 -33
- package/dist/providers/core/utils/http-client.js.map +1 -1
- package/dist/providers/core/utils/provider-error-reporter.d.ts +2 -2
- package/dist/providers/core/utils/provider-error-reporter.js +9 -85
- package/dist/providers/core/utils/provider-error-reporter.js.map +1 -1
- package/dist/providers/core/utils/provider-type-utils.js +1 -3
- package/dist/providers/core/utils/provider-type-utils.js.map +1 -1
- package/dist/providers/core/utils/qwen-client-fingerprint.d.ts +15 -0
- package/dist/providers/core/utils/qwen-client-fingerprint.js +47 -0
- package/dist/providers/core/utils/qwen-client-fingerprint.js.map +1 -0
- package/dist/providers/core/utils/snapshot-writer.d.ts +3 -0
- package/dist/providers/core/utils/snapshot-writer.js +385 -26
- package/dist/providers/core/utils/snapshot-writer.js.map +1 -1
- package/dist/providers/profile/families/qwen-profile.js +285 -22
- package/dist/providers/profile/families/qwen-profile.js.map +1 -1
- package/dist/providers/profile/profile-registry.js +0 -2
- package/dist/providers/profile/profile-registry.js.map +1 -1
- package/dist/providers/profile/provider-directory.js +0 -1
- package/dist/providers/profile/provider-directory.js.map +1 -1
- package/dist/providers/profile/provider-profile-loader.js +1 -1
- package/dist/providers/profile/provider-profile-loader.js.map +1 -1
- package/dist/server/handlers/chat-handler.js +6 -5
- package/dist/server/handlers/chat-handler.js.map +1 -1
- package/dist/server/handlers/config-admin-handler.js +44 -69
- package/dist/server/handlers/config-admin-handler.js.map +1 -1
- package/dist/server/handlers/handler-response-utils.js +190 -27
- package/dist/server/handlers/handler-response-utils.js.map +1 -1
- package/dist/server/handlers/handler-utils.d.ts +3 -0
- package/dist/server/handlers/handler-utils.js +72 -0
- package/dist/server/handlers/handler-utils.js.map +1 -1
- package/dist/server/handlers/images-handler.js +7 -7
- package/dist/server/handlers/images-handler.js.map +1 -1
- package/dist/server/handlers/messages-handler.js +6 -5
- package/dist/server/handlers/messages-handler.js.map +1 -1
- package/dist/server/handlers/responses-handler.js +32 -14
- package/dist/server/handlers/responses-handler.js.map +1 -1
- package/dist/server/handlers/sse-dispatcher.js +55 -13
- package/dist/server/handlers/sse-dispatcher.js.map +1 -1
- package/dist/server/handlers/types.d.ts +12 -0
- package/dist/server/runtime/http-server/daemon-admin/auth-handler.js +5 -2
- package/dist/server/runtime/http-server/daemon-admin/auth-handler.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/control-handler.js +62 -18
- package/dist/server/runtime/http-server/daemon-admin/control-handler.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/credentials-handler-utils.js +1 -1
- package/dist/server/runtime/http-server/daemon-admin/credentials-handler-utils.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/credentials-handler.js +9 -9
- package/dist/server/runtime/http-server/daemon-admin/credentials-handler.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/providers-handler-routing-utils.js +36 -10
- package/dist/server/runtime/http-server/daemon-admin/providers-handler-routing-utils.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/providers-handler.js +11 -11
- package/dist/server/runtime/http-server/daemon-admin/providers-handler.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin/quota-handler.js +49 -8
- package/dist/server/runtime/http-server/daemon-admin/quota-handler.js.map +1 -1
- package/dist/server/runtime/http-server/daemon-admin-routes.d.ts +1 -0
- package/dist/server/runtime/http-server/daemon-admin-routes.js +46 -0
- package/dist/server/runtime/http-server/daemon-admin-routes.js.map +1 -1
- package/dist/server/runtime/http-server/executor/client-injection-flow.js +2 -0
- package/dist/server/runtime/http-server/executor/client-injection-flow.js.map +1 -1
- package/dist/server/runtime/http-server/executor/log-rollup.d.ts +38 -0
- package/dist/server/runtime/http-server/executor/log-rollup.js +775 -0
- package/dist/server/runtime/http-server/executor/log-rollup.js.map +1 -0
- package/dist/server/runtime/http-server/executor/provider-response-converter.js +654 -281
- package/dist/server/runtime/http-server/executor/provider-response-converter.js.map +1 -1
- package/dist/server/runtime/http-server/executor/provider-response-utils.js +34 -3
- package/dist/server/runtime/http-server/executor/provider-response-utils.js.map +1 -1
- package/dist/server/runtime/http-server/executor/provider-runtime-resolver.js +14 -68
- package/dist/server/runtime/http-server/executor/provider-runtime-resolver.js.map +1 -1
- package/dist/server/runtime/http-server/executor/request-executor-core-utils.js +26 -2
- package/dist/server/runtime/http-server/executor/request-executor-core-utils.js.map +1 -1
- package/dist/server/runtime/http-server/executor/request-retry-helpers.js +20 -2
- package/dist/server/runtime/http-server/executor/request-retry-helpers.js.map +1 -1
- package/dist/server/runtime/http-server/executor/servertool-adapter-context.d.ts +10 -0
- package/dist/server/runtime/http-server/executor/servertool-adapter-context.js +120 -0
- package/dist/server/runtime/http-server/executor/servertool-adapter-context.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-dispatch.d.ts +35 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-dispatch.js +101 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-dispatch.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-error.d.ts +46 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-error.js +127 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-error.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-metadata.d.ts +7 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-metadata.js +186 -0
- package/dist/server/runtime/http-server/executor/servertool-followup-metadata.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-request-normalizer.d.ts +2 -0
- package/dist/server/runtime/http-server/executor/servertool-request-normalizer.js +56 -0
- package/dist/server/runtime/http-server/executor/servertool-request-normalizer.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-response-normalizer.d.ts +8 -0
- package/dist/server/runtime/http-server/executor/servertool-response-normalizer.js +31 -0
- package/dist/server/runtime/http-server/executor/servertool-response-normalizer.js.map +1 -0
- package/dist/server/runtime/http-server/executor/servertool-runtime-log.d.ts +8 -0
- package/dist/server/runtime/http-server/executor/servertool-runtime-log.js +33 -0
- package/dist/server/runtime/http-server/executor/servertool-runtime-log.js.map +1 -0
- package/dist/server/runtime/http-server/executor/sse-error-handler.js +37 -0
- package/dist/server/runtime/http-server/executor/sse-error-handler.js.map +1 -1
- package/dist/server/runtime/http-server/executor/usage-aggregator.js +32 -2
- package/dist/server/runtime/http-server/executor/usage-aggregator.js.map +1 -1
- package/dist/server/runtime/http-server/executor/usage-logger.d.ts +12 -0
- package/dist/server/runtime/http-server/executor/usage-logger.js +56 -1
- package/dist/server/runtime/http-server/executor/usage-logger.js.map +1 -1
- package/dist/server/runtime/http-server/executor-metadata.d.ts +15 -0
- package/dist/server/runtime/http-server/executor-metadata.js +60 -18
- package/dist/server/runtime/http-server/executor-metadata.js.map +1 -1
- package/dist/server/runtime/http-server/executor-provider.d.ts +2 -0
- package/dist/server/runtime/http-server/executor-provider.js +88 -205
- package/dist/server/runtime/http-server/executor-provider.js.map +1 -1
- package/dist/server/runtime/http-server/executor-response.js +30 -149
- package/dist/server/runtime/http-server/executor-response.js.map +1 -1
- package/dist/server/runtime/http-server/http-server-bootstrap.js +53 -17
- package/dist/server/runtime/http-server/http-server-bootstrap.js.map +1 -1
- package/dist/server/runtime/http-server/http-server-legacy-pipeline.js +28 -1
- package/dist/server/runtime/http-server/http-server-legacy-pipeline.js.map +1 -1
- package/dist/server/runtime/http-server/http-server-lifecycle.js +17 -4
- package/dist/server/runtime/http-server/http-server-lifecycle.js.map +1 -1
- package/dist/server/runtime/http-server/http-server-runtime-providers.js +42 -13
- package/dist/server/runtime/http-server/http-server-runtime-providers.js.map +1 -1
- package/dist/server/runtime/http-server/http-server-runtime-setup.js +40 -2
- package/dist/server/runtime/http-server/http-server-runtime-setup.js.map +1 -1
- package/dist/server/runtime/http-server/hub-shadow-compare.js +8 -4
- package/dist/server/runtime/http-server/hub-shadow-compare.js.map +1 -1
- package/dist/server/runtime/http-server/managed-process-probe.js +30 -4
- package/dist/server/runtime/http-server/managed-process-probe.js.map +1 -1
- package/dist/server/runtime/http-server/middleware.js +32 -4
- package/dist/server/runtime/http-server/middleware.js.map +1 -1
- package/dist/server/runtime/http-server/provider-traffic-governor.d.ts +45 -1
- package/dist/server/runtime/http-server/provider-traffic-governor.js +697 -111
- package/dist/server/runtime/http-server/provider-traffic-governor.js.map +1 -1
- package/dist/server/runtime/http-server/provider-utils.js +2 -6
- package/dist/server/runtime/http-server/provider-utils.js.map +1 -1
- package/dist/server/runtime/http-server/request-activity-tracker.d.ts +4 -0
- package/dist/server/runtime/http-server/request-activity-tracker.js +54 -11
- package/dist/server/runtime/http-server/request-activity-tracker.js.map +1 -1
- package/dist/server/runtime/http-server/request-executor.d.ts +247 -0
- package/dist/server/runtime/http-server/request-executor.js +2444 -334
- package/dist/server/runtime/http-server/request-executor.js.map +1 -1
- package/dist/server/runtime/http-server/routes.js +59 -16
- package/dist/server/runtime/http-server/routes.js.map +1 -1
- package/dist/server/runtime/http-server/runtime-manager.js +0 -15
- package/dist/server/runtime/http-server/runtime-manager.js.map +1 -1
- package/dist/server/runtime/http-server/session-client-registry-utils.js +46 -10
- package/dist/server/runtime/http-server/session-client-registry-utils.js.map +1 -1
- package/dist/server/runtime/http-server/session-client-registry.js +22 -4
- package/dist/server/runtime/http-server/session-client-registry.js.map +1 -1
- package/dist/server/runtime/http-server/session-client-routes.js +40 -18
- package/dist/server/runtime/http-server/session-client-routes.js.map +1 -1
- package/dist/server/runtime/http-server/session-dir.js +35 -2
- package/dist/server/runtime/http-server/session-dir.js.map +1 -1
- package/dist/server/runtime/http-server/session-scope-resolution.js +38 -9
- package/dist/server/runtime/http-server/session-scope-resolution.js.map +1 -1
- package/dist/server/runtime/http-server/session-storage-cleanup.js +64 -27
- package/dist/server/runtime/http-server/session-storage-cleanup.js.map +1 -1
- package/dist/server/runtime/http-server/stats-manager.d.ts +5 -0
- package/dist/server/runtime/http-server/stats-manager.js +138 -6
- package/dist/server/runtime/http-server/stats-manager.js.map +1 -1
- package/dist/server/runtime/http-server/tmux-injection-history.js +30 -4
- package/dist/server/runtime/http-server/tmux-injection-history.js.map +1 -1
- package/dist/server/runtime/http-server/tmux-session-probe.d.ts +3 -1
- package/dist/server/runtime/http-server/tmux-session-probe.js +198 -9
- package/dist/server/runtime/http-server/tmux-session-probe.js.map +1 -1
- package/dist/server/utils/client-connection-state.d.ts +1 -0
- package/dist/server/utils/client-connection-state.js +45 -5
- package/dist/server/utils/client-connection-state.js.map +1 -1
- package/dist/server/utils/finish-reason.js +61 -2
- package/dist/server/utils/finish-reason.js.map +1 -1
- package/dist/server/utils/http-error-mapper.d.ts +4 -0
- package/dist/server/utils/http-error-mapper.js +31 -6
- package/dist/server/utils/http-error-mapper.js.map +1 -1
- package/dist/server/utils/stage-logger.js +42 -13
- package/dist/server/utils/stage-logger.js.map +1 -1
- package/dist/server-lifecycle/port-utils.js +6 -2
- package/dist/server-lifecycle/port-utils.js.map +1 -1
- package/dist/token-daemon/index.js +44 -15
- package/dist/token-daemon/index.js.map +1 -1
- package/dist/token-daemon/leader-lock.js +65 -7
- package/dist/token-daemon/leader-lock.js.map +1 -1
- package/dist/token-daemon/provider-registry.js +1 -1
- package/dist/token-daemon/provider-registry.js.map +1 -1
- package/dist/token-daemon/server-utils.d.ts +11 -0
- package/dist/token-daemon/server-utils.js +71 -18
- package/dist/token-daemon/server-utils.js.map +1 -1
- package/dist/token-daemon/token-daemon.d.ts +0 -1
- package/dist/token-daemon/token-daemon.js +81 -63
- package/dist/token-daemon/token-daemon.js.map +1 -1
- package/dist/token-daemon/token-types.d.ts +1 -1
- package/dist/token-daemon/token-types.js +0 -1
- package/dist/token-daemon/token-types.js.map +1 -1
- package/dist/token-daemon/token-utils.js +4 -2
- package/dist/token-daemon/token-utils.js.map +1 -1
- package/dist/tools/provider-update/fetch-models.js +1 -5
- package/dist/tools/provider-update/fetch-models.js.map +1 -1
- package/dist/utils/error-handler-registry.js +50 -13
- package/dist/utils/error-handler-registry.js.map +1 -1
- package/dist/utils/errorsamples.d.ts +3 -1
- package/dist/utils/errorsamples.js +198 -14
- package/dist/utils/errorsamples.js.map +1 -1
- package/dist/utils/http-health-probe.d.ts +42 -0
- package/dist/utils/http-health-probe.js +231 -0
- package/dist/utils/http-health-probe.js.map +1 -0
- package/dist/utils/managed-server-pids.js +2 -2
- package/dist/utils/managed-server-pids.js.map +1 -1
- package/dist/utils/module-config-reader.js +11 -1
- package/dist/utils/module-config-reader.js.map +1 -1
- package/dist/utils/runtime-package-root.d.ts +2 -0
- package/dist/utils/runtime-package-root.js +47 -0
- package/dist/utils/runtime-package-root.js.map +1 -0
- package/dist/utils/snapshot-local-disk-gate.d.ts +3 -0
- package/dist/utils/snapshot-local-disk-gate.js +50 -0
- package/dist/utils/snapshot-local-disk-gate.js.map +1 -0
- package/dist/utils/snapshot-payload-guard.d.ts +1 -0
- package/dist/utils/snapshot-payload-guard.js +234 -0
- package/dist/utils/snapshot-payload-guard.js.map +1 -0
- package/dist/utils/snapshot-request-retention.d.ts +3 -0
- package/dist/utils/snapshot-request-retention.js +128 -0
- package/dist/utils/snapshot-request-retention.js.map +1 -0
- package/dist/utils/snapshot-stage-policy.d.ts +3 -0
- package/dist/utils/snapshot-stage-policy.js +111 -0
- package/dist/utils/snapshot-stage-policy.js.map +1 -0
- package/dist/utils/snapshot-writer.js +124 -127
- package/dist/utils/snapshot-writer.js.map +1 -1
- package/docs/CHAT_PROCESS_PROTOCOL_AND_PIPELINE.md +4 -0
- package/docs/CONFIG_ARCHITECTURE.md +2 -2
- package/docs/INSTALLATION_AND_QUICKSTART.md +5 -5
- package/docs/OAUTH.md +2 -5
- package/docs/PORTS.md +1 -1
- package/docs/PROVIDERS_BUILTIN.md +1 -5
- package/docs/PROVIDER_TYPES.md +1 -1
- package/docs/agent-routing/10-runtime-ssot-routing.md +4 -0
- package/docs/agent-routing/20-build-test-release-routing.md +2 -2
- package/docs/agent-routing/30-servertool-lifecycle-routing.md +2 -0
- package/docs/agent-routing/40-task-memory-routing.md +22 -2
- package/docs/audit/286.1-provider-failure-policy-audit.md +341 -0
- package/docs/audits/2026-04-26-fallback-silent-failure-audit.md +119 -0
- package/docs/audits/2026-04-27-provider-failure-policy-ssot-audit.md +153 -0
- package/docs/chat-process-continuation-state-contract.md +196 -0
- package/docs/chat-semantic-expansion-plan.md +2 -0
- package/docs/daemon-admin-api-design.md +9 -9
- package/docs/design/provider-failure-policy-ssot.md +215 -0
- package/docs/design/reasoning-stop-lifecycle.md +90 -0
- package/docs/design/servertool-unified-skeleton.md +202 -0
- package/docs/design/websearch-servertool-orchestration.md +190 -0
- package/docs/error-handling-v2.md +120 -11
- package/docs/protocol-compatibility-matrix.md +147 -0
- package/docs/providers/provider-composite-design.md +4 -5
- package/docs/providers/provider-composite-testing.md +1 -1
- package/docs/providers/provider-type-only-migration.md +2 -15
- package/docs/refactoring/compatibility-v2-architecture-design.md +1 -3
- package/docs/refactoring/host-164.3-responsibility-migration.md +0 -1
- package/docs/routing-instructions.md +42 -1
- package/docs/stop-message-auto.md +0 -1
- package/docs/token-refresh-daemon-plan.md +14 -14
- package/docs/v2-architecture/IMPLEMENTATION-ROADMAP.md +1 -1
- package/docs/v2-architecture/PROVIDER-V2-CHANGESET-RELEASE-CHECKLIST.md +1 -9
- package/docs/v2-architecture/PROVIDER-V2-LAYERING-ADR-DRAFT.md +3 -4
- package/docs/v2-architecture/PROVIDER-V2-MIGRATION-MATRIX-DRAFT.md +4 -15
- package/docs/v2-architecture/PROVIDER-V2-PHASED-MIGRATION-ROLLBACK-DRAFT.md +4 -5
- package/docs/v2-architecture/PROVIDER-V2-PROFILE-API-REGISTRY-DRAFT.md +4 -4
- package/docs/v2-architecture/PROVIDER-V2-REFACTOR-OVERVIEW-DRAFT.md +1 -2
- package/docs/v2-architecture/PROVIDER-V2-VERIFICATION-MATRIX-DRAFT.md +2 -2
- package/node_modules/@jsonstudio/llms/dist/config-unified/unified-config.js +36 -10
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/deepseek-web-request.js +7 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/deepseek-web-response.js +2 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/antigravity-session-signature.js +33 -17
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/header-policies.d.ts +20 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/header-policies.js +79 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/policy-overrides.d.ts +16 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/policy-overrides.js +27 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/provider-resolver.d.ts +26 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/provider-resolver.js +59 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/registry.d.ts +35 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/registry.js +154 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/types.d.ts +75 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profile-registry/types.js +8 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/anthropic-claude-code.json +13 -7
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/chat-claude-code.json +13 -8
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/chat-deepseek-web.json +37 -8
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/chat-qwen.json +48 -11
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/compat-passthrough.json +6 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/provider-resolution-config.json +24 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/anthropic-mapper-config.js +0 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/anthropic-mapper-from-chat.js +54 -7
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/anthropic-mapper-to-chat.js +15 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/anthropic-semantics-audit.d.ts +11 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/anthropic-semantics-audit.js +16 -30
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/gemini-mapper-config.js +0 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/gemini-mapper-from-chat.js +15 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/gemini-mapping-audit.d.ts +10 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/gemini-mapping-audit.js +9 -30
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/protocol-mapping-audit.d.ts +43 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/protocol-mapping-audit.js +148 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper-from-chat.js +16 -6
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper-helpers.d.ts +2 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper-helpers.js +65 -3
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper-to-chat.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/operation-table/semantic-mappers/responses-mapper.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/compat/compat-types.d.ts +14 -154
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-chat-process-request-utils.d.ts +6 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-chat-process-request-utils.js +83 -3
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-execute-chat-process-entry.js +6 -5
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-execute-request-stage-inbound.d.ts +0 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-execute-request-stage-inbound.js +9 -5
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-execute-request-stage-provider-payload.js +132 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-execute-request-stage.js +0 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-heavy-input-fastpath.d.ts +4 -22
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-heavy-input-fastpath.js +109 -139
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-max-tokens-policy.js +40 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-route-and-outbound.d.ts +0 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline-route-and-outbound.js +19 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline.d.ts +0 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/hub-pipeline.js +45 -43
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/route-aware-responses-continuation.d.ts +10 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/route-aware-responses-continuation.js +143 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/req_inbound/req_inbound_stage2_semantic_map/index.js +37 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/req_inbound/req_inbound_stage2_semantic_map/semantic-lift.js +14 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/req_inbound/req_inbound_stage3_context_capture/responses-context-snapshot.d.ts +3 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/req_inbound/req_inbound_stage3_context_capture/responses-context-snapshot.js +22 -11
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_outbound/resp_outbound_stage1_client_remap/chat-process-semantics-bridge.js +59 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_outbound/resp_outbound_stage1_client_remap/client-remap-protocol-switch.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_outbound/resp_outbound_stage1_client_remap/client-remap-protocol-switch.js +3 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_outbound/resp_outbound_stage1_client_remap/index.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_outbound/resp_outbound_stage1_client_remap/index.js +2 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_process/resp_process_stage1_tool_governance/index.d.ts +2 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_process/resp_process_stage1_tool_governance/index.js +53 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_process/resp_process_stage2_finalize/index.js +4 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_process/resp_process_stage3_servertool_orchestration/index.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/pipeline/stages/resp_process/resp_process_stage3_servertool_orchestration/index.js +16 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/policy/policy-engine.js +14 -4
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/process/chat-process-anthropic-alias.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/process/chat-process-anthropic-alias.js +29 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/process/chat-process-pending-tool-sync.js +38 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/process/chat-process-request-sanitizer.js +164 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/response/provider-response.js +27 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/response/response-mappers.d.ts +9 -3
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/response/response-mappers.js +8 -7
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/response/response-runtime-anthropic.js +20 -5
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/snapshot-recorder.js +1 -247
- package/node_modules/@jsonstudio/llms/dist/conversion/hub/types/chat-envelope.d.ts +84 -4
- package/node_modules/@jsonstudio/llms/dist/conversion/pipeline/codecs/v2/responses-openai-pipeline.js +26 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/responses/responses-openai-bridge/response-payload.js +10 -9
- package/node_modules/@jsonstudio/llms/dist/conversion/responses/responses-openai-bridge/utils.js +20 -5
- package/node_modules/@jsonstudio/llms/dist/conversion/responses/responses-openai-bridge.js +88 -4
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/anthropic-message-utils-core.js +5 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/anthropic-message-utils-openai-request.js +55 -14
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/anthropic-message-utils-tool-schema.js +1 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/anthropic-message-utils.js +38 -15
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/openai-message-normalize.js +45 -6
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/responses-conversation-store.d.ts +15 -0
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/responses-conversation-store.js +157 -13
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/responses-reasoning-registry.d.ts +8 -2
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/responses-reasoning-registry.js +103 -9
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/responses-response-utils.js +10 -10
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/tool-filter-pipeline.js +59 -9
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/tool-governor-guards.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/tool-governor-guards.js +54 -43
- package/node_modules/@jsonstudio/llms/dist/conversion/shared/tool-governor.js +62 -8
- package/node_modules/@jsonstudio/llms/dist/conversion/snapshot-utils.js +121 -8
- package/node_modules/@jsonstudio/llms/dist/filters/engine.js +64 -6
- package/node_modules/@jsonstudio/llms/dist/filters/special/request-tool-list-filter.js +32 -11
- package/node_modules/@jsonstudio/llms/dist/filters/special/request-toolcalls-stringify.js +26 -1
- package/node_modules/@jsonstudio/llms/dist/filters/special/request-tools-normalize.js +46 -7
- package/node_modules/@jsonstudio/llms/dist/filters/special/response-tool-arguments-blacklist.js +10 -1
- package/node_modules/@jsonstudio/llms/dist/filters/special/response-tool-arguments-schema-converge.js +10 -1
- package/node_modules/@jsonstudio/llms/dist/filters/special/response-tool-arguments-stringify.js +30 -5
- package/node_modules/@jsonstudio/llms/dist/filters/special/response-tool-arguments-whitelist.js +10 -1
- package/node_modules/@jsonstudio/llms/dist/filters/special/tool-filter-hooks.js +16 -14
- package/node_modules/@jsonstudio/llms/dist/filters/utils/snapshot-writer.js +47 -5
- package/node_modules/@jsonstudio/llms/dist/guidance/index.js +16 -2
- package/node_modules/@jsonstudio/llms/dist/http/sse-response.js +42 -6
- package/node_modules/@jsonstudio/llms/dist/native/router_hotpath_napi.node +0 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/auth-utils.js +33 -31
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/provider-normalization.d.ts +6 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/provider-normalization.js +78 -98
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/responses-helpers.js +22 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/routing-config.js +56 -3
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap/streaming-helpers.js +19 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/bootstrap.js +62 -293
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/classifier.js +11 -82
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/antigravity/alias-lease.js +42 -15
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/cooldown-manager.js +55 -3
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/health/index.js +66 -15
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/routing-pools/index.js +44 -8
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/routing-state/keys.d.ts +7 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/routing-state/keys.js +65 -21
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/routing-state/store.js +19 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/sticky-session-manager.d.ts +9 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine/sticky-session-manager.js +139 -4
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-logging.js +2 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-chat-process-clock-reminder-semantics.js +68 -59
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-chat-process-governance-semantics.js +156 -157
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-chat-process-node-result-semantics.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-chat-process-node-result-semantics.js +31 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-chat-process-servertool-orchestration-semantics.js +94 -95
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-compat-action-semantics.d.ts +0 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-compat-action-semantics.js +0 -7
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-action-semantics-parsers.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-action-semantics-parsers.js +203 -305
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-action-semantics-tools-post.js +35 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-action-semantics-tools-request.js +2 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-action-semantics-types.d.ts +2 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-bridge-policy-semantics.js +173 -154
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-edge-stage-semantics.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-edge-stage-semantics.js +106 -57
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-inbound-outbound-semantics.js +122 -114
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-orchestration-semantics-builders.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-orchestration-semantics-metadata-policy.js +100 -86
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-orchestration-semantics-passthrough.js +52 -33
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-orchestration-semantics-protocol.js +79 -62
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-orchestration-semantics.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-req-inbound-semantics-parsers.js +50 -33
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-req-inbound-semantics-types.d.ts +2 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-req-outbound-semantics-parsers.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-req-outbound-semantics-parsers.js +161 -177
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-resp-semantics-outbound-tools.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-resp-semantics-outbound-tools.js +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-resp-semantics-parsers.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-hub-pipeline-resp-semantics-parsers.js +231 -286
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-router-hotpath-analysis.js +143 -164
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-router-hotpath-required-exports.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-router-hotpath-required-exports.js +8 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-shared-conversion-semantics-reasoning.js +57 -28
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-shared-conversion-semantics-responses.d.ts +8 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-shared-conversion-semantics-responses.js +76 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-shared-conversion-semantics.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-shared-conversion-semantics.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-virtual-router-bootstrap-providers.d.ts +24 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-virtual-router-bootstrap-providers.js +78 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-virtual-router-bootstrap-routing.d.ts +17 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/native-virtual-router-bootstrap-routing.js +72 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/route-utils.js +9 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-selection/tier-selection-select.js +11 -5
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine.js +4 -39
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/features.js +199 -31
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/health-manager.js +7 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/load-balancer.d.ts +3 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/load-balancer.js +47 -3
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/native-error.d.ts +11 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/native-error.js +71 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/provider-registry.js +81 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/provider-runtime-ingress.d.ts +24 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/provider-runtime-ingress.js +139 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/routing-instructions/parse.js +21 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/routing-instructions/state.js +46 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/routing-instructions/types.d.ts +7 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/routing-stop-message-state-codec.js +30 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/sticky-session-store.js +44 -15
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/stop-message-state-sync.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/stop-message-state-sync.js +10 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/success-center.js +39 -2
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/tool-signals.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/tool-signals.js +25 -24
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/types.d.ts +3 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/engine.js +255 -71
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/clock.js +20 -2
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/followup-request-builder.js +120 -85
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/memory-appender.d.ts +6 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/memory-appender.js +42 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/reasoning-only-continue.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/reasoning-stop-guard.js +630 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/reasoning-stop-state.d.ts +24 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/reasoning-stop-state.js +606 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/reasoning-stop.js +292 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/{iflow-followup.d.ts → ai-followup.d.ts} +3 -3
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/{iflow-followup.js → ai-followup.js} +76 -81
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/config.d.ts +1 -2
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/config.js +2 -6
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/runtime-utils.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto/runtime-utils.js +51 -11
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/stop-message-auto.js +5 -4
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/web-search-auto-trigger.d.ts +9 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/web-search-auto-trigger.js +91 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/web-search.js +1 -131
- package/node_modules/@jsonstudio/llms/dist/servertool/heartbeat/session-store.js +49 -18
- package/node_modules/@jsonstudio/llms/dist/servertool/registry.js +3 -0
- package/node_modules/@jsonstudio/llms/dist/servertool/server-side-tools.d.ts +2 -2
- package/node_modules/@jsonstudio/llms/dist/servertool/server-side-tools.js +14 -10
- package/node_modules/@jsonstudio/llms/dist/servertool/strip-servertool-calls.js +2 -3
- package/node_modules/@jsonstudio/llms/dist/servertool/types.d.ts +11 -1
- package/node_modules/@jsonstudio/llms/dist/sse/json-to-sse/chat-json-to-sse-converter.js +26 -1
- package/node_modules/@jsonstudio/llms/dist/sse/json-to-sse/event-generators/chat.d.ts +2 -2
- package/node_modules/@jsonstudio/llms/dist/sse/json-to-sse/event-generators/chat.js +38 -3
- package/node_modules/@jsonstudio/llms/dist/sse/json-to-sse/sequencers/chat-sequencer.js +6 -3
- package/node_modules/@jsonstudio/llms/dist/sse/shared/chat-serializer.js +27 -3
- package/node_modules/@jsonstudio/llms/dist/sse/shared/constants.d.ts +6 -6
- package/node_modules/@jsonstudio/llms/dist/sse/shared/constants.js +3 -3
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/anthropic-sse-to-json-converter.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/anthropic-sse-to-json-converter.js +36 -1
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/builders/anthropic-response-builder.js +6 -20
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/builders/response-builder.d.ts +3 -0
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/builders/response-builder.js +125 -42
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/chat-sse-to-json-converter.js +59 -10
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/parsers/sse-parser.js +2 -0
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/responses-sse-to-json-converter.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/sse/sse-to-json/responses-sse-to-json-converter.js +71 -2
- package/node_modules/@jsonstudio/llms/dist/sse/types/chat-types.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/sse/types/chat-types.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/sse/types/conversion-context.js +2 -2
- package/node_modules/@jsonstudio/llms/dist/sse/types/index.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/sse/types/responses-types.d.ts +1 -1
- package/node_modules/@jsonstudio/llms/dist/sse/types/responses-types.js +1 -1
- package/node_modules/@jsonstudio/llms/dist/telemetry/stats-center.js +34 -12
- package/node_modules/@jsonstudio/llms/dist/tools/apply-patch/patch-text/fuzzy-match.d.ts +14 -0
- package/node_modules/@jsonstudio/llms/dist/tools/apply-patch/patch-text/fuzzy-match.js +174 -0
- package/node_modules/@jsonstudio/llms/dist/tools/apply-patch/patch-text/normalize.js +148 -0
- package/node_modules/@jsonstudio/llms/dist/tools/apply-patch/structured.js +31 -2
- package/node_modules/@jsonstudio/llms/dist/tools/apply-patch/validator.js +121 -4
- package/node_modules/@jsonstudio/llms/dist/tools/exec-command/normalize.d.ts +4 -1
- package/node_modules/@jsonstudio/llms/dist/tools/exec-command/normalize.js +31 -16
- package/node_modules/@jsonstudio/llms/dist/tools/exec-command/validator.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/tools/exec-command/validator.js +3 -1
- package/node_modules/@jsonstudio/llms/dist/tools/tool-registry.d.ts +1 -0
- package/node_modules/@jsonstudio/llms/dist/tools/tool-registry.js +5 -27
- package/node_modules/@jsonstudio/llms/package.json +1 -1
- package/package.json +6 -7
- package/scripts/ci/repo-sanity.mjs +1 -0
- package/scripts/ci/silent-failure-audit.mjs +112 -70
- package/scripts/cleanup-stale-server-pids.mjs +0 -6
- package/scripts/ensure-cli-command-shim.mjs +49 -15
- package/scripts/ensure-cli-executable.mjs +1 -1
- package/scripts/install-global.sh +7 -6
- package/scripts/install-release-snapshot.mjs +232 -0
- package/scripts/install-release.sh +30 -24
- package/scripts/link-global-llms-local.mjs +1 -2
- package/scripts/pack-mode.mjs +0 -4
- package/scripts/pack-rcc.mjs +17 -58
- package/scripts/run-bg.sh +0 -6
- package/scripts/run-fg-gtimeout.sh +0 -6
- package/scripts/tests/antigravity-codex-sample-pipeline-compare.mjs +1 -1
- package/scripts/tests/blackbox-rcc-vs-routecodex-antigravity.mjs +60 -39
- package/scripts/verify-install-e2e.mjs +64 -26
- package/configsamples/provider-default/qwenchat/config.v2.json +0 -53
- package/dist/providers/auth/iflow-cookie-auth.d.ts +0 -27
- package/dist/providers/auth/iflow-cookie-auth.js +0 -207
- package/dist/providers/auth/iflow-cookie-auth.js.map +0 -1
- package/dist/providers/auth/iflow-userinfo-helper.d.ts +0 -32
- package/dist/providers/auth/iflow-userinfo-helper.js +0 -81
- package/dist/providers/auth/iflow-userinfo-helper.js.map +0 -1
- package/dist/providers/core/runtime/iflow-http-provider.d.ts +0 -13
- package/dist/providers/core/runtime/iflow-http-provider.js +0 -22
- package/dist/providers/core/runtime/iflow-http-provider.js.map +0 -1
- package/dist/providers/core/runtime/provider-iflow-business-error-utils.d.ts +0 -15
- package/dist/providers/core/runtime/provider-iflow-business-error-utils.js +0 -49
- package/dist/providers/core/runtime/provider-iflow-business-error-utils.js.map +0 -1
- package/dist/providers/core/runtime/qwenchat-http-provider-helpers.d.ts +0 -89
- package/dist/providers/core/runtime/qwenchat-http-provider-helpers.js +0 -1698
- package/dist/providers/core/runtime/qwenchat-http-provider-helpers.js.map +0 -1
- package/dist/providers/core/runtime/qwenchat-http-provider.d.ts +0 -9
- package/dist/providers/core/runtime/qwenchat-http-provider.js +0 -78
- package/dist/providers/core/runtime/qwenchat-http-provider.js.map +0 -1
- package/dist/providers/core/runtime/standard-tool-text-request-transform.d.ts +0 -13
- package/dist/providers/core/runtime/standard-tool-text-request-transform.js +0 -138
- package/dist/providers/core/runtime/standard-tool-text-request-transform.js.map +0 -1
- package/dist/providers/core/runtime/transport/iflow-signer.d.ts +0 -12
- package/dist/providers/core/runtime/transport/iflow-signer.js +0 -63
- package/dist/providers/core/runtime/transport/iflow-signer.js.map +0 -1
- package/dist/providers/profile/families/iflow-profile.d.ts +0 -2
- package/dist/providers/profile/families/iflow-profile.js +0 -384
- package/dist/providers/profile/families/iflow-profile.js.map +0 -1
- package/docs/mapping-tables/iflow-to-openai.json +0 -215
- package/docs/mapping-tables/openai-to-iflow.json +0 -227
- package/docs/multi-token-auth-guide.md +0 -66
- package/docs/oauth-authentication-guide.md +0 -172
- package/docs/oauth-iflow-implementation.md +0 -157
- package/docs/release-iflow-400-gate.md +0 -58
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-cli-defaults.d.ts +0 -10
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-cli-defaults.js +0 -31
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-history-media-placeholder.d.ts +0 -7
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-history-media-placeholder.js +0 -25
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-thinking-reasoning-fill.d.ts +0 -12
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-kimi-thinking-reasoning-fill.js +0 -46
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-native-compat.d.ts +0 -6
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-native-compat.js +0 -36
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-response-body-unwrap.d.ts +0 -9
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-response-body-unwrap.js +0 -25
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-tool-text-fallback.d.ts +0 -5
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-tool-text-fallback.js +0 -29
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-web-search.d.ts +0 -18
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/iflow-web-search.js +0 -49
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/qwenchat-web-request.d.ts +0 -3
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/actions/qwenchat-web-request.js +0 -62
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/chat-iflow.json +0 -353
- package/node_modules/@jsonstudio/llms/dist/conversion/compat/profiles/chat-qwenchat-web.json +0 -47
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-health.d.ts +0 -1
- package/node_modules/@jsonstudio/llms/dist/router/virtual-router/engine-health.js +0 -1
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/iflow-model-error-retry.js +0 -92
- package/node_modules/@jsonstudio/llms/dist/servertool/handlers/review.js +0 -202
- package/scripts/publish-rcc.mjs +0 -81
- /package/node_modules/@jsonstudio/llms/dist/servertool/handlers/{iflow-model-error-retry.d.ts → reasoning-stop-guard.d.ts} +0 -0
- /package/node_modules/@jsonstudio/llms/dist/servertool/handlers/{review.d.ts → reasoning-stop.d.ts} +0 -0
|
@@ -1,22 +1,108 @@
|
|
|
1
1
|
import { attachProviderRuntimeMetadata } from '../../../providers/core/runtime/provider-runtime-metadata.js';
|
|
2
|
+
import { computeProviderFailureBackoffDelayMs, describeProviderFailureDecision, isBlockingRecoverableProviderFailure, resolveProviderFailureExclusionDecision, isProviderFailureHealthNeutral, normalizeProviderFailureCodeKey, resolveProviderFailureRetryEligibility, resolveProviderFailureClassification, resolveProviderFailureActionPlan } from '../../../providers/core/runtime/provider-failure-policy.js';
|
|
2
3
|
import { buildRequestMetadata, cloneClientHeaders, decorateMetadataForAttempt, ensureClientHeadersOnPayload, resolveClientRequestId } from './executor-metadata.js';
|
|
4
|
+
import { loadRoutingInstructionStateSync, rebindResponsesConversationRequestId } from '../../../modules/llmswitch/bridge.js';
|
|
3
5
|
import { convertProviderResponseIfNeeded as convertProviderResponseWithBridge } from './executor/provider-response-converter.js';
|
|
4
6
|
import { ensureHubPipeline, runHubPipeline } from './executor-pipeline.js';
|
|
5
7
|
// Import from new executor submodules
|
|
6
8
|
import { isVerboseErrorLoggingEnabled } from './executor/env-config.js';
|
|
7
|
-
import { resolveMaxProviderAttempts, describeRetryReason, isPromptTooLongError,
|
|
9
|
+
import { resolveMaxProviderAttempts, describeRetryReason, isPromptTooLongError, waitBeforeRetry } from './executor/retry-engine.js';
|
|
10
|
+
import { isClientDisconnectAbortError } from './executor-provider.js';
|
|
8
11
|
import {} from './executor/sse-error-handler.js';
|
|
9
12
|
import { extractUsageFromResult, mergeUsageMetrics } from './executor/usage-aggregator.js';
|
|
10
13
|
import { bindSessionConversationSession, extractRetryErrorSignature, extractStatusCodeFromError, injectAntigravityRetrySignal, isAntigravityProviderKey, isAntigravityReauthRequired403, isGoogleAccountVerificationRequiredError, isSseDecodeRetryableNetworkError, isSseDecodeRateLimitError, resolveAntigravityMaxProviderAttempts, shouldRotateAntigravityAliasOnRetry } from './executor/request-retry-helpers.js';
|
|
11
|
-
import {
|
|
14
|
+
import { extractProviderModel, extractResponseStatus, normalizeProviderResponse, resolveRequestSemantics } from './executor/provider-response-utils.js';
|
|
12
15
|
import { isPoolExhaustedPipelineError, mergeMetadataPreservingDefined, resolvePoolCooldownWaitMs, writeInboundClientSnapshot } from './executor/request-executor-core-utils.js';
|
|
13
16
|
import { resolveProviderRuntimeOrThrow } from './executor/provider-runtime-resolver.js';
|
|
14
17
|
import { resolveProviderRequestContext } from './executor/provider-request-context.js';
|
|
15
18
|
import { isServerToolEnabled } from './servertool-admin-state.js';
|
|
16
|
-
import { registerRequestLogContext
|
|
17
|
-
import {
|
|
19
|
+
import { registerRequestLogContext } from '../../utils/request-log-color.js';
|
|
20
|
+
import { getClientConnectionAbortSignal } from '../../utils/client-connection-state.js';
|
|
21
|
+
import { deriveFinishReason, STREAM_LOG_FINISH_REASON_KEY } from '../../utils/finish-reason.js';
|
|
22
|
+
import { allowSnapshotLocalDiskWrite } from '../../../utils/snapshot-local-disk-gate.js';
|
|
23
|
+
import { writeProviderSnapshot } from '../../../providers/core/utils/snapshot-writer.js';
|
|
24
|
+
import { REASONING_STOP_FINALIZED_FLAG_KEY, REASONING_STOP_FINALIZED_MARKER } from './executor/servertool-response-normalizer.js';
|
|
18
25
|
import { createNoopProviderTrafficGovernor, getSharedProviderTrafficGovernor } from './provider-traffic-governor.js';
|
|
26
|
+
import { recordVirtualRouterHitRollup } from './executor/log-rollup.js';
|
|
19
27
|
const DEFAULT_MAX_PROVIDER_ATTEMPTS = 6;
|
|
28
|
+
const NON_BLOCKING_LOG_THROTTLE_MS = 60_000;
|
|
29
|
+
const nonBlockingLogState = new Map();
|
|
30
|
+
const requestDegradedLogState = new Set();
|
|
31
|
+
const RECOVERABLE_BACKOFF_TTL_MS = 5 * 60_000;
|
|
32
|
+
const recoverableErrorBackoffState = new Map();
|
|
33
|
+
const recoverableRetryGateState = new Map();
|
|
34
|
+
const recoverableRetryWaiterState = new Map();
|
|
35
|
+
const providerTransportBackoffState = new Map();
|
|
36
|
+
const providerTransportBackoffGateState = new Map();
|
|
37
|
+
const SESSION_STORM_BACKOFF_TTL_MS = 10 * 60_000;
|
|
38
|
+
const sessionStormBackoffState = new Map();
|
|
39
|
+
const sessionStormBackoffGateState = new Map();
|
|
40
|
+
const logicalChainRetryState = new Map();
|
|
41
|
+
const PROVIDER_SWITCH_LOG_THROTTLE_MS = 5_000;
|
|
42
|
+
const providerSwitchLogState = new Map();
|
|
43
|
+
const RETRY_SNAPSHOT_PARSE_MAX_CHARS = 256 * 1024;
|
|
44
|
+
const RETRY_SNAPSHOT_RESTORE_MAX_CHARS = 2 * 1024 * 1024;
|
|
45
|
+
const RETRY_SNAPSHOT_SERIALIZE_MAX_CHARS = 256 * 1024;
|
|
46
|
+
const RETRY_PAYLOAD_ESTIMATE_MAX_BYTES = RETRY_SNAPSHOT_SERIALIZE_MAX_CHARS * 2;
|
|
47
|
+
const RETRY_PAYLOAD_ESTIMATE_NODE_BUDGET = 4000;
|
|
48
|
+
const MAX_CONTEXT_OVERFLOW_RETRIES = 3;
|
|
49
|
+
function formatUnknownError(error) {
|
|
50
|
+
if (error instanceof Error) {
|
|
51
|
+
return error.stack || `${error.name}: ${error.message}`;
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
return JSON.stringify(error);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return String(error);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function cloneErrorForReporting(error) {
|
|
61
|
+
if (!error || typeof error !== 'object') {
|
|
62
|
+
return error;
|
|
63
|
+
}
|
|
64
|
+
if (error instanceof Error) {
|
|
65
|
+
const cloned = new Error(error.message);
|
|
66
|
+
cloned.name = error.name;
|
|
67
|
+
if (typeof error.stack === 'string') {
|
|
68
|
+
cloned.stack = error.stack;
|
|
69
|
+
}
|
|
70
|
+
return Object.assign(cloned, error);
|
|
71
|
+
}
|
|
72
|
+
if (Array.isArray(error)) {
|
|
73
|
+
return [...error];
|
|
74
|
+
}
|
|
75
|
+
return { ...error };
|
|
76
|
+
}
|
|
77
|
+
function logRequestExecutorNonBlockingError(stage, error, details) {
|
|
78
|
+
const now = Date.now();
|
|
79
|
+
const last = nonBlockingLogState.get(stage) ?? 0;
|
|
80
|
+
if (now - last < NON_BLOCKING_LOG_THROTTLE_MS) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
nonBlockingLogState.set(stage, now);
|
|
84
|
+
try {
|
|
85
|
+
const detailSuffix = details && Object.keys(details).length > 0 ? ` details=${JSON.stringify(details)}` : '';
|
|
86
|
+
console.warn(`[request-executor] ${stage} failed (non-blocking): ${formatUnknownError(error)}${detailSuffix}`);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// Never throw from non-blocking logging.
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function logRequestExecutorDegraded(stage, requestId, details) {
|
|
93
|
+
const key = `${requestId}:${stage}`;
|
|
94
|
+
if (requestDegradedLogState.has(key)) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
requestDegradedLogState.add(key);
|
|
98
|
+
try {
|
|
99
|
+
const detailSuffix = details && Object.keys(details).length > 0 ? ` details=${JSON.stringify(details)}` : '';
|
|
100
|
+
console.warn(`[request-executor][degraded] req=${requestId} stage=${stage}${detailSuffix}`);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// Never throw from degraded logging.
|
|
104
|
+
}
|
|
105
|
+
}
|
|
20
106
|
function readString(value) {
|
|
21
107
|
if (typeof value !== 'string') {
|
|
22
108
|
return undefined;
|
|
@@ -24,6 +110,55 @@ function readString(value) {
|
|
|
24
110
|
const normalized = value.trim();
|
|
25
111
|
return normalized || undefined;
|
|
26
112
|
}
|
|
113
|
+
function normalizeStoplessLogMode(value) {
|
|
114
|
+
if (typeof value !== 'string') {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
const normalized = value.trim().toLowerCase();
|
|
118
|
+
if (normalized === 'on' || normalized === 'off' || normalized === 'endless') {
|
|
119
|
+
return normalized;
|
|
120
|
+
}
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
123
|
+
function readPersistedStoplessLogState(stickyKey) {
|
|
124
|
+
if (!stickyKey) {
|
|
125
|
+
return {};
|
|
126
|
+
}
|
|
127
|
+
const state = loadRoutingInstructionStateSync(stickyKey);
|
|
128
|
+
if (!state || typeof state !== 'object' || Array.isArray(state)) {
|
|
129
|
+
return {};
|
|
130
|
+
}
|
|
131
|
+
const record = state;
|
|
132
|
+
const mode = normalizeStoplessLogMode(record.reasoningStopMode);
|
|
133
|
+
return {
|
|
134
|
+
...(mode ? { mode } : {}),
|
|
135
|
+
...(typeof record.reasoningStopArmed === 'boolean'
|
|
136
|
+
? { armed: record.reasoningStopArmed }
|
|
137
|
+
: {})
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function resolveStoplessLogState(metadata) {
|
|
141
|
+
const sessionId = readString(metadata.sessionId);
|
|
142
|
+
const conversationId = readString(metadata.conversationId);
|
|
143
|
+
const directMode = normalizeStoplessLogMode(metadata.reasoningStopMode)
|
|
144
|
+
?? normalizeStoplessLogMode(metadata.stoplessMode);
|
|
145
|
+
const directArmed = typeof metadata.reasoningStopArmed === 'boolean'
|
|
146
|
+
? metadata.reasoningStopArmed
|
|
147
|
+
: (typeof metadata.stoplessArmed === 'boolean' ? metadata.stoplessArmed : undefined);
|
|
148
|
+
const persistedBySession = readPersistedStoplessLogState(sessionId ? `session:${sessionId}` : '');
|
|
149
|
+
const persistedByConversation = readPersistedStoplessLogState(conversationId ? `conversation:${conversationId}` : '');
|
|
150
|
+
const mode = directMode ??
|
|
151
|
+
persistedBySession.mode ??
|
|
152
|
+
persistedByConversation.mode;
|
|
153
|
+
if (!mode) {
|
|
154
|
+
return {};
|
|
155
|
+
}
|
|
156
|
+
const armed = directArmed ??
|
|
157
|
+
persistedBySession.armed ??
|
|
158
|
+
persistedByConversation.armed ??
|
|
159
|
+
false;
|
|
160
|
+
return { mode, armed };
|
|
161
|
+
}
|
|
27
162
|
function readStatusCodeCandidate(value) {
|
|
28
163
|
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
29
164
|
return value;
|
|
@@ -40,10 +175,26 @@ function readStatusCodeCandidate(value) {
|
|
|
40
175
|
return undefined;
|
|
41
176
|
}
|
|
42
177
|
function parseJsonRecordFromText(text) {
|
|
178
|
+
if (typeof text !== 'string' || !text) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
if (text.length > RETRY_SNAPSHOT_PARSE_MAX_CHARS) {
|
|
182
|
+
logRequestExecutorNonBlockingError('parseJsonRecordFromText.oversized_skip', new Error('candidate text too large'), { candidateLength: text.length, maxChars: RETRY_SNAPSHOT_PARSE_MAX_CHARS });
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
43
185
|
const normalized = text.trim();
|
|
44
186
|
if (!normalized) {
|
|
45
187
|
return null;
|
|
46
188
|
}
|
|
189
|
+
const shouldLogParseFailure = (candidate) => {
|
|
190
|
+
const trimmed = candidate.trimStart();
|
|
191
|
+
if (trimmed.startsWith('{')) {
|
|
192
|
+
return true;
|
|
193
|
+
}
|
|
194
|
+
// Only treat as JSON array candidate when '[' is followed by JSON-looking payload,
|
|
195
|
+
// otherwise strings like "[servertool] xxx" should not trigger JSON parse noise.
|
|
196
|
+
return /^\[\s*[\[{"]/u.test(trimmed);
|
|
197
|
+
};
|
|
47
198
|
const parseCandidate = (candidate) => {
|
|
48
199
|
try {
|
|
49
200
|
const parsed = JSON.parse(candidate);
|
|
@@ -51,14 +202,21 @@ function parseJsonRecordFromText(text) {
|
|
|
51
202
|
return parsed;
|
|
52
203
|
}
|
|
53
204
|
}
|
|
54
|
-
catch {
|
|
205
|
+
catch (error) {
|
|
206
|
+
if (shouldLogParseFailure(candidate)) {
|
|
207
|
+
logRequestExecutorNonBlockingError('parseJsonRecordFromText.parseCandidate', error, {
|
|
208
|
+
candidateLength: candidate.length
|
|
209
|
+
});
|
|
210
|
+
}
|
|
55
211
|
return null;
|
|
56
212
|
}
|
|
57
213
|
return null;
|
|
58
214
|
};
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
215
|
+
if (shouldLogParseFailure(normalized)) {
|
|
216
|
+
const direct = parseCandidate(normalized);
|
|
217
|
+
if (direct) {
|
|
218
|
+
return direct;
|
|
219
|
+
}
|
|
62
220
|
}
|
|
63
221
|
const firstBrace = normalized.indexOf('{');
|
|
64
222
|
const lastBrace = normalized.lastIndexOf('}');
|
|
@@ -208,12 +366,1343 @@ function readHubStageTop(metadata) {
|
|
|
208
366
|
.filter((entry) => Boolean(entry));
|
|
209
367
|
return normalized.length ? normalized : undefined;
|
|
210
368
|
}
|
|
369
|
+
function isServerToolFollowupRequest(metadata) {
|
|
370
|
+
if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) {
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
const rt = metadata.__rt && typeof metadata.__rt === 'object' && !Array.isArray(metadata.__rt)
|
|
374
|
+
? metadata.__rt
|
|
375
|
+
: undefined;
|
|
376
|
+
const raw = rt?.serverToolFollowup;
|
|
377
|
+
return raw === true || (typeof raw === 'string' && raw.trim().toLowerCase() === 'true');
|
|
378
|
+
}
|
|
379
|
+
function readHubDecodeBreakdown(hubStageTop) {
|
|
380
|
+
if (!Array.isArray(hubStageTop) || hubStageTop.length === 0) {
|
|
381
|
+
return { sseDecodeMs: 0, codecDecodeMs: 0 };
|
|
382
|
+
}
|
|
383
|
+
let sseDecodeMs = 0;
|
|
384
|
+
let codecDecodeMs = 0;
|
|
385
|
+
for (const entry of hubStageTop) {
|
|
386
|
+
const stage = String(entry.stage || '').trim().toLowerCase();
|
|
387
|
+
const totalMs = Number.isFinite(entry.totalMs) ? Math.max(0, Math.round(entry.totalMs)) : 0;
|
|
388
|
+
if (!(totalMs > 0) || !stage) {
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
// `resp_inbound.stage1_sse_decode` is a stable stage checkpoint even for non-stream JSON
|
|
392
|
+
// responses; counting it as "SSE decode time" makes non-stream requests look like they spent
|
|
393
|
+
// seconds decoding SSE when they only performed wrapper/text probes. Only explicit codec
|
|
394
|
+
// decoding work should contribute decode timings.
|
|
395
|
+
if (stage.includes('codec_decode')) {
|
|
396
|
+
codecDecodeMs += totalMs;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return { sseDecodeMs, codecDecodeMs };
|
|
400
|
+
}
|
|
401
|
+
function serializeRequestPayloadForRetry(payload) {
|
|
402
|
+
if (!payload || typeof payload !== 'object') {
|
|
403
|
+
return undefined;
|
|
404
|
+
}
|
|
405
|
+
try {
|
|
406
|
+
return JSON.stringify(payload);
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
logRequestExecutorNonBlockingError('serializeRequestPayloadForRetry', error);
|
|
410
|
+
return undefined;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
function cloneRequestPayloadForRetry(payload) {
|
|
414
|
+
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
|
415
|
+
return undefined;
|
|
416
|
+
}
|
|
417
|
+
try {
|
|
418
|
+
const cloned = structuredClone(payload);
|
|
419
|
+
if (cloned && typeof cloned === 'object' && !Array.isArray(cloned)) {
|
|
420
|
+
return cloned;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
logRequestExecutorNonBlockingError('cloneRequestPayloadForRetry.structuredClone', error);
|
|
425
|
+
}
|
|
426
|
+
return undefined;
|
|
427
|
+
}
|
|
428
|
+
function estimateRetryPayloadBytes(value, options) {
|
|
429
|
+
const maxBytes = options?.maxBytes ?? Number.POSITIVE_INFINITY;
|
|
430
|
+
const depth = options?.depth ?? 0;
|
|
431
|
+
const seen = options?.seen ?? new Set();
|
|
432
|
+
const nodeBudget = options?.nodeBudget ?? RETRY_PAYLOAD_ESTIMATE_NODE_BUDGET;
|
|
433
|
+
const visitedNodes = (options?.visitedNodes ?? 0) + 1;
|
|
434
|
+
if (visitedNodes > nodeBudget) {
|
|
435
|
+
return maxBytes + 1;
|
|
436
|
+
}
|
|
437
|
+
if (value === null || value === undefined) {
|
|
438
|
+
return 4;
|
|
439
|
+
}
|
|
440
|
+
const valueType = typeof value;
|
|
441
|
+
if (valueType === 'string') {
|
|
442
|
+
return Math.min(maxBytes + 1, value.length * 2 + 2);
|
|
443
|
+
}
|
|
444
|
+
if (valueType === 'number') {
|
|
445
|
+
return 8;
|
|
446
|
+
}
|
|
447
|
+
if (valueType === 'boolean') {
|
|
448
|
+
return 4;
|
|
449
|
+
}
|
|
450
|
+
if (valueType === 'bigint') {
|
|
451
|
+
return String(value).length + 8;
|
|
452
|
+
}
|
|
453
|
+
if (valueType === 'symbol' || valueType === 'function') {
|
|
454
|
+
return 16;
|
|
455
|
+
}
|
|
456
|
+
if (seen.has(value)) {
|
|
457
|
+
return 8;
|
|
458
|
+
}
|
|
459
|
+
seen.add(value);
|
|
460
|
+
if (depth >= 8) {
|
|
461
|
+
return 64;
|
|
462
|
+
}
|
|
463
|
+
let bytes = 0;
|
|
464
|
+
if (Array.isArray(value)) {
|
|
465
|
+
bytes += 2;
|
|
466
|
+
for (const item of value) {
|
|
467
|
+
bytes += estimateRetryPayloadBytes(item, {
|
|
468
|
+
maxBytes: Math.max(0, maxBytes - bytes),
|
|
469
|
+
depth: depth + 1,
|
|
470
|
+
seen,
|
|
471
|
+
nodeBudget,
|
|
472
|
+
visitedNodes
|
|
473
|
+
});
|
|
474
|
+
if (bytes > maxBytes) {
|
|
475
|
+
return maxBytes + 1;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return bytes;
|
|
479
|
+
}
|
|
480
|
+
if (value && typeof value === 'object') {
|
|
481
|
+
bytes += 2;
|
|
482
|
+
for (const [key, child] of Object.entries(value)) {
|
|
483
|
+
bytes += key.length * 2 + 4;
|
|
484
|
+
bytes += estimateRetryPayloadBytes(child, {
|
|
485
|
+
maxBytes: Math.max(0, maxBytes - bytes),
|
|
486
|
+
depth: depth + 1,
|
|
487
|
+
seen,
|
|
488
|
+
nodeBudget,
|
|
489
|
+
visitedNodes
|
|
490
|
+
});
|
|
491
|
+
if (bytes > maxBytes) {
|
|
492
|
+
return maxBytes + 1;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
return bytes;
|
|
496
|
+
}
|
|
497
|
+
return 16;
|
|
498
|
+
}
|
|
499
|
+
function prepareRequestPayloadRetrySeed(payload) {
|
|
500
|
+
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
|
501
|
+
return { mode: 'none' };
|
|
502
|
+
}
|
|
503
|
+
const estimatedBytes = estimateRetryPayloadBytes(payload, {
|
|
504
|
+
maxBytes: RETRY_PAYLOAD_ESTIMATE_MAX_BYTES + 1
|
|
505
|
+
});
|
|
506
|
+
if (estimatedBytes <= RETRY_PAYLOAD_ESTIMATE_MAX_BYTES) {
|
|
507
|
+
const serializedPayload = serializeRequestPayloadForRetry(payload);
|
|
508
|
+
if (typeof serializedPayload === 'string'
|
|
509
|
+
&& serializedPayload.length <= RETRY_SNAPSHOT_SERIALIZE_MAX_CHARS) {
|
|
510
|
+
return {
|
|
511
|
+
mode: 'serialized',
|
|
512
|
+
serializedPayload
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
const snapshotPayload = cloneRequestPayloadForRetry(payload);
|
|
517
|
+
if (snapshotPayload) {
|
|
518
|
+
return {
|
|
519
|
+
mode: 'snapshot',
|
|
520
|
+
snapshotPayload
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
const serializedPayload = serializeRequestPayloadForRetry(payload);
|
|
524
|
+
if (typeof serializedPayload === 'string'
|
|
525
|
+
&& serializedPayload.length <= RETRY_SNAPSHOT_SERIALIZE_MAX_CHARS) {
|
|
526
|
+
return {
|
|
527
|
+
mode: 'serialized',
|
|
528
|
+
serializedPayload
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
return { mode: 'none' };
|
|
532
|
+
}
|
|
533
|
+
function restoreRequestPayloadFromRetrySeed(seed) {
|
|
534
|
+
if (seed.mode === 'serialized') {
|
|
535
|
+
return restoreRequestPayloadFromRetrySnapshot(seed.serializedPayload);
|
|
536
|
+
}
|
|
537
|
+
if (seed.mode === 'snapshot') {
|
|
538
|
+
return cloneRequestPayloadForRetry(seed.snapshotPayload) ?? { ...seed.snapshotPayload };
|
|
539
|
+
}
|
|
540
|
+
return undefined;
|
|
541
|
+
}
|
|
542
|
+
function resolveOriginalRequestForResponseConversion(seed) {
|
|
543
|
+
if (seed.mode === 'snapshot') {
|
|
544
|
+
return seed.snapshotPayload;
|
|
545
|
+
}
|
|
546
|
+
return restoreRequestPayloadFromRetrySeed(seed);
|
|
547
|
+
}
|
|
548
|
+
function restoreRequestPayloadFromRetrySnapshot(serializedPayload, fallbackPayload) {
|
|
549
|
+
if (serializedPayload && typeof serializedPayload === 'string') {
|
|
550
|
+
if (serializedPayload.length > RETRY_SNAPSHOT_RESTORE_MAX_CHARS) {
|
|
551
|
+
logRequestExecutorNonBlockingError('restoreRequestPayloadFromRetrySnapshot.oversized_skip', 'serialized retry payload too large', { payloadLength: serializedPayload.length, maxChars: RETRY_SNAPSHOT_RESTORE_MAX_CHARS });
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
try {
|
|
555
|
+
const parsed = JSON.parse(serializedPayload);
|
|
556
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
557
|
+
return parsed;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
catch (error) {
|
|
561
|
+
logRequestExecutorNonBlockingError('restoreRequestPayloadFromRetrySnapshot.parseSerialized', error, {
|
|
562
|
+
payloadLength: serializedPayload.length
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
if (!fallbackPayload || typeof fallbackPayload !== 'object') {
|
|
568
|
+
return undefined;
|
|
569
|
+
}
|
|
570
|
+
const clonedFallback = cloneRequestPayloadForRetry(fallbackPayload);
|
|
571
|
+
if (clonedFallback && typeof clonedFallback === 'object') {
|
|
572
|
+
return clonedFallback;
|
|
573
|
+
}
|
|
574
|
+
return { ...fallbackPayload };
|
|
575
|
+
}
|
|
211
576
|
function truncateReason(reason, maxLength = 220) {
|
|
212
577
|
if (reason.length <= maxLength) {
|
|
213
578
|
return reason;
|
|
214
579
|
}
|
|
215
580
|
return `${reason.slice(0, Math.max(0, maxLength - 1))}…`;
|
|
216
581
|
}
|
|
582
|
+
function normalizeCodeKey(value) {
|
|
583
|
+
if (typeof value !== 'string') {
|
|
584
|
+
return undefined;
|
|
585
|
+
}
|
|
586
|
+
const normalized = value.trim().toUpperCase();
|
|
587
|
+
return normalized || undefined;
|
|
588
|
+
}
|
|
589
|
+
function isServerToolFollowupErrorCode(value) {
|
|
590
|
+
const normalized = normalizeCodeKey(value);
|
|
591
|
+
return Boolean(normalized && normalized.startsWith('SERVERTOOL_'));
|
|
592
|
+
}
|
|
593
|
+
function createClientDisconnectedAbortError(reason) {
|
|
594
|
+
const message = typeof reason === 'string' && reason.trim()
|
|
595
|
+
? reason.trim()
|
|
596
|
+
: reason instanceof Error && typeof reason.message === 'string' && reason.message.trim()
|
|
597
|
+
? reason.message.trim()
|
|
598
|
+
: 'CLIENT_DISCONNECTED';
|
|
599
|
+
return Object.assign(new Error(message), {
|
|
600
|
+
code: 'CLIENT_DISCONNECTED',
|
|
601
|
+
name: 'AbortError',
|
|
602
|
+
retryable: false
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
function throwIfClientAbortSignalAborted(signal) {
|
|
606
|
+
if (!signal?.aborted) {
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
const reason = signal.reason;
|
|
610
|
+
throw reason instanceof Error ? reason : createClientDisconnectedAbortError(reason);
|
|
611
|
+
}
|
|
612
|
+
async function waitWithClientAbortSignal(ms, signal) {
|
|
613
|
+
throwIfClientAbortSignalAborted(signal);
|
|
614
|
+
if (!(ms > 0)) {
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
await new Promise((resolve, reject) => {
|
|
618
|
+
const timer = setTimeout(() => {
|
|
619
|
+
cleanup();
|
|
620
|
+
resolve();
|
|
621
|
+
}, ms);
|
|
622
|
+
const onAbort = () => {
|
|
623
|
+
cleanup();
|
|
624
|
+
const reason = signal.reason;
|
|
625
|
+
reject(reason instanceof Error ? reason : createClientDisconnectedAbortError(reason));
|
|
626
|
+
};
|
|
627
|
+
const cleanup = () => {
|
|
628
|
+
clearTimeout(timer);
|
|
629
|
+
try {
|
|
630
|
+
signal?.removeEventListener?.('abort', onAbort);
|
|
631
|
+
}
|
|
632
|
+
catch (error) {
|
|
633
|
+
logRequestExecutorNonBlockingError('waitWithClientAbortSignal.removeEventListener', error);
|
|
634
|
+
}
|
|
635
|
+
};
|
|
636
|
+
try {
|
|
637
|
+
signal?.addEventListener?.('abort', onAbort, { once: true });
|
|
638
|
+
}
|
|
639
|
+
catch (error) {
|
|
640
|
+
logRequestExecutorNonBlockingError('waitWithClientAbortSignal.addEventListener', error);
|
|
641
|
+
}
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
function isRequestExecutorProviderErrorStage(value) {
|
|
645
|
+
return (value === 'provider.runtime_resolve'
|
|
646
|
+
|| value === 'provider.send'
|
|
647
|
+
|| value === 'host.response_contract'
|
|
648
|
+
|| value === 'host.stopless_contract'
|
|
649
|
+
|| value === 'provider.followup'
|
|
650
|
+
|| value === 'provider.sse_decode'
|
|
651
|
+
|| value === 'provider.http');
|
|
652
|
+
}
|
|
653
|
+
function isHostRequestExecutorErrorStage(stage) {
|
|
654
|
+
return stage === 'host.stopless_contract' || stage === 'host.response_contract';
|
|
655
|
+
}
|
|
656
|
+
function resolveRequestExecutorProviderErrorClassification(args) {
|
|
657
|
+
return resolveProviderFailureClassification({
|
|
658
|
+
error: args.error,
|
|
659
|
+
stage: args.stage,
|
|
660
|
+
statusCode: typeof args.retryError.statusCode === 'number'
|
|
661
|
+
? args.retryError.statusCode
|
|
662
|
+
: extractStatusCodeFromError(args.error),
|
|
663
|
+
errorCode: normalizeProviderFailureCodeKey(args.error?.code)
|
|
664
|
+
?? normalizeProviderFailureCodeKey(args.retryError.errorCode),
|
|
665
|
+
upstreamCode: normalizeProviderFailureCodeKey(args.error?.upstreamCode)
|
|
666
|
+
?? normalizeProviderFailureCodeKey(args.retryError.upstreamCode),
|
|
667
|
+
reason: String(args.retryError.reason || args.error?.message || '')
|
|
668
|
+
});
|
|
669
|
+
}
|
|
670
|
+
function isLastAvailableProvider429(args) {
|
|
671
|
+
const status = typeof args.retryError.statusCode === 'number' ? args.retryError.statusCode : undefined;
|
|
672
|
+
const errorCode = normalizeCodeKey(args.retryError.errorCode);
|
|
673
|
+
const upstreamCode = normalizeCodeKey(args.retryError.upstreamCode);
|
|
674
|
+
const is429 = status === 429 || errorCode === 'HTTP_429' || upstreamCode === 'HTTP_429';
|
|
675
|
+
if (!is429 || !readString(args.providerKey)) {
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
if (!Array.isArray(args.routePool) || args.routePool.length === 0) {
|
|
679
|
+
return true;
|
|
680
|
+
}
|
|
681
|
+
return !hasAlternativeRouteCandidate({
|
|
682
|
+
providerKey: args.providerKey,
|
|
683
|
+
routePool: args.routePool,
|
|
684
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
function shouldApplyProviderTransportBackoff(args) {
|
|
688
|
+
const stage = args.stage ?? 'provider.send';
|
|
689
|
+
if (stage === 'provider.followup' || isHostRequestExecutorErrorStage(stage)) {
|
|
690
|
+
return false;
|
|
691
|
+
}
|
|
692
|
+
// Fast path: blocking-recoverable errors (429/5xx/network/traffic-saturated etc.)
|
|
693
|
+
// always get transport backoff. Reuses shared provider-failure policy to avoid
|
|
694
|
+
// duplicate status/code predicate logic.
|
|
695
|
+
if (isBlockingRecoverableProviderFailure({
|
|
696
|
+
statusCode: args.retryError.statusCode,
|
|
697
|
+
errorCode: args.retryError.errorCode,
|
|
698
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
699
|
+
reason: args.retryError.reason
|
|
700
|
+
})) {
|
|
701
|
+
return true;
|
|
702
|
+
}
|
|
703
|
+
// Broader catch: also apply for any classified-recoverable error (includes network
|
|
704
|
+
// transport codes like ECONNRESET that aren't covered by blocking-recoverable's
|
|
705
|
+
// status/code checks).
|
|
706
|
+
return resolveRequestExecutorProviderErrorClassification({
|
|
707
|
+
error: args.error,
|
|
708
|
+
retryError: args.retryError,
|
|
709
|
+
stage
|
|
710
|
+
}) === 'recoverable';
|
|
711
|
+
}
|
|
712
|
+
function extractRequestExecutorProviderErrorStage(error) {
|
|
713
|
+
if (!error || typeof error !== 'object') {
|
|
714
|
+
return undefined;
|
|
715
|
+
}
|
|
716
|
+
const record = error;
|
|
717
|
+
const directStage = record.requestExecutorProviderErrorStage;
|
|
718
|
+
if (isRequestExecutorProviderErrorStage(directStage)) {
|
|
719
|
+
return directStage;
|
|
720
|
+
}
|
|
721
|
+
const details = record.details && typeof record.details === 'object' && !Array.isArray(record.details)
|
|
722
|
+
? record.details
|
|
723
|
+
: undefined;
|
|
724
|
+
const detailStage = details?.requestExecutorProviderErrorStage
|
|
725
|
+
?? details?.source;
|
|
726
|
+
return isRequestExecutorProviderErrorStage(detailStage) ? detailStage : undefined;
|
|
727
|
+
}
|
|
728
|
+
function resolveRequestExecutorProviderErrorReportPlan(args) {
|
|
729
|
+
const errorCode = normalizeCodeKey(args.error?.code)
|
|
730
|
+
?? normalizeCodeKey(args.retryError.errorCode);
|
|
731
|
+
const upstreamCode = normalizeCodeKey(args.error?.upstreamCode)
|
|
732
|
+
?? normalizeCodeKey(args.retryError.upstreamCode);
|
|
733
|
+
const statusCode = typeof args.retryError.statusCode === 'number'
|
|
734
|
+
? args.retryError.statusCode
|
|
735
|
+
: extractStatusCodeFromError(args.error);
|
|
736
|
+
const explicitStage = extractRequestExecutorProviderErrorStage(args.error);
|
|
737
|
+
const stageHint = explicitStage
|
|
738
|
+
? explicitStage
|
|
739
|
+
: (args.fallbackStage === 'provider.runtime_resolve'
|
|
740
|
+
? 'provider.runtime_resolve'
|
|
741
|
+
: (args.fallbackStage === 'provider.http'
|
|
742
|
+
? 'provider.http'
|
|
743
|
+
: (args.fallbackStage === 'host.response_contract'
|
|
744
|
+
? 'host.response_contract'
|
|
745
|
+
: (args.fallbackStage === 'host.stopless_contract'
|
|
746
|
+
? 'host.stopless_contract'
|
|
747
|
+
: (isSseDecodeRateLimitError(args.error, statusCode) || isSseDecodeRetryableNetworkError(args.error, statusCode)
|
|
748
|
+
? 'provider.sse_decode'
|
|
749
|
+
: (isServerToolFollowupErrorCode(errorCode) || isServerToolFollowupErrorCode(upstreamCode)
|
|
750
|
+
? 'provider.followup'
|
|
751
|
+
: args.fallbackStage))))));
|
|
752
|
+
return {
|
|
753
|
+
...(errorCode ? { errorCode } : {}),
|
|
754
|
+
...(upstreamCode ? { upstreamCode } : {}),
|
|
755
|
+
...(typeof statusCode === 'number' ? { statusCode } : {}),
|
|
756
|
+
stageHint
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
function isHealthNeutralProviderError(args) {
|
|
760
|
+
return isProviderFailureHealthNeutral({
|
|
761
|
+
stage: args.stage,
|
|
762
|
+
error: args.error,
|
|
763
|
+
errorCode: args.errorCode,
|
|
764
|
+
upstreamCode: args.upstreamCode,
|
|
765
|
+
statusCode: args.statusCode,
|
|
766
|
+
classification: args.classification
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
function resolveReportedProviderErrorRecoverable(args) {
|
|
770
|
+
if (args.stage === 'provider.followup' || isHostRequestExecutorErrorStage(args.stage)) {
|
|
771
|
+
return false;
|
|
772
|
+
}
|
|
773
|
+
const classification = resolveRequestExecutorProviderErrorClassification({
|
|
774
|
+
error: args.error,
|
|
775
|
+
retryError: args.retryError,
|
|
776
|
+
stage: args.stage
|
|
777
|
+
});
|
|
778
|
+
if (classification === 'special_400') {
|
|
779
|
+
return false;
|
|
780
|
+
}
|
|
781
|
+
if (classification === 'unrecoverable') {
|
|
782
|
+
return false;
|
|
783
|
+
}
|
|
784
|
+
if (classification === 'recoverable') {
|
|
785
|
+
return true;
|
|
786
|
+
}
|
|
787
|
+
return false;
|
|
788
|
+
}
|
|
789
|
+
async function reportRequestExecutorProviderError(args) {
|
|
790
|
+
const reportPlan = resolveRequestExecutorProviderErrorReportPlan({
|
|
791
|
+
error: args.error,
|
|
792
|
+
retryError: args.retryError,
|
|
793
|
+
fallbackStage: args.stageHint ?? 'provider.send'
|
|
794
|
+
});
|
|
795
|
+
const errorCode = reportPlan.errorCode;
|
|
796
|
+
const upstreamCode = reportPlan.upstreamCode;
|
|
797
|
+
const statusCode = reportPlan.statusCode;
|
|
798
|
+
const stage = reportPlan.stageHint;
|
|
799
|
+
const classification = resolveRequestExecutorProviderErrorClassification({
|
|
800
|
+
error: args.error,
|
|
801
|
+
retryError: args.retryError,
|
|
802
|
+
stage
|
|
803
|
+
});
|
|
804
|
+
const affectsHealth = !isHealthNeutralProviderError({
|
|
805
|
+
stage,
|
|
806
|
+
error: args.error,
|
|
807
|
+
errorCode,
|
|
808
|
+
upstreamCode,
|
|
809
|
+
statusCode,
|
|
810
|
+
classification
|
|
811
|
+
});
|
|
812
|
+
if (isHostRequestExecutorErrorStage(stage)) {
|
|
813
|
+
args.logStage('host.contract_failure.classified', args.requestId, {
|
|
814
|
+
providerKey: args.providerKey,
|
|
815
|
+
stage,
|
|
816
|
+
...(typeof statusCode === 'number' ? { statusCode } : {}),
|
|
817
|
+
...(errorCode ? { errorCode } : {}),
|
|
818
|
+
...(upstreamCode ? { upstreamCode } : {}),
|
|
819
|
+
reason: args.retryError.reason,
|
|
820
|
+
attempt: args.attempt
|
|
821
|
+
});
|
|
822
|
+
return;
|
|
823
|
+
}
|
|
824
|
+
try {
|
|
825
|
+
const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
|
|
826
|
+
emitProviderError({
|
|
827
|
+
error: args.error,
|
|
828
|
+
stage,
|
|
829
|
+
runtime: {
|
|
830
|
+
requestId: args.requestId,
|
|
831
|
+
providerKey: args.providerKey,
|
|
832
|
+
providerId: args.providerId,
|
|
833
|
+
providerType: args.providerType,
|
|
834
|
+
providerFamily: args.providerFamily,
|
|
835
|
+
providerProtocol: args.providerProtocol,
|
|
836
|
+
routeName: args.routeName,
|
|
837
|
+
pipelineId: args.providerKey,
|
|
838
|
+
target: args.target,
|
|
839
|
+
runtimeKey: args.runtimeKey
|
|
840
|
+
},
|
|
841
|
+
dependencies: args.dependencies,
|
|
842
|
+
statusCode,
|
|
843
|
+
recoverable: resolveReportedProviderErrorRecoverable({
|
|
844
|
+
stage,
|
|
845
|
+
error: args.error,
|
|
846
|
+
retryError: args.retryError
|
|
847
|
+
}),
|
|
848
|
+
affectsHealth,
|
|
849
|
+
details: {
|
|
850
|
+
source: stage,
|
|
851
|
+
...(classification ? { errorClassification: classification } : {}),
|
|
852
|
+
...(errorCode ? { errorCode } : {}),
|
|
853
|
+
...(upstreamCode ? { upstreamCode } : {}),
|
|
854
|
+
reason: args.retryError.reason,
|
|
855
|
+
attempt: args.attempt,
|
|
856
|
+
...(args.extraDetails ?? {})
|
|
857
|
+
}
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
catch (reportError) {
|
|
861
|
+
args.logStage('provider.error_reporter.failed', args.requestId, {
|
|
862
|
+
providerKey: args.providerKey,
|
|
863
|
+
stage,
|
|
864
|
+
...(typeof statusCode === 'number' ? { statusCode } : {}),
|
|
865
|
+
message: reportError instanceof Error ? reportError.message : String(reportError ?? 'Unknown reporter error'),
|
|
866
|
+
attempt: args.attempt
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
function buildRecoverableErrorBackoffKey(args) {
|
|
871
|
+
const providerScope = (() => {
|
|
872
|
+
const raw = (typeof args.providerKey === 'string' && args.providerKey.trim())
|
|
873
|
+
|| (typeof args.runtimeKey === 'string' && args.runtimeKey.trim())
|
|
874
|
+
|| 'unknown';
|
|
875
|
+
return `provider:${raw}`;
|
|
876
|
+
})();
|
|
877
|
+
const statusPart = typeof args.statusCode === 'number' ? `status:${args.statusCode}` : 'status:none';
|
|
878
|
+
const errorPart = normalizeCodeKey(args.errorCode) ?? 'error:none';
|
|
879
|
+
const upstreamPart = normalizeCodeKey(args.upstreamCode) ?? 'upstream:none';
|
|
880
|
+
const reasonPart = (() => {
|
|
881
|
+
if (typeof args.reason !== 'string') {
|
|
882
|
+
return 'reason:none';
|
|
883
|
+
}
|
|
884
|
+
const normalized = args.reason.trim().toLowerCase();
|
|
885
|
+
if (!normalized) {
|
|
886
|
+
return 'reason:none';
|
|
887
|
+
}
|
|
888
|
+
if (normalized.includes('fetch failed'))
|
|
889
|
+
return 'reason:fetch_failed';
|
|
890
|
+
if (normalized.includes('building not completed'))
|
|
891
|
+
return 'reason:building_not_completed';
|
|
892
|
+
if (normalized.includes('network'))
|
|
893
|
+
return 'reason:network';
|
|
894
|
+
if (normalized.includes('timeout'))
|
|
895
|
+
return 'reason:timeout';
|
|
896
|
+
return 'reason:other';
|
|
897
|
+
})();
|
|
898
|
+
return `${providerScope}|${statusPart}|${errorPart}|${upstreamPart}|${reasonPart}`;
|
|
899
|
+
}
|
|
900
|
+
function consumeRecoverableErrorBackoffMs(key, args) {
|
|
901
|
+
const now = Date.now();
|
|
902
|
+
for (const [existingKey, state] of recoverableErrorBackoffState.entries()) {
|
|
903
|
+
if (now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
904
|
+
recoverableErrorBackoffState.delete(existingKey);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
const previous = recoverableErrorBackoffState.get(key);
|
|
908
|
+
const consecutive = previous && now - previous.updatedAtMs < RECOVERABLE_BACKOFF_TTL_MS
|
|
909
|
+
? Math.min(previous.consecutive + 1, 16)
|
|
910
|
+
: 1;
|
|
911
|
+
recoverableErrorBackoffState.set(key, {
|
|
912
|
+
consecutive,
|
|
913
|
+
updatedAtMs: now
|
|
914
|
+
});
|
|
915
|
+
return computeProviderFailureBackoffDelayMs({
|
|
916
|
+
scope: 'recoverable',
|
|
917
|
+
statusCode: args.statusCode,
|
|
918
|
+
consecutive
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
function isNetworkTransportLikeError(error) {
|
|
922
|
+
if (!error || typeof error !== 'object') {
|
|
923
|
+
return false;
|
|
924
|
+
}
|
|
925
|
+
const record = error;
|
|
926
|
+
const code = typeof record.code === 'string' ? record.code.trim().toUpperCase() : '';
|
|
927
|
+
if (code === 'ECONNRESET'
|
|
928
|
+
|| code === 'ECONNREFUSED'
|
|
929
|
+
|| code === 'EHOSTUNREACH'
|
|
930
|
+
|| code === 'ENOTFOUND'
|
|
931
|
+
|| code === 'EAI_AGAIN'
|
|
932
|
+
|| code === 'EPIPE'
|
|
933
|
+
|| code === 'ETIMEDOUT'
|
|
934
|
+
|| code === 'ECONNABORTED') {
|
|
935
|
+
return true;
|
|
936
|
+
}
|
|
937
|
+
const name = typeof record.name === 'string' ? record.name : '';
|
|
938
|
+
const message = typeof record.message === 'string' ? record.message.toLowerCase() : '';
|
|
939
|
+
if (name === 'AbortError' || message.includes('operation was aborted')) {
|
|
940
|
+
return true;
|
|
941
|
+
}
|
|
942
|
+
return (message.includes('fetch failed')
|
|
943
|
+
|| message.includes('network timeout')
|
|
944
|
+
|| message.includes('socket hang up')
|
|
945
|
+
|| message.includes('client network socket disconnected')
|
|
946
|
+
|| message.includes('tls handshake timeout')
|
|
947
|
+
|| message.includes('unable to verify the first certificate')
|
|
948
|
+
|| message.includes('network error')
|
|
949
|
+
|| message.includes('temporarily unreachable'));
|
|
950
|
+
}
|
|
951
|
+
function consumeProviderScopedRetryBackoffMs(key, args) {
|
|
952
|
+
const now = Date.now();
|
|
953
|
+
for (const [existingKey, state] of recoverableErrorBackoffState.entries()) {
|
|
954
|
+
if (now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
955
|
+
recoverableErrorBackoffState.delete(existingKey);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
const previous = recoverableErrorBackoffState.get(key);
|
|
959
|
+
const consecutive = previous && now - previous.updatedAtMs < RECOVERABLE_BACKOFF_TTL_MS
|
|
960
|
+
? Math.min(previous.consecutive + 1, 16)
|
|
961
|
+
: 1;
|
|
962
|
+
recoverableErrorBackoffState.set(key, {
|
|
963
|
+
consecutive,
|
|
964
|
+
updatedAtMs: now
|
|
965
|
+
});
|
|
966
|
+
return computeProviderFailureBackoffDelayMs({
|
|
967
|
+
scope: 'provider',
|
|
968
|
+
error: args.error,
|
|
969
|
+
statusCode: args.statusCode,
|
|
970
|
+
consecutive
|
|
971
|
+
});
|
|
972
|
+
}
|
|
973
|
+
function buildProviderTransportBackoffKey(args) {
|
|
974
|
+
const runtimeKey = normalizeRuntimeKey(args.runtimeKey);
|
|
975
|
+
if (runtimeKey) {
|
|
976
|
+
return `runtime:${runtimeKey}`;
|
|
977
|
+
}
|
|
978
|
+
const providerKey = readString(args.providerKey);
|
|
979
|
+
if (providerKey) {
|
|
980
|
+
return `provider:${providerKey}`;
|
|
981
|
+
}
|
|
982
|
+
return undefined;
|
|
983
|
+
}
|
|
984
|
+
function consumeProviderTransportBackoffMs(key, args) {
|
|
985
|
+
const now = Date.now();
|
|
986
|
+
for (const [existingKey, state] of providerTransportBackoffState.entries()) {
|
|
987
|
+
if (now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
988
|
+
providerTransportBackoffState.delete(existingKey);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
const previous = providerTransportBackoffState.get(key);
|
|
992
|
+
const consecutive = previous && now - previous.updatedAtMs < RECOVERABLE_BACKOFF_TTL_MS
|
|
993
|
+
? Math.min(previous.consecutive + 1, 16)
|
|
994
|
+
: 1;
|
|
995
|
+
const delayMs = computeProviderFailureBackoffDelayMs({
|
|
996
|
+
scope: 'provider',
|
|
997
|
+
error: args.error,
|
|
998
|
+
statusCode: args.statusCode,
|
|
999
|
+
consecutive
|
|
1000
|
+
});
|
|
1001
|
+
providerTransportBackoffState.set(key, {
|
|
1002
|
+
consecutive,
|
|
1003
|
+
updatedAtMs: now,
|
|
1004
|
+
nextAllowedAtMs: now + delayMs
|
|
1005
|
+
});
|
|
1006
|
+
return delayMs;
|
|
1007
|
+
}
|
|
1008
|
+
function peekProviderTransportBackoffWaitMs(key) {
|
|
1009
|
+
const state = providerTransportBackoffState.get(key);
|
|
1010
|
+
if (!state) {
|
|
1011
|
+
return 0;
|
|
1012
|
+
}
|
|
1013
|
+
const now = Date.now();
|
|
1014
|
+
if (now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
1015
|
+
providerTransportBackoffState.delete(key);
|
|
1016
|
+
return 0;
|
|
1017
|
+
}
|
|
1018
|
+
return Math.max(0, state.nextAllowedAtMs - now);
|
|
1019
|
+
}
|
|
1020
|
+
function clearProviderTransportBackoff(key) {
|
|
1021
|
+
if (!key) {
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
providerTransportBackoffState.delete(key);
|
|
1025
|
+
}
|
|
1026
|
+
async function waitProviderTransportBackoffWithGate(key, ms, signal) {
|
|
1027
|
+
if (!(ms > 0)) {
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
const normalizedKey = key.trim() || 'provider:unknown';
|
|
1031
|
+
const previous = providerTransportBackoffGateState.get(normalizedKey) ?? Promise.resolve();
|
|
1032
|
+
let release = () => undefined;
|
|
1033
|
+
const current = new Promise((resolve) => {
|
|
1034
|
+
release = resolve;
|
|
1035
|
+
});
|
|
1036
|
+
providerTransportBackoffGateState.set(normalizedKey, current);
|
|
1037
|
+
try {
|
|
1038
|
+
await previous.catch((error) => {
|
|
1039
|
+
logRequestExecutorNonBlockingError('waitProviderTransportBackoffWithGate.previous', error, {
|
|
1040
|
+
key: normalizedKey
|
|
1041
|
+
});
|
|
1042
|
+
});
|
|
1043
|
+
await waitWithClientAbortSignal(ms, signal);
|
|
1044
|
+
}
|
|
1045
|
+
finally {
|
|
1046
|
+
release();
|
|
1047
|
+
if (providerTransportBackoffGateState.get(normalizedKey) === current) {
|
|
1048
|
+
providerTransportBackoffGateState.delete(normalizedKey);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
function resolveProviderRetryEligibilityPlan(args) {
|
|
1053
|
+
const antigravityRecoveryEligible = args.allowAntigravityRecovery
|
|
1054
|
+
&& isAntigravityProviderKey(args.providerKey)
|
|
1055
|
+
&& (args.isVerify || args.isReauth);
|
|
1056
|
+
const eligibility = resolveProviderFailureRetryEligibility({
|
|
1057
|
+
error: args.error,
|
|
1058
|
+
stage: args.stage,
|
|
1059
|
+
statusCode: args.retryError.statusCode,
|
|
1060
|
+
errorCode: args.retryError.errorCode,
|
|
1061
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1062
|
+
reason: args.retryError.reason,
|
|
1063
|
+
classification: resolveRequestExecutorProviderErrorClassification({
|
|
1064
|
+
error: args.error,
|
|
1065
|
+
retryError: args.retryError,
|
|
1066
|
+
stage: args.stage
|
|
1067
|
+
}),
|
|
1068
|
+
attempt: args.attempt,
|
|
1069
|
+
maxAttempts: args.maxAttempts,
|
|
1070
|
+
promptTooLong: args.promptTooLong,
|
|
1071
|
+
contextOverflowRetries: args.contextOverflowRetries,
|
|
1072
|
+
maxContextOverflowRetries: args.maxContextOverflowRetries ?? MAX_CONTEXT_OVERFLOW_RETRIES,
|
|
1073
|
+
allowNonPolicyRetry: antigravityRecoveryEligible,
|
|
1074
|
+
stageOutsideProviderFailurePolicy: args.stage === 'host.response_contract'
|
|
1075
|
+
|| args.stage === 'host.stopless_contract'
|
|
1076
|
+
});
|
|
1077
|
+
return {
|
|
1078
|
+
shouldRetry: eligibility.shouldRetry,
|
|
1079
|
+
blockingRecoverable: eligibility.blockingRecoverable
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
async function resolveProviderRetryExecutionPlan(args) {
|
|
1083
|
+
const hostContractFailure = isHostRequestExecutorErrorStage(args.stage ?? 'provider.send');
|
|
1084
|
+
const rerouteHostContractFailure = args.stage === 'host.stopless_contract';
|
|
1085
|
+
const classification = resolveRequestExecutorProviderErrorClassification({
|
|
1086
|
+
error: args.error,
|
|
1087
|
+
retryError: args.retryError,
|
|
1088
|
+
stage: args.stage
|
|
1089
|
+
});
|
|
1090
|
+
const eligibilityPlan = resolveProviderRetryEligibilityPlan({
|
|
1091
|
+
error: args.error,
|
|
1092
|
+
retryError: args.retryError,
|
|
1093
|
+
attempt: args.attempt,
|
|
1094
|
+
maxAttempts: args.maxAttempts,
|
|
1095
|
+
stage: args.stage,
|
|
1096
|
+
providerKey: args.providerKey,
|
|
1097
|
+
promptTooLong: args.promptTooLong,
|
|
1098
|
+
contextOverflowRetries: args.contextOverflowRetries,
|
|
1099
|
+
maxContextOverflowRetries: args.maxContextOverflowRetries,
|
|
1100
|
+
isVerify: args.isVerify,
|
|
1101
|
+
isReauth: args.isReauth,
|
|
1102
|
+
allowAntigravityRecovery: args.allowAntigravityRecovery
|
|
1103
|
+
});
|
|
1104
|
+
args.recordAttempt({ error: true });
|
|
1105
|
+
if (!eligibilityPlan.shouldRetry) {
|
|
1106
|
+
return {
|
|
1107
|
+
shouldRetry: false,
|
|
1108
|
+
blockingRecoverable: eligibilityPlan.blockingRecoverable,
|
|
1109
|
+
excludedCurrentProvider: false,
|
|
1110
|
+
holdOnLastAvailable429: false,
|
|
1111
|
+
retryBackoffMs: 0,
|
|
1112
|
+
recoverableBackoffMs: 0,
|
|
1113
|
+
antigravityRetrySignal: args.antigravityRetrySignal ?? null
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
const exclusionPlan = hostContractFailure
|
|
1117
|
+
? {
|
|
1118
|
+
excludedCurrentProvider: rerouteHostContractFailure
|
|
1119
|
+
? applyRetryExclusionForCurrentProvider({
|
|
1120
|
+
providerKey: args.providerKey,
|
|
1121
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1122
|
+
})
|
|
1123
|
+
: false,
|
|
1124
|
+
antigravityRetrySignal: args.antigravityRetrySignal ?? null
|
|
1125
|
+
}
|
|
1126
|
+
: args.forceExcludeCurrentProviderOnRetry
|
|
1127
|
+
? {
|
|
1128
|
+
excludedCurrentProvider: applyRetryExclusionForCurrentProvider({
|
|
1129
|
+
providerKey: args.providerKey,
|
|
1130
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1131
|
+
}),
|
|
1132
|
+
antigravityRetrySignal: args.antigravityRetrySignal ?? null
|
|
1133
|
+
}
|
|
1134
|
+
: resolveProviderRetryExclusionPlan({
|
|
1135
|
+
providerKey: args.providerKey,
|
|
1136
|
+
status: args.status,
|
|
1137
|
+
error: args.error,
|
|
1138
|
+
classification,
|
|
1139
|
+
promptTooLong: Boolean(args.promptTooLong),
|
|
1140
|
+
isVerify: Boolean(args.isVerify),
|
|
1141
|
+
isReauth: Boolean(args.isReauth),
|
|
1142
|
+
antigravityRetrySignal: args.antigravityRetrySignal ?? null,
|
|
1143
|
+
routePool: args.routePool,
|
|
1144
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1145
|
+
});
|
|
1146
|
+
const holdOnLastAvailable429 = isLastAvailableProvider429({
|
|
1147
|
+
providerKey: args.providerKey,
|
|
1148
|
+
routePool: args.routePool,
|
|
1149
|
+
excludedProviderKeys: args.excludedProviderKeys,
|
|
1150
|
+
retryError: args.retryError
|
|
1151
|
+
});
|
|
1152
|
+
if (classification === 'unrecoverable'
|
|
1153
|
+
&& !exclusionPlan.excludedCurrentProvider
|
|
1154
|
+
&& args.error?.retryable !== true) {
|
|
1155
|
+
return {
|
|
1156
|
+
shouldRetry: false,
|
|
1157
|
+
blockingRecoverable: eligibilityPlan.blockingRecoverable,
|
|
1158
|
+
excludedCurrentProvider: false,
|
|
1159
|
+
holdOnLastAvailable429,
|
|
1160
|
+
retryBackoffMs: 0,
|
|
1161
|
+
recoverableBackoffMs: 0,
|
|
1162
|
+
antigravityRetrySignal: exclusionPlan.antigravityRetrySignal
|
|
1163
|
+
};
|
|
1164
|
+
}
|
|
1165
|
+
const retryBackoffPlan = await resolveProviderRetryBackoffPlan({
|
|
1166
|
+
error: args.error,
|
|
1167
|
+
retryError: args.retryError,
|
|
1168
|
+
providerKey: args.providerKey,
|
|
1169
|
+
runtimeKey: args.runtimeKey,
|
|
1170
|
+
logicalRequestChainKey: args.logicalRequestChainKey,
|
|
1171
|
+
logicalChainRetryLimitStageRequestId: args.logicalChainRetryLimitStageRequestId,
|
|
1172
|
+
attempt: args.attempt,
|
|
1173
|
+
forceProviderScopedBackoff: exclusionPlan.excludedCurrentProvider,
|
|
1174
|
+
forceAttemptScopedBackoff: hostContractFailure && !exclusionPlan.excludedCurrentProvider,
|
|
1175
|
+
abortSignal: args.abortSignal,
|
|
1176
|
+
logStage: args.logStage
|
|
1177
|
+
});
|
|
1178
|
+
const retrySwitchPlan = buildProviderRetrySwitchPlan({
|
|
1179
|
+
runtimeKey: args.runtimeKey,
|
|
1180
|
+
routePool: args.routePool,
|
|
1181
|
+
runtimeManager: args.runtimeManager,
|
|
1182
|
+
excludedProviderKeys: args.excludedProviderKeys,
|
|
1183
|
+
excludedCurrentProvider: exclusionPlan.excludedCurrentProvider,
|
|
1184
|
+
promptTooLong: args.promptTooLong,
|
|
1185
|
+
error: args.error,
|
|
1186
|
+
retryError: args.retryError,
|
|
1187
|
+
backoffScope: retryBackoffPlan.backoffScope
|
|
1188
|
+
});
|
|
1189
|
+
if (classification === 'unrecoverable'
|
|
1190
|
+
&& retrySwitchPlan.switchAction === 'exclude_and_reroute'
|
|
1191
|
+
&& !hasAlternativeRouteCandidate({
|
|
1192
|
+
providerKey: args.providerKey,
|
|
1193
|
+
routePool: args.routePool,
|
|
1194
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1195
|
+
})) {
|
|
1196
|
+
return {
|
|
1197
|
+
shouldRetry: false,
|
|
1198
|
+
blockingRecoverable: eligibilityPlan.blockingRecoverable,
|
|
1199
|
+
excludedCurrentProvider: exclusionPlan.excludedCurrentProvider,
|
|
1200
|
+
holdOnLastAvailable429,
|
|
1201
|
+
retryBackoffMs: 0,
|
|
1202
|
+
recoverableBackoffMs: 0,
|
|
1203
|
+
antigravityRetrySignal: exclusionPlan.antigravityRetrySignal
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
return {
|
|
1207
|
+
shouldRetry: true,
|
|
1208
|
+
blockingRecoverable: eligibilityPlan.blockingRecoverable,
|
|
1209
|
+
excludedCurrentProvider: exclusionPlan.excludedCurrentProvider,
|
|
1210
|
+
holdOnLastAvailable429,
|
|
1211
|
+
retryBackoffMs: retryBackoffPlan.retryBackoffMs,
|
|
1212
|
+
recoverableBackoffMs: retryBackoffPlan.recoverableBackoffMs,
|
|
1213
|
+
backoffScope: retryBackoffPlan.backoffScope,
|
|
1214
|
+
retrySwitchPlan,
|
|
1215
|
+
antigravityRetrySignal: exclusionPlan.antigravityRetrySignal
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
function hasAlternativeRouteCandidate(args) {
|
|
1219
|
+
const currentProviderKey = readString(args.providerKey);
|
|
1220
|
+
if (!Array.isArray(args.routePool) || args.routePool.length === 0) {
|
|
1221
|
+
return true;
|
|
1222
|
+
}
|
|
1223
|
+
return args.routePool.some((candidate) => {
|
|
1224
|
+
const normalized = readString(candidate);
|
|
1225
|
+
if (!normalized) {
|
|
1226
|
+
return false;
|
|
1227
|
+
}
|
|
1228
|
+
if (currentProviderKey && normalized === currentProviderKey) {
|
|
1229
|
+
return false;
|
|
1230
|
+
}
|
|
1231
|
+
return !args.excludedProviderKeys.has(normalized);
|
|
1232
|
+
});
|
|
1233
|
+
}
|
|
1234
|
+
function resolveExcludedProviderReselectionPlan(args) {
|
|
1235
|
+
const hasAlternativeCandidate = hasAlternativeRouteCandidate({
|
|
1236
|
+
providerKey: args.providerKey,
|
|
1237
|
+
routePool: args.routePool,
|
|
1238
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1239
|
+
});
|
|
1240
|
+
const classification = args.lastError
|
|
1241
|
+
? resolveRequestExecutorProviderErrorClassification({
|
|
1242
|
+
error: args.lastError,
|
|
1243
|
+
retryError: extractRetryErrorSnapshot(args.lastError),
|
|
1244
|
+
stage: 'provider.send'
|
|
1245
|
+
})
|
|
1246
|
+
: undefined;
|
|
1247
|
+
return {
|
|
1248
|
+
hasAlternativeCandidate,
|
|
1249
|
+
keepExcludedForNextAttempt: classification === 'unrecoverable' || hasAlternativeCandidate
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
async function resolveRequestExecutorProviderFailurePlan(args) {
|
|
1253
|
+
const reportPlan = resolveRequestExecutorProviderErrorReportPlan({
|
|
1254
|
+
error: args.error,
|
|
1255
|
+
retryError: args.retryError,
|
|
1256
|
+
fallbackStage: args.stage
|
|
1257
|
+
});
|
|
1258
|
+
const retryExecutionPlan = await resolveProviderRetryExecutionPlan({
|
|
1259
|
+
error: args.error,
|
|
1260
|
+
retryError: args.retryError,
|
|
1261
|
+
attempt: args.attempt,
|
|
1262
|
+
maxAttempts: args.maxAttempts,
|
|
1263
|
+
stage: reportPlan.stageHint,
|
|
1264
|
+
providerKey: args.providerKey,
|
|
1265
|
+
runtimeKey: args.runtimeKey,
|
|
1266
|
+
logicalRequestChainKey: args.logicalRequestChainKey,
|
|
1267
|
+
logicalChainRetryLimitStageRequestId: args.logicalChainRetryLimitStageRequestId,
|
|
1268
|
+
routePool: args.routePool,
|
|
1269
|
+
runtimeManager: args.runtimeManager,
|
|
1270
|
+
excludedProviderKeys: args.excludedProviderKeys,
|
|
1271
|
+
recordAttempt: args.recordAttempt,
|
|
1272
|
+
logStage: args.logStage,
|
|
1273
|
+
promptTooLong: args.promptTooLong,
|
|
1274
|
+
contextOverflowRetries: args.contextOverflowRetries,
|
|
1275
|
+
maxContextOverflowRetries: args.maxContextOverflowRetries,
|
|
1276
|
+
isVerify: args.isVerify,
|
|
1277
|
+
isReauth: args.isReauth,
|
|
1278
|
+
allowAntigravityRecovery: args.allowAntigravityRecovery,
|
|
1279
|
+
antigravityRetrySignal: args.antigravityRetrySignal,
|
|
1280
|
+
status: args.status,
|
|
1281
|
+
forceExcludeCurrentProviderOnRetry: args.forceExcludeCurrentProviderOnRetry,
|
|
1282
|
+
abortSignal: args.abortSignal
|
|
1283
|
+
});
|
|
1284
|
+
// `reportRequestExecutorProviderError -> emitProviderError` may back-propagate
|
|
1285
|
+
// `recoverable` onto the original error object (`err.retryable = recoverable`).
|
|
1286
|
+
// Retry planning must use the provider's original retry hint, not the router-facing
|
|
1287
|
+
// classification side effect, otherwise retryable 403/compat cases can be downgraded
|
|
1288
|
+
// before host failover sees them.
|
|
1289
|
+
await reportRequestExecutorProviderError({
|
|
1290
|
+
error: cloneErrorForReporting(args.error),
|
|
1291
|
+
retryError: args.retryError,
|
|
1292
|
+
requestId: args.requestId,
|
|
1293
|
+
providerKey: args.providerKey,
|
|
1294
|
+
providerId: args.providerId,
|
|
1295
|
+
providerType: args.providerType,
|
|
1296
|
+
providerFamily: args.providerFamily,
|
|
1297
|
+
providerProtocol: args.providerProtocol,
|
|
1298
|
+
routeName: args.routeName,
|
|
1299
|
+
runtimeKey: args.runtimeKey,
|
|
1300
|
+
target: args.target,
|
|
1301
|
+
dependencies: args.dependencies,
|
|
1302
|
+
attempt: args.attempt,
|
|
1303
|
+
logStage: args.logStage,
|
|
1304
|
+
stageHint: reportPlan.stageHint
|
|
1305
|
+
});
|
|
1306
|
+
const retryTelemetryPlan = retryExecutionPlan.shouldRetry && retryExecutionPlan.retrySwitchPlan && retryExecutionPlan.backoffScope
|
|
1307
|
+
? buildProviderRetryTelemetryPlan({
|
|
1308
|
+
requestId: args.requestId,
|
|
1309
|
+
attempt: args.attempt,
|
|
1310
|
+
maxAttempts: args.maxAttempts,
|
|
1311
|
+
providerKey: args.providerKey,
|
|
1312
|
+
retryError: args.retryError,
|
|
1313
|
+
excludedProviderKeys: args.excludedProviderKeys,
|
|
1314
|
+
routeHint: args.routeHint,
|
|
1315
|
+
retryExecutionPlan,
|
|
1316
|
+
stage: args.stage,
|
|
1317
|
+
runtimeKey: args.runtimeKey,
|
|
1318
|
+
promptTooLong: args.promptTooLong,
|
|
1319
|
+
contextOverflowRetries: args.contextOverflowRetries,
|
|
1320
|
+
maxContextOverflowRetries: args.maxContextOverflowRetries
|
|
1321
|
+
})
|
|
1322
|
+
: undefined;
|
|
1323
|
+
return {
|
|
1324
|
+
reportPlan,
|
|
1325
|
+
retryExecutionPlan,
|
|
1326
|
+
...(retryTelemetryPlan ? { retryTelemetryPlan } : {})
|
|
1327
|
+
};
|
|
1328
|
+
}
|
|
1329
|
+
function emitRequestExecutorProviderRetryTelemetry(args) {
|
|
1330
|
+
if (args.retryTelemetryPlan.runtimeScopeExcludeDetails) {
|
|
1331
|
+
args.logStage('provider.retry.runtime_scope_exclude', args.requestId, args.retryTelemetryPlan.runtimeScopeExcludeDetails);
|
|
1332
|
+
}
|
|
1333
|
+
args.logProviderRetrySwitch(args.retryTelemetryPlan.switchLogArgs);
|
|
1334
|
+
args.logStage('provider.retry', args.requestId, args.retryTelemetryPlan.retryStageDetails);
|
|
1335
|
+
}
|
|
1336
|
+
function buildProviderRetryTelemetryPlan(args) {
|
|
1337
|
+
if (!args.retryExecutionPlan.retrySwitchPlan || !args.retryExecutionPlan.backoffScope) {
|
|
1338
|
+
throw new Error('retry telemetry requires retrySwitchPlan/backoffScope');
|
|
1339
|
+
}
|
|
1340
|
+
const retrySwitchPlan = args.retryExecutionPlan.retrySwitchPlan;
|
|
1341
|
+
const nextAttempt = Math.min(args.maxAttempts, args.attempt + 1);
|
|
1342
|
+
const switchLogArgs = {
|
|
1343
|
+
requestId: args.requestId,
|
|
1344
|
+
attempt: args.attempt,
|
|
1345
|
+
maxAttempts: args.maxAttempts,
|
|
1346
|
+
providerKey: args.providerKey,
|
|
1347
|
+
nextAttempt,
|
|
1348
|
+
reason: args.retryError.reason,
|
|
1349
|
+
backoffMs: args.retryExecutionPlan.retryBackoffMs,
|
|
1350
|
+
statusCode: args.retryError.statusCode,
|
|
1351
|
+
errorCode: args.retryError.errorCode,
|
|
1352
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1353
|
+
switchAction: retrySwitchPlan.switchAction,
|
|
1354
|
+
backoffScope: args.retryExecutionPlan.backoffScope,
|
|
1355
|
+
decisionLabel: retrySwitchPlan.decisionLabel,
|
|
1356
|
+
stage: args.stage,
|
|
1357
|
+
runtimeScopeExcludedCount: retrySwitchPlan.runtimeScopeExcludedCount
|
|
1358
|
+
};
|
|
1359
|
+
const retryStageDetails = {
|
|
1360
|
+
providerKey: args.providerKey,
|
|
1361
|
+
attempt: args.attempt,
|
|
1362
|
+
nextAttempt,
|
|
1363
|
+
excluded: Array.from(args.excludedProviderKeys),
|
|
1364
|
+
reason: args.retryError.reason,
|
|
1365
|
+
routeHint: args.routeHint,
|
|
1366
|
+
switchAction: retrySwitchPlan.switchAction,
|
|
1367
|
+
...(typeof args.retryError.statusCode === 'number' ? { statusCode: args.retryError.statusCode } : {}),
|
|
1368
|
+
...(args.retryError.errorCode ? { errorCode: args.retryError.errorCode } : {}),
|
|
1369
|
+
...(args.retryError.upstreamCode ? { upstreamCode: args.retryError.upstreamCode } : {}),
|
|
1370
|
+
retryBackoffMs: args.retryExecutionPlan.retryBackoffMs,
|
|
1371
|
+
recoverableBackoffMs: args.retryExecutionPlan.recoverableBackoffMs,
|
|
1372
|
+
backoffScope: args.retryExecutionPlan.backoffScope,
|
|
1373
|
+
decisionLabel: retrySwitchPlan.decisionLabel,
|
|
1374
|
+
...(retrySwitchPlan.runtimeScopeExcludedCount > 0
|
|
1375
|
+
? { runtimeScopeExcludedCount: retrySwitchPlan.runtimeScopeExcludedCount }
|
|
1376
|
+
: {}),
|
|
1377
|
+
holdOnLastAvailable429: args.retryExecutionPlan.holdOnLastAvailable429,
|
|
1378
|
+
blockingRecoverable: args.retryExecutionPlan.blockingRecoverable,
|
|
1379
|
+
...(args.promptTooLong
|
|
1380
|
+
? {
|
|
1381
|
+
contextOverflowRetries: args.contextOverflowRetries,
|
|
1382
|
+
maxContextOverflowRetries: args.maxContextOverflowRetries
|
|
1383
|
+
}
|
|
1384
|
+
: {})
|
|
1385
|
+
};
|
|
1386
|
+
const runtimeScopeExcludeDetails = retrySwitchPlan.runtimeScopeExcluded.length > 0
|
|
1387
|
+
? {
|
|
1388
|
+
providerKey: args.providerKey,
|
|
1389
|
+
runtimeKey: args.runtimeKey,
|
|
1390
|
+
excludedRuntimeScope: retrySwitchPlan.runtimeScopeExcluded,
|
|
1391
|
+
attempt: args.attempt
|
|
1392
|
+
}
|
|
1393
|
+
: undefined;
|
|
1394
|
+
return {
|
|
1395
|
+
switchLogArgs,
|
|
1396
|
+
retryStageDetails,
|
|
1397
|
+
runtimeScopeExcludeDetails
|
|
1398
|
+
};
|
|
1399
|
+
}
|
|
1400
|
+
async function resolveProviderRetryBackoffPlan(args) {
|
|
1401
|
+
const actionPlan = resolveProviderFailureActionPlan({
|
|
1402
|
+
error: args.error,
|
|
1403
|
+
statusCode: args.retryError.statusCode,
|
|
1404
|
+
errorCode: args.retryError.errorCode,
|
|
1405
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1406
|
+
reason: args.retryError.reason,
|
|
1407
|
+
forceProviderScopedBackoff: args.forceProviderScopedBackoff,
|
|
1408
|
+
forceAttemptScopedBackoff: args.forceAttemptScopedBackoff,
|
|
1409
|
+
retryAction: args.forceProviderScopedBackoff ? 'reroute_explicit_alternative' : 'retry_same_provider'
|
|
1410
|
+
});
|
|
1411
|
+
const blockingRecoverable = actionPlan.blockingRecoverable;
|
|
1412
|
+
if (blockingRecoverable) {
|
|
1413
|
+
const logicalChainRetry = consumeLogicalChainRecoverableRetry(args.logicalRequestChainKey);
|
|
1414
|
+
if (!logicalChainRetry.allowed) {
|
|
1415
|
+
args.logStage('provider.retry.logical_chain_limit_hit', args.logicalChainRetryLimitStageRequestId, {
|
|
1416
|
+
providerKey: args.providerKey,
|
|
1417
|
+
logicalRequestChainKey: args.logicalRequestChainKey,
|
|
1418
|
+
logicalChainRecoverableRetries: logicalChainRetry.count,
|
|
1419
|
+
logicalChainRecoverableRetryLimit: logicalChainRetry.limit,
|
|
1420
|
+
attempt: args.attempt,
|
|
1421
|
+
...(typeof args.retryError.statusCode === 'number' ? { statusCode: args.retryError.statusCode } : {}),
|
|
1422
|
+
...(args.retryError.errorCode ? { errorCode: args.retryError.errorCode } : {}),
|
|
1423
|
+
...(args.retryError.upstreamCode ? { upstreamCode: args.retryError.upstreamCode } : {}),
|
|
1424
|
+
reason: args.retryError.reason
|
|
1425
|
+
});
|
|
1426
|
+
throw args.error;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
if (actionPlan.backoff.scope === 'attempt') {
|
|
1430
|
+
const retryBackoffMs = await waitBeforeRetry(args.error, {
|
|
1431
|
+
attempt: args.attempt,
|
|
1432
|
+
signal: args.abortSignal
|
|
1433
|
+
});
|
|
1434
|
+
return {
|
|
1435
|
+
blockingRecoverable,
|
|
1436
|
+
retryBackoffMs,
|
|
1437
|
+
recoverableBackoffMs: 0,
|
|
1438
|
+
backoffScope: 'attempt'
|
|
1439
|
+
};
|
|
1440
|
+
}
|
|
1441
|
+
if (actionPlan.backoff.scope === 'provider') {
|
|
1442
|
+
const providerScopedKey = buildRecoverableErrorBackoffKey({
|
|
1443
|
+
providerKey: args.providerKey,
|
|
1444
|
+
runtimeKey: args.runtimeKey,
|
|
1445
|
+
statusCode: args.retryError.statusCode,
|
|
1446
|
+
errorCode: args.retryError.errorCode,
|
|
1447
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1448
|
+
reason: args.retryError.reason
|
|
1449
|
+
});
|
|
1450
|
+
const retryBackoffMs = consumeProviderScopedRetryBackoffMs(providerScopedKey, {
|
|
1451
|
+
error: args.error,
|
|
1452
|
+
statusCode: args.retryError.statusCode
|
|
1453
|
+
});
|
|
1454
|
+
await waitRecoverableBackoffWithGlobalGate(providerScopedKey, retryBackoffMs, args.abortSignal);
|
|
1455
|
+
return {
|
|
1456
|
+
blockingRecoverable,
|
|
1457
|
+
retryBackoffMs,
|
|
1458
|
+
recoverableBackoffMs: 0,
|
|
1459
|
+
backoffScope: 'provider'
|
|
1460
|
+
};
|
|
1461
|
+
}
|
|
1462
|
+
if (actionPlan.backoff.scope !== 'recoverable') {
|
|
1463
|
+
const retryBackoffMs = await waitBeforeRetry(args.error, {
|
|
1464
|
+
attempt: args.attempt,
|
|
1465
|
+
signal: args.abortSignal
|
|
1466
|
+
});
|
|
1467
|
+
return {
|
|
1468
|
+
blockingRecoverable,
|
|
1469
|
+
retryBackoffMs,
|
|
1470
|
+
recoverableBackoffMs: 0,
|
|
1471
|
+
backoffScope: 'attempt'
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
const recoverableKey = buildRecoverableErrorBackoffKey({
|
|
1475
|
+
providerKey: args.providerKey,
|
|
1476
|
+
runtimeKey: args.runtimeKey,
|
|
1477
|
+
statusCode: args.retryError.statusCode,
|
|
1478
|
+
errorCode: args.retryError.errorCode,
|
|
1479
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1480
|
+
reason: args.retryError.reason
|
|
1481
|
+
});
|
|
1482
|
+
const recoverableBackoffMs = consumeRecoverableErrorBackoffMs(recoverableKey, {
|
|
1483
|
+
statusCode: args.retryError.statusCode,
|
|
1484
|
+
errorCode: args.retryError.errorCode,
|
|
1485
|
+
upstreamCode: args.retryError.upstreamCode,
|
|
1486
|
+
reason: args.retryError.reason
|
|
1487
|
+
});
|
|
1488
|
+
await waitRecoverableBackoffWithGlobalGate(recoverableKey, recoverableBackoffMs, args.abortSignal);
|
|
1489
|
+
return {
|
|
1490
|
+
blockingRecoverable,
|
|
1491
|
+
retryBackoffMs: recoverableBackoffMs,
|
|
1492
|
+
recoverableBackoffMs,
|
|
1493
|
+
backoffScope: 'recoverable'
|
|
1494
|
+
};
|
|
1495
|
+
}
|
|
1496
|
+
function buildProviderRetrySwitchPlan(args) {
|
|
1497
|
+
const switchAction = args.excludedCurrentProvider ? 'exclude_and_reroute' : 'retry_same_provider';
|
|
1498
|
+
let runtimeScopeExcluded = [];
|
|
1499
|
+
const isProviderTrafficSaturated = args.retryError?.errorCode === 'PROVIDER_TRAFFIC_SATURATED'
|
|
1500
|
+
|| (typeof args.error?.code === 'string'
|
|
1501
|
+
&& args.error.code === 'PROVIDER_TRAFFIC_SATURATED');
|
|
1502
|
+
if (!args.promptTooLong
|
|
1503
|
+
&& args.excludedCurrentProvider
|
|
1504
|
+
&& isProviderTrafficSaturated
|
|
1505
|
+
&& Array.isArray(args.routePool)
|
|
1506
|
+
&& args.routePool.length > 0
|
|
1507
|
+
&& args.runtimeManager) {
|
|
1508
|
+
runtimeScopeExcluded = excludeProvidersSharingRuntimeFromRoutePool({
|
|
1509
|
+
routePool: args.routePool,
|
|
1510
|
+
runtimeKey: args.runtimeKey ?? '',
|
|
1511
|
+
runtimeManager: args.runtimeManager,
|
|
1512
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1513
|
+
});
|
|
1514
|
+
}
|
|
1515
|
+
return {
|
|
1516
|
+
switchAction,
|
|
1517
|
+
decisionLabel: describeProviderFailureDecision({
|
|
1518
|
+
action: switchAction === 'exclude_and_reroute' ? 'reroute_explicit_alternative' : 'retry_same_provider',
|
|
1519
|
+
backoffScope: args.backoffScope
|
|
1520
|
+
}),
|
|
1521
|
+
runtimeScopeExcluded,
|
|
1522
|
+
runtimeScopeExcludedCount: runtimeScopeExcluded.length
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
async function waitRecoverableBackoffMs(ms, signal) {
|
|
1526
|
+
await waitWithClientAbortSignal(ms, signal);
|
|
1527
|
+
}
|
|
1528
|
+
function resolveRecoverableBackoffMaxWaiters() {
|
|
1529
|
+
const raw = process.env.ROUTECODEX_RECOVERABLE_BACKOFF_MAX_WAITERS
|
|
1530
|
+
?? process.env.RCC_RECOVERABLE_BACKOFF_MAX_WAITERS
|
|
1531
|
+
?? '';
|
|
1532
|
+
const parsed = Number.parseInt(String(raw).trim(), 10);
|
|
1533
|
+
if (Number.isFinite(parsed) && parsed >= 1) {
|
|
1534
|
+
return parsed;
|
|
1535
|
+
}
|
|
1536
|
+
return 64;
|
|
1537
|
+
}
|
|
1538
|
+
function acquireRecoverableWaiterSlot(key) {
|
|
1539
|
+
const normalizedKey = key.trim() || 'recoverable:unknown';
|
|
1540
|
+
const now = Date.now();
|
|
1541
|
+
for (const [existingKey, state] of recoverableRetryWaiterState.entries()) {
|
|
1542
|
+
if (state.activeWaiters <= 0 || now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
1543
|
+
recoverableRetryWaiterState.delete(existingKey);
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
const current = recoverableRetryWaiterState.get(normalizedKey);
|
|
1547
|
+
const nextActiveWaiters = (current?.activeWaiters ?? 0) + 1;
|
|
1548
|
+
const maxWaiters = resolveRecoverableBackoffMaxWaiters();
|
|
1549
|
+
if (nextActiveWaiters > maxWaiters) {
|
|
1550
|
+
throw Object.assign(new Error(`recoverable retry waiters overloaded for key ${normalizedKey}`), {
|
|
1551
|
+
statusCode: 429,
|
|
1552
|
+
code: 'PROVIDER_TRAFFIC_SATURATED',
|
|
1553
|
+
retryable: true,
|
|
1554
|
+
details: {
|
|
1555
|
+
reason: 'recoverable_waiter_overload',
|
|
1556
|
+
recoverableKey: normalizedKey,
|
|
1557
|
+
activeWaiters: current?.activeWaiters ?? 0,
|
|
1558
|
+
maxWaiters
|
|
1559
|
+
}
|
|
1560
|
+
});
|
|
1561
|
+
}
|
|
1562
|
+
recoverableRetryWaiterState.set(normalizedKey, {
|
|
1563
|
+
activeWaiters: nextActiveWaiters,
|
|
1564
|
+
updatedAtMs: now
|
|
1565
|
+
});
|
|
1566
|
+
return {
|
|
1567
|
+
key: normalizedKey,
|
|
1568
|
+
activeWaiters: nextActiveWaiters
|
|
1569
|
+
};
|
|
1570
|
+
}
|
|
1571
|
+
function releaseRecoverableWaiterSlot(key) {
|
|
1572
|
+
const normalizedKey = key.trim() || 'recoverable:unknown';
|
|
1573
|
+
const current = recoverableRetryWaiterState.get(normalizedKey);
|
|
1574
|
+
if (!current) {
|
|
1575
|
+
return;
|
|
1576
|
+
}
|
|
1577
|
+
const nextActiveWaiters = Math.max(0, current.activeWaiters - 1);
|
|
1578
|
+
if (nextActiveWaiters === 0) {
|
|
1579
|
+
recoverableRetryWaiterState.delete(normalizedKey);
|
|
1580
|
+
return;
|
|
1581
|
+
}
|
|
1582
|
+
recoverableRetryWaiterState.set(normalizedKey, {
|
|
1583
|
+
activeWaiters: nextActiveWaiters,
|
|
1584
|
+
updatedAtMs: Date.now()
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
function acquireRecoverableRetryWaiterSlotForTests(key) {
|
|
1588
|
+
return acquireRecoverableWaiterSlot(key);
|
|
1589
|
+
}
|
|
1590
|
+
function releaseRecoverableRetryWaiterSlotForTests(key) {
|
|
1591
|
+
releaseRecoverableWaiterSlot(key);
|
|
1592
|
+
}
|
|
1593
|
+
async function waitRecoverableBackoffWithGlobalGate(key, ms, signal) {
|
|
1594
|
+
const waiter = acquireRecoverableWaiterSlot(key);
|
|
1595
|
+
const normalizedKey = waiter.key;
|
|
1596
|
+
const previous = recoverableRetryGateState.get(normalizedKey) ?? Promise.resolve();
|
|
1597
|
+
let release = () => undefined;
|
|
1598
|
+
const current = new Promise((resolve) => {
|
|
1599
|
+
release = resolve;
|
|
1600
|
+
});
|
|
1601
|
+
recoverableRetryGateState.set(normalizedKey, current);
|
|
1602
|
+
try {
|
|
1603
|
+
await previous.catch((error) => {
|
|
1604
|
+
logRequestExecutorNonBlockingError('waitRecoverableBackoffWithGlobalGate.previous', error, {
|
|
1605
|
+
key: normalizedKey
|
|
1606
|
+
});
|
|
1607
|
+
});
|
|
1608
|
+
await waitRecoverableBackoffMs(ms, signal);
|
|
1609
|
+
}
|
|
1610
|
+
finally {
|
|
1611
|
+
release();
|
|
1612
|
+
if (recoverableRetryGateState.get(normalizedKey) === current) {
|
|
1613
|
+
recoverableRetryGateState.delete(normalizedKey);
|
|
1614
|
+
}
|
|
1615
|
+
releaseRecoverableWaiterSlot(normalizedKey);
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
function deriveLogicalRequestChainKey(requestId) {
|
|
1619
|
+
const normalized = typeof requestId === 'string' ? requestId.trim() : '';
|
|
1620
|
+
if (!normalized) {
|
|
1621
|
+
return 'request-chain:unknown';
|
|
1622
|
+
}
|
|
1623
|
+
const root = normalized.split(':')[0]?.trim() || normalized;
|
|
1624
|
+
return root || 'request-chain:unknown';
|
|
1625
|
+
}
|
|
1626
|
+
function resolveLogicalChainRecoverableRetryLimit() {
|
|
1627
|
+
const raw = process.env.ROUTECODEX_LOGICAL_CHAIN_RECOVERABLE_RETRY_LIMIT
|
|
1628
|
+
?? process.env.RCC_LOGICAL_CHAIN_RECOVERABLE_RETRY_LIMIT
|
|
1629
|
+
?? '';
|
|
1630
|
+
const parsed = Number.parseInt(String(raw).trim(), 10);
|
|
1631
|
+
if (Number.isFinite(parsed) && parsed >= 1) {
|
|
1632
|
+
return parsed;
|
|
1633
|
+
}
|
|
1634
|
+
return 8;
|
|
1635
|
+
}
|
|
1636
|
+
function retainLogicalRequestChain(key) {
|
|
1637
|
+
const normalizedKey = key.trim() || 'request-chain:unknown';
|
|
1638
|
+
const now = Date.now();
|
|
1639
|
+
for (const [existingKey, state] of logicalChainRetryState.entries()) {
|
|
1640
|
+
if (state.activeExecutions <= 0 && now - state.updatedAtMs >= RECOVERABLE_BACKOFF_TTL_MS) {
|
|
1641
|
+
logicalChainRetryState.delete(existingKey);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
const current = logicalChainRetryState.get(normalizedKey);
|
|
1645
|
+
logicalChainRetryState.set(normalizedKey, {
|
|
1646
|
+
recoverableRetries: current?.recoverableRetries ?? 0,
|
|
1647
|
+
updatedAtMs: now,
|
|
1648
|
+
activeExecutions: (current?.activeExecutions ?? 0) + 1
|
|
1649
|
+
});
|
|
1650
|
+
return normalizedKey;
|
|
1651
|
+
}
|
|
1652
|
+
function releaseLogicalRequestChain(key) {
|
|
1653
|
+
const current = logicalChainRetryState.get(key);
|
|
1654
|
+
if (!current) {
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
const nextActiveExecutions = Math.max(0, current.activeExecutions - 1);
|
|
1658
|
+
if (nextActiveExecutions === 0) {
|
|
1659
|
+
logicalChainRetryState.delete(key);
|
|
1660
|
+
return;
|
|
1661
|
+
}
|
|
1662
|
+
logicalChainRetryState.set(key, {
|
|
1663
|
+
...current,
|
|
1664
|
+
activeExecutions: nextActiveExecutions,
|
|
1665
|
+
updatedAtMs: Date.now()
|
|
1666
|
+
});
|
|
1667
|
+
}
|
|
1668
|
+
function consumeLogicalChainRecoverableRetry(key) {
|
|
1669
|
+
const normalizedKey = key.trim() || 'request-chain:unknown';
|
|
1670
|
+
const limit = resolveLogicalChainRecoverableRetryLimit();
|
|
1671
|
+
const current = logicalChainRetryState.get(normalizedKey) ?? {
|
|
1672
|
+
recoverableRetries: 0,
|
|
1673
|
+
updatedAtMs: 0,
|
|
1674
|
+
activeExecutions: 0
|
|
1675
|
+
};
|
|
1676
|
+
const count = current.recoverableRetries + 1;
|
|
1677
|
+
const next = {
|
|
1678
|
+
...current,
|
|
1679
|
+
recoverableRetries: count,
|
|
1680
|
+
updatedAtMs: Date.now()
|
|
1681
|
+
};
|
|
1682
|
+
logicalChainRetryState.set(normalizedKey, next);
|
|
1683
|
+
return {
|
|
1684
|
+
allowed: count <= limit,
|
|
1685
|
+
count,
|
|
1686
|
+
limit
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
function resetRequestExecutorInternalStateForTests() {
|
|
1690
|
+
nonBlockingLogState.clear();
|
|
1691
|
+
requestDegradedLogState.clear();
|
|
1692
|
+
recoverableErrorBackoffState.clear();
|
|
1693
|
+
recoverableRetryGateState.clear();
|
|
1694
|
+
recoverableRetryWaiterState.clear();
|
|
1695
|
+
providerTransportBackoffState.clear();
|
|
1696
|
+
providerTransportBackoffGateState.clear();
|
|
1697
|
+
sessionStormBackoffState.clear();
|
|
1698
|
+
sessionStormBackoffGateState.clear();
|
|
1699
|
+
logicalChainRetryState.clear();
|
|
1700
|
+
providerSwitchLogState.clear();
|
|
1701
|
+
}
|
|
1702
|
+
function peekRecoverableRetryWaitersForTests(key) {
|
|
1703
|
+
const normalizedKey = key.trim() || 'recoverable:unknown';
|
|
1704
|
+
return recoverableRetryWaiterState.get(normalizedKey)?.activeWaiters ?? 0;
|
|
1705
|
+
}
|
|
217
1706
|
function resolveTrafficRuntimeProfile(runtimeKey, handle, providerKey) {
|
|
218
1707
|
const runtimeCandidate = handle.runtime;
|
|
219
1708
|
if (runtimeCandidate && typeof runtimeCandidate === 'object') {
|
|
@@ -247,6 +1736,127 @@ function resolveTrafficRuntimeProfile(runtimeKey, handle, providerKey) {
|
|
|
247
1736
|
}
|
|
248
1737
|
};
|
|
249
1738
|
}
|
|
1739
|
+
function normalizeRuntimeKey(value) {
|
|
1740
|
+
if (typeof value !== 'string') {
|
|
1741
|
+
return undefined;
|
|
1742
|
+
}
|
|
1743
|
+
const normalized = value.trim();
|
|
1744
|
+
return normalized || undefined;
|
|
1745
|
+
}
|
|
1746
|
+
function resolveRuntimeKeyForProvider(runtimeManager, providerKey) {
|
|
1747
|
+
return normalizeRuntimeKey(runtimeManager.resolveRuntimeKey(providerKey));
|
|
1748
|
+
}
|
|
1749
|
+
function applyRetryExclusionForCurrentProvider(args) {
|
|
1750
|
+
const providerKey = readString(args.providerKey);
|
|
1751
|
+
if (!providerKey) {
|
|
1752
|
+
return false;
|
|
1753
|
+
}
|
|
1754
|
+
args.excludedProviderKeys.add(providerKey);
|
|
1755
|
+
return true;
|
|
1756
|
+
}
|
|
1757
|
+
function isProviderTrafficSaturatedRetryError(args) {
|
|
1758
|
+
const code = normalizeCodeKey(args.error?.code);
|
|
1759
|
+
const upstreamCode = normalizeCodeKey(args.error?.upstreamCode);
|
|
1760
|
+
if (code === 'PROVIDER_TRAFFIC_SATURATED' || upstreamCode === 'PROVIDER_TRAFFIC_SATURATED') {
|
|
1761
|
+
return true;
|
|
1762
|
+
}
|
|
1763
|
+
return args.status === 429 && code === 'PROVIDER_TRAFFIC_SATURATED';
|
|
1764
|
+
}
|
|
1765
|
+
function resolveProviderRetryExclusionPlan(args) {
|
|
1766
|
+
const providerKey = readString(args.providerKey);
|
|
1767
|
+
let nextAntigravityRetrySignal = args.antigravityRetrySignal;
|
|
1768
|
+
if (!providerKey) {
|
|
1769
|
+
return {
|
|
1770
|
+
excludedCurrentProvider: false,
|
|
1771
|
+
antigravityRetrySignal: nextAntigravityRetrySignal
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
const isAntigravity = isAntigravityProviderKey(providerKey);
|
|
1775
|
+
const is429 = args.status === 429;
|
|
1776
|
+
const exclusionDecision = resolveProviderFailureExclusionDecision({
|
|
1777
|
+
promptTooLong: args.promptTooLong,
|
|
1778
|
+
classification: args.classification,
|
|
1779
|
+
isProviderTrafficSaturated: isProviderTrafficSaturatedRetryError({ status: args.status, error: args.error }),
|
|
1780
|
+
isNetworkTransport: isNetworkTransportLikeError(args.error),
|
|
1781
|
+
hasAlternativeCandidate: hasExplicitAlternativeRouteCandidate({
|
|
1782
|
+
providerKey,
|
|
1783
|
+
routePool: args.routePool,
|
|
1784
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1785
|
+
}),
|
|
1786
|
+
isAntigravity,
|
|
1787
|
+
is429,
|
|
1788
|
+
isVerify: args.isVerify,
|
|
1789
|
+
isReauth: args.isReauth,
|
|
1790
|
+
shouldRotateAntigravityAlias: shouldRotateAntigravityAliasOnRetry(args.error)
|
|
1791
|
+
});
|
|
1792
|
+
if (exclusionDecision.excludeCurrentProvider && isAntigravity && (args.isVerify || is429)) {
|
|
1793
|
+
const excludedCurrentProvider = applyRetryExclusionForCurrentProvider({
|
|
1794
|
+
providerKey,
|
|
1795
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1796
|
+
});
|
|
1797
|
+
nextAntigravityRetrySignal = nextAntigravityRetrySignal
|
|
1798
|
+
? { ...nextAntigravityRetrySignal, avoidAllOnRetry: true }
|
|
1799
|
+
: { signature: extractRetryErrorSignature(args.error), consecutive: 1, avoidAllOnRetry: true };
|
|
1800
|
+
return {
|
|
1801
|
+
excludedCurrentProvider,
|
|
1802
|
+
antigravityRetrySignal: nextAntigravityRetrySignal
|
|
1803
|
+
};
|
|
1804
|
+
}
|
|
1805
|
+
if (exclusionDecision.excludeCurrentProvider && isAntigravity && args.isReauth) {
|
|
1806
|
+
return {
|
|
1807
|
+
excludedCurrentProvider: applyRetryExclusionForCurrentProvider({
|
|
1808
|
+
providerKey,
|
|
1809
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1810
|
+
}),
|
|
1811
|
+
antigravityRetrySignal: nextAntigravityRetrySignal
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
if (exclusionDecision.excludeCurrentProvider) {
|
|
1815
|
+
return {
|
|
1816
|
+
excludedCurrentProvider: applyRetryExclusionForCurrentProvider({
|
|
1817
|
+
providerKey,
|
|
1818
|
+
excludedProviderKeys: args.excludedProviderKeys
|
|
1819
|
+
}),
|
|
1820
|
+
antigravityRetrySignal: nextAntigravityRetrySignal
|
|
1821
|
+
};
|
|
1822
|
+
}
|
|
1823
|
+
return {
|
|
1824
|
+
excludedCurrentProvider: false,
|
|
1825
|
+
antigravityRetrySignal: nextAntigravityRetrySignal
|
|
1826
|
+
};
|
|
1827
|
+
}
|
|
1828
|
+
function excludeProvidersSharingRuntimeFromRoutePool(args) {
|
|
1829
|
+
const currentRuntimeKey = normalizeRuntimeKey(args.runtimeKey);
|
|
1830
|
+
if (!currentRuntimeKey) {
|
|
1831
|
+
return [];
|
|
1832
|
+
}
|
|
1833
|
+
const added = [];
|
|
1834
|
+
for (const providerKey of args.routePool) {
|
|
1835
|
+
if (typeof providerKey !== 'string') {
|
|
1836
|
+
continue;
|
|
1837
|
+
}
|
|
1838
|
+
const normalizedProviderKey = providerKey.trim();
|
|
1839
|
+
if (!normalizedProviderKey) {
|
|
1840
|
+
continue;
|
|
1841
|
+
}
|
|
1842
|
+
const candidateRuntimeKey = resolveRuntimeKeyForProvider(args.runtimeManager, normalizedProviderKey);
|
|
1843
|
+
if (candidateRuntimeKey !== currentRuntimeKey) {
|
|
1844
|
+
continue;
|
|
1845
|
+
}
|
|
1846
|
+
if (args.excludedProviderKeys.has(normalizedProviderKey)) {
|
|
1847
|
+
continue;
|
|
1848
|
+
}
|
|
1849
|
+
args.excludedProviderKeys.add(normalizedProviderKey);
|
|
1850
|
+
added.push(normalizedProviderKey);
|
|
1851
|
+
}
|
|
1852
|
+
return added;
|
|
1853
|
+
}
|
|
1854
|
+
function hasExplicitAlternativeRouteCandidate(args) {
|
|
1855
|
+
if (!Array.isArray(args.routePool) || args.routePool.length === 0) {
|
|
1856
|
+
return false;
|
|
1857
|
+
}
|
|
1858
|
+
return hasAlternativeRouteCandidate(args);
|
|
1859
|
+
}
|
|
250
1860
|
function isRecord(value) {
|
|
251
1861
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
252
1862
|
}
|
|
@@ -318,26 +1928,19 @@ function backfillResponsesOutputTextIfMissing(body) {
|
|
|
318
1928
|
body.output_text = text;
|
|
319
1929
|
}
|
|
320
1930
|
function emitVirtualRouterConcurrencyLog(args) {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
const prefixColor = '\x1b[38;5;208m';
|
|
335
|
-
const timeColor = '\x1b[90m';
|
|
336
|
-
const routeColor = resolveSessionLogColor(args.sessionId);
|
|
337
|
-
const white = '\x1b[97m';
|
|
338
|
-
const reset = '\x1b[0m';
|
|
339
|
-
const concurrencyLabel = `${white}[concurrency:${Math.max(0, Math.floor(args.activeInFlight))}/${Math.max(1, Math.floor(args.maxInFlight))}]${reset}`;
|
|
340
|
-
console.log(`${prefixColor}[virtual-router-hit]${reset} ${concurrencyLabel} ${timeColor}${timestamp}${reset}${sessionLabel} ${routeColor}${routeLabel} -> ${providerLabel}${modelLabel}${reasonLabel}${reset}`);
|
|
1931
|
+
recordVirtualRouterHitRollup({
|
|
1932
|
+
routeName: args.routeName,
|
|
1933
|
+
poolId: args.poolId,
|
|
1934
|
+
providerKey: args.providerKey,
|
|
1935
|
+
model: args.model,
|
|
1936
|
+
sessionId: args.sessionId,
|
|
1937
|
+
projectPath: args.projectPath,
|
|
1938
|
+
reason: args.reason,
|
|
1939
|
+
stoplessMode: args.stoplessMode,
|
|
1940
|
+
stoplessArmed: args.stoplessArmed,
|
|
1941
|
+
activeInFlight: args.activeInFlight,
|
|
1942
|
+
maxInFlight: args.maxInFlight
|
|
1943
|
+
});
|
|
341
1944
|
}
|
|
342
1945
|
function hasNonEmptyToolCalls(value) {
|
|
343
1946
|
if (!Array.isArray(value) || value.length <= 0) {
|
|
@@ -378,11 +1981,280 @@ function containsToolRegistryMissingText(value) {
|
|
|
378
1981
|
}
|
|
379
1982
|
return false;
|
|
380
1983
|
}
|
|
381
|
-
function
|
|
382
|
-
|
|
1984
|
+
function shouldBypassProviderResponseConversion(normalized) {
|
|
1985
|
+
return typeof normalized.status === 'number' && normalized.status >= 400;
|
|
1986
|
+
}
|
|
1987
|
+
function resolveSessionStormBackoffScope(metadata) {
|
|
1988
|
+
const sessionId = readString(metadata.sessionId);
|
|
1989
|
+
if (sessionId) {
|
|
1990
|
+
return `session:${sessionId}`;
|
|
1991
|
+
}
|
|
1992
|
+
const conversationId = readString(metadata.conversationId);
|
|
1993
|
+
if (conversationId) {
|
|
1994
|
+
return `conversation:${conversationId}`;
|
|
1995
|
+
}
|
|
1996
|
+
const workdir = readString(metadata.clientWorkdir)
|
|
1997
|
+
?? readString(metadata.client_workdir)
|
|
1998
|
+
?? readString(metadata.workdir)
|
|
1999
|
+
?? readString(metadata.cwd);
|
|
2000
|
+
if (workdir) {
|
|
2001
|
+
return `workdir:${workdir}`;
|
|
2002
|
+
}
|
|
2003
|
+
return undefined;
|
|
2004
|
+
}
|
|
2005
|
+
function isSessionStormBackoffCandidate(error) {
|
|
2006
|
+
const codeSource = error && typeof error === 'object'
|
|
2007
|
+
? error.code
|
|
2008
|
+
: undefined;
|
|
2009
|
+
const code = normalizeCodeKey(codeSource);
|
|
2010
|
+
if (code === 'PROVIDER_NOT_AVAILABLE' || code === 'ERR_NO_PROVIDER_TARGET') {
|
|
2011
|
+
return true;
|
|
2012
|
+
}
|
|
2013
|
+
const status = extractStatusCodeFromError(error);
|
|
2014
|
+
if (status === 429 || status === 502 || status === 503 || status === 504) {
|
|
2015
|
+
return true;
|
|
2016
|
+
}
|
|
2017
|
+
if (isNetworkTransportLikeError(error)) {
|
|
2018
|
+
return true;
|
|
2019
|
+
}
|
|
2020
|
+
const message = error instanceof Error
|
|
2021
|
+
? error.message
|
|
2022
|
+
: error && typeof error === 'object' && typeof error.message === 'string'
|
|
2023
|
+
? String(error.message)
|
|
2024
|
+
: String(error ?? '');
|
|
2025
|
+
const normalized = message.trim().toLowerCase();
|
|
2026
|
+
return (normalized.includes('fetch failed')
|
|
2027
|
+
|| normalized.includes('all providers unavailable')
|
|
2028
|
+
|| normalized.includes('no available providers after applying routing instructions')
|
|
2029
|
+
|| normalized.includes('connect timeout')
|
|
2030
|
+
|| normalized.includes('request timeout'));
|
|
2031
|
+
}
|
|
2032
|
+
function resolveSessionStormBackoffBaseMs() {
|
|
2033
|
+
const raw = process.env.ROUTECODEX_SESSION_STORM_BACKOFF_BASE_MS
|
|
2034
|
+
?? process.env.RCC_SESSION_STORM_BACKOFF_BASE_MS
|
|
2035
|
+
?? '';
|
|
2036
|
+
const parsed = Number.parseInt(String(raw).trim(), 10);
|
|
2037
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
2038
|
+
return parsed;
|
|
2039
|
+
}
|
|
2040
|
+
return process.env.NODE_ENV === 'test' ? 200 : 1_000;
|
|
2041
|
+
}
|
|
2042
|
+
function resolveSessionStormBackoffMaxMs() {
|
|
2043
|
+
const raw = process.env.ROUTECODEX_SESSION_STORM_BACKOFF_MAX_MS
|
|
2044
|
+
?? process.env.RCC_SESSION_STORM_BACKOFF_MAX_MS
|
|
2045
|
+
?? '';
|
|
2046
|
+
const parsed = Number.parseInt(String(raw).trim(), 10);
|
|
2047
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
2048
|
+
return parsed;
|
|
2049
|
+
}
|
|
2050
|
+
return process.env.NODE_ENV === 'test' ? 5_000 : 30_000;
|
|
2051
|
+
}
|
|
2052
|
+
function consumeSessionStormBackoffMs(key) {
|
|
2053
|
+
const now = Date.now();
|
|
2054
|
+
for (const [existingKey, state] of sessionStormBackoffState.entries()) {
|
|
2055
|
+
if (now - state.updatedAtMs >= SESSION_STORM_BACKOFF_TTL_MS) {
|
|
2056
|
+
sessionStormBackoffState.delete(existingKey);
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
const previous = sessionStormBackoffState.get(key);
|
|
2060
|
+
const consecutive = previous && now - previous.updatedAtMs < SESSION_STORM_BACKOFF_TTL_MS
|
|
2061
|
+
? Math.min(previous.consecutive + 1, 16)
|
|
2062
|
+
: 1;
|
|
2063
|
+
const delayMs = Math.min(resolveSessionStormBackoffMaxMs(), resolveSessionStormBackoffBaseMs() * Math.pow(2, Math.max(0, consecutive - 1)));
|
|
2064
|
+
sessionStormBackoffState.set(key, {
|
|
2065
|
+
consecutive,
|
|
2066
|
+
updatedAtMs: now,
|
|
2067
|
+
nextAllowedAtMs: now + delayMs
|
|
2068
|
+
});
|
|
2069
|
+
return delayMs;
|
|
2070
|
+
}
|
|
2071
|
+
function peekSessionStormBackoffWaitMs(key) {
|
|
2072
|
+
const state = sessionStormBackoffState.get(key);
|
|
2073
|
+
if (!state) {
|
|
2074
|
+
return 0;
|
|
2075
|
+
}
|
|
2076
|
+
const now = Date.now();
|
|
2077
|
+
if (now - state.updatedAtMs >= SESSION_STORM_BACKOFF_TTL_MS) {
|
|
2078
|
+
sessionStormBackoffState.delete(key);
|
|
2079
|
+
return 0;
|
|
2080
|
+
}
|
|
2081
|
+
return Math.max(0, state.nextAllowedAtMs - now);
|
|
2082
|
+
}
|
|
2083
|
+
function clearSessionStormBackoff(key) {
|
|
2084
|
+
if (!key) {
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
sessionStormBackoffState.delete(key);
|
|
2088
|
+
}
|
|
2089
|
+
async function waitSessionStormBackoffWithGate(key, ms, signal) {
|
|
2090
|
+
if (!(ms > 0)) {
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
const normalizedKey = key.trim() || 'session:unknown';
|
|
2094
|
+
const previous = sessionStormBackoffGateState.get(normalizedKey) ?? Promise.resolve();
|
|
2095
|
+
let release = () => undefined;
|
|
2096
|
+
const current = new Promise((resolve) => {
|
|
2097
|
+
release = resolve;
|
|
2098
|
+
});
|
|
2099
|
+
sessionStormBackoffGateState.set(normalizedKey, current);
|
|
2100
|
+
try {
|
|
2101
|
+
await previous.catch((error) => {
|
|
2102
|
+
logRequestExecutorNonBlockingError('waitSessionStormBackoffWithGate.previous', error, {
|
|
2103
|
+
key: normalizedKey
|
|
2104
|
+
});
|
|
2105
|
+
});
|
|
2106
|
+
await waitWithClientAbortSignal(ms, signal);
|
|
2107
|
+
}
|
|
2108
|
+
finally {
|
|
2109
|
+
release();
|
|
2110
|
+
if (sessionStormBackoffGateState.get(normalizedKey) === current) {
|
|
2111
|
+
sessionStormBackoffGateState.delete(normalizedKey);
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2115
|
+
function detectRetryableEmptyAssistantResponse(body) {
|
|
2116
|
+
if (!isRecord(body)) {
|
|
2117
|
+
return null;
|
|
2118
|
+
}
|
|
2119
|
+
if (Object.prototype.hasOwnProperty.call(body, '__sse_responses')) {
|
|
2120
|
+
return null;
|
|
2121
|
+
}
|
|
2122
|
+
const choices = Array.isArray(body.choices) ? body.choices : [];
|
|
2123
|
+
if (choices.length > 0) {
|
|
2124
|
+
const firstChoice = isRecord(choices[0]) ? choices[0] : undefined;
|
|
2125
|
+
if (!firstChoice) {
|
|
2126
|
+
return null;
|
|
2127
|
+
}
|
|
2128
|
+
const finishReason = readString(firstChoice.finish_reason)?.toLowerCase() ?? '';
|
|
2129
|
+
const message = isRecord(firstChoice.message) ? firstChoice.message : undefined;
|
|
2130
|
+
const hasToolCalls = hasNonEmptyToolCalls(message?.tool_calls);
|
|
2131
|
+
const hasText = valueHasNonEmptyText(message?.content)
|
|
2132
|
+
|| valueHasNonEmptyText(message?.reasoning_content)
|
|
2133
|
+
|| valueHasNonEmptyText(message?.reasoning)
|
|
2134
|
+
|| valueHasNonEmptyText(firstChoice.content);
|
|
2135
|
+
const combinedText = [
|
|
2136
|
+
message?.content,
|
|
2137
|
+
message?.reasoning_content,
|
|
2138
|
+
message?.reasoning,
|
|
2139
|
+
firstChoice.content
|
|
2140
|
+
]
|
|
2141
|
+
.filter((item) => valueHasNonEmptyText(item))
|
|
2142
|
+
.map((item) => String(item))
|
|
2143
|
+
.join('\n');
|
|
2144
|
+
if ((finishReason === 'stop' || finishReason === 'tool_calls' || !finishReason) && !hasToolCalls && !hasText) {
|
|
2145
|
+
return {
|
|
2146
|
+
reason: `finish_reason=${finishReason || 'unknown'} but assistant text/tool_calls are empty`,
|
|
2147
|
+
marker: 'chat_empty_assistant'
|
|
2148
|
+
};
|
|
2149
|
+
}
|
|
2150
|
+
if ((finishReason === 'stop' || finishReason === 'tool_calls' || !finishReason) && !hasToolCalls && containsToolRegistryMissingText(combinedText)) {
|
|
2151
|
+
return {
|
|
2152
|
+
reason: 'assistant emitted textual tool-not-found complaint without structured tool_calls',
|
|
2153
|
+
marker: 'chat_textual_tool_registry_missing'
|
|
2154
|
+
};
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
const status = readString(body.status)?.toLowerCase() ?? '';
|
|
2158
|
+
if (status === 'completed' || status === 'stop') {
|
|
2159
|
+
const requiredAction = isRecord(body.required_action) ? body.required_action : undefined;
|
|
2160
|
+
const submitToolOutputs = requiredAction && isRecord(requiredAction.submit_tool_outputs)
|
|
2161
|
+
? requiredAction.submit_tool_outputs
|
|
2162
|
+
: undefined;
|
|
2163
|
+
const hasRequiredActionToolCalls = hasNonEmptyToolCalls(submitToolOutputs?.tool_calls);
|
|
2164
|
+
const hasFunctionCalls = hasOutputFunctionCalls(body.output);
|
|
2165
|
+
const hasText = valueHasNonEmptyText(body.output_text)
|
|
2166
|
+
|| valueHasNonEmptyText(body.output)
|
|
2167
|
+
|| valueHasNonEmptyText(body.reasoning);
|
|
2168
|
+
if (!hasRequiredActionToolCalls && !hasFunctionCalls && !hasText) {
|
|
2169
|
+
return {
|
|
2170
|
+
reason: `responses status=${status} but output text/tool_calls are empty`,
|
|
2171
|
+
marker: 'responses_empty_output'
|
|
2172
|
+
};
|
|
2173
|
+
}
|
|
2174
|
+
if (!hasRequiredActionToolCalls &&
|
|
2175
|
+
!hasFunctionCalls &&
|
|
2176
|
+
containsToolRegistryMissingText(body.output_text)) {
|
|
2177
|
+
return {
|
|
2178
|
+
reason: 'responses completed with textual tool-not-found complaint but no function_call output',
|
|
2179
|
+
marker: 'responses_textual_tool_registry_missing'
|
|
2180
|
+
};
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
return null;
|
|
2184
|
+
}
|
|
2185
|
+
async function persistEmptyAssistantProviderSnapshots(args) {
|
|
2186
|
+
const requestPayload = args.providerRequestPayload && typeof args.providerRequestPayload === 'object'
|
|
2187
|
+
? args.providerRequestPayload
|
|
2188
|
+
: { payload: args.providerRequestPayload };
|
|
2189
|
+
await writeProviderSnapshot({
|
|
2190
|
+
phase: 'provider-request',
|
|
2191
|
+
requestId: args.requestId,
|
|
2192
|
+
clientRequestId: args.requestId,
|
|
2193
|
+
entryEndpoint: args.entryEndpoint,
|
|
2194
|
+
providerKey: args.providerKey,
|
|
2195
|
+
providerId: args.providerId,
|
|
2196
|
+
headers: args.providerRequestHeaders,
|
|
2197
|
+
url: args.providerRequestUrl,
|
|
2198
|
+
data: requestPayload,
|
|
2199
|
+
forceLocalDiskWriteWhenDisabled: true
|
|
2200
|
+
});
|
|
2201
|
+
await writeProviderSnapshot({
|
|
2202
|
+
phase: 'provider-response',
|
|
2203
|
+
requestId: args.requestId,
|
|
2204
|
+
clientRequestId: args.requestId,
|
|
2205
|
+
entryEndpoint: args.entryEndpoint,
|
|
2206
|
+
providerKey: args.providerKey,
|
|
2207
|
+
providerId: args.providerId,
|
|
2208
|
+
headers: args.normalizedResponse.headers && typeof args.normalizedResponse.headers === 'object'
|
|
2209
|
+
? args.normalizedResponse.headers
|
|
2210
|
+
: undefined,
|
|
2211
|
+
url: typeof args.normalizedResponse.url === 'string'
|
|
2212
|
+
? String(args.normalizedResponse.url)
|
|
2213
|
+
: args.providerRequestUrl,
|
|
2214
|
+
data: {
|
|
2215
|
+
emptyAssistantSignal: args.emptyAssistantSignal,
|
|
2216
|
+
normalizedResponse: {
|
|
2217
|
+
status: args.normalizedResponse.status ?? null,
|
|
2218
|
+
headers: args.normalizedResponse.headers ?? null,
|
|
2219
|
+
body: args.normalizedResponse.body ?? null
|
|
2220
|
+
},
|
|
2221
|
+
convertedResponse: {
|
|
2222
|
+
status: args.convertedResponse.status ?? null,
|
|
2223
|
+
headers: args.convertedResponse.headers ?? null,
|
|
2224
|
+
body: args.convertedResponse.body ?? null
|
|
2225
|
+
}
|
|
2226
|
+
},
|
|
2227
|
+
forceLocalDiskWriteWhenDisabled: true
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
function bodyContainsReasoningStopFinalizedMarker(body) {
|
|
2231
|
+
if (!body || typeof body !== 'object') {
|
|
2232
|
+
return false;
|
|
2233
|
+
}
|
|
2234
|
+
try {
|
|
2235
|
+
return JSON.stringify(body).includes(REASONING_STOP_FINALIZED_MARKER);
|
|
2236
|
+
}
|
|
2237
|
+
catch (error) {
|
|
2238
|
+
logRequestExecutorNonBlockingError('bodyContainsReasoningStopFinalizedMarker.stringify', error);
|
|
2239
|
+
return false;
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
function detectStoplessTerminationWithoutFinalization(body, stoplessMode) {
|
|
2243
|
+
if ((stoplessMode !== 'on' && stoplessMode !== 'endless') || !isRecord(body)) {
|
|
383
2244
|
return null;
|
|
384
2245
|
}
|
|
385
2246
|
if (Object.prototype.hasOwnProperty.call(body, '__sse_responses')) {
|
|
2247
|
+
const finishReason = readString(body[STREAM_LOG_FINISH_REASON_KEY])?.toLowerCase() ?? '';
|
|
2248
|
+
const finalized = body[REASONING_STOP_FINALIZED_FLAG_KEY] === true;
|
|
2249
|
+
if (!finalized && finishReason === 'stop') {
|
|
2250
|
+
return {
|
|
2251
|
+
reason: `stopless=${stoplessMode} but streamed wrapper completed with finish_reason=stop without reasoning.stop finalized marker`,
|
|
2252
|
+
marker: 'stream_wrapper_stopless_missing_reasoning_stop_finalization'
|
|
2253
|
+
};
|
|
2254
|
+
}
|
|
2255
|
+
return null;
|
|
2256
|
+
}
|
|
2257
|
+
if (bodyContainsReasoningStopFinalizedMarker(body)) {
|
|
386
2258
|
return null;
|
|
387
2259
|
}
|
|
388
2260
|
const choices = Array.isArray(body.choices) ? body.choices : [];
|
|
@@ -394,29 +2266,10 @@ function detectRetryableEmptyAssistantResponse(body) {
|
|
|
394
2266
|
const finishReason = readString(firstChoice.finish_reason)?.toLowerCase() ?? '';
|
|
395
2267
|
const message = isRecord(firstChoice.message) ? firstChoice.message : undefined;
|
|
396
2268
|
const hasToolCalls = hasNonEmptyToolCalls(message?.tool_calls);
|
|
397
|
-
|
|
398
|
-
|| valueHasNonEmptyText(message?.reasoning_content)
|
|
399
|
-
|| valueHasNonEmptyText(message?.reasoning)
|
|
400
|
-
|| valueHasNonEmptyText(firstChoice.content);
|
|
401
|
-
const combinedText = [
|
|
402
|
-
message?.content,
|
|
403
|
-
message?.reasoning_content,
|
|
404
|
-
message?.reasoning,
|
|
405
|
-
firstChoice.content
|
|
406
|
-
]
|
|
407
|
-
.filter((item) => valueHasNonEmptyText(item))
|
|
408
|
-
.map((item) => String(item))
|
|
409
|
-
.join('\n');
|
|
410
|
-
if ((finishReason === 'stop' || finishReason === 'tool_calls' || !finishReason) && !hasToolCalls && !hasText) {
|
|
411
|
-
return {
|
|
412
|
-
reason: `finish_reason=${finishReason || 'unknown'} but assistant text/tool_calls are empty`,
|
|
413
|
-
marker: 'chat_empty_assistant'
|
|
414
|
-
};
|
|
415
|
-
}
|
|
416
|
-
if ((finishReason === 'stop' || finishReason === 'tool_calls' || !finishReason) && !hasToolCalls && containsToolRegistryMissingText(combinedText)) {
|
|
2269
|
+
if (finishReason === 'stop' && !hasToolCalls) {
|
|
417
2270
|
return {
|
|
418
|
-
reason:
|
|
419
|
-
marker: '
|
|
2271
|
+
reason: `stopless=${stoplessMode} but chat completion stopped without reasoning.stop finalized marker`,
|
|
2272
|
+
marker: 'chat_stopless_missing_reasoning_stop_finalization'
|
|
420
2273
|
};
|
|
421
2274
|
}
|
|
422
2275
|
}
|
|
@@ -428,21 +2281,10 @@ function detectRetryableEmptyAssistantResponse(body) {
|
|
|
428
2281
|
: undefined;
|
|
429
2282
|
const hasRequiredActionToolCalls = hasNonEmptyToolCalls(submitToolOutputs?.tool_calls);
|
|
430
2283
|
const hasFunctionCalls = hasOutputFunctionCalls(body.output);
|
|
431
|
-
|
|
432
|
-
|| valueHasNonEmptyText(body.output)
|
|
433
|
-
|| valueHasNonEmptyText(body.reasoning);
|
|
434
|
-
if (!hasRequiredActionToolCalls && !hasFunctionCalls && !hasText) {
|
|
435
|
-
return {
|
|
436
|
-
reason: `responses status=${status} but output text/tool_calls are empty`,
|
|
437
|
-
marker: 'responses_empty_output'
|
|
438
|
-
};
|
|
439
|
-
}
|
|
440
|
-
if (!hasRequiredActionToolCalls &&
|
|
441
|
-
!hasFunctionCalls &&
|
|
442
|
-
containsToolRegistryMissingText(body.output_text)) {
|
|
2284
|
+
if (!hasRequiredActionToolCalls && !hasFunctionCalls) {
|
|
443
2285
|
return {
|
|
444
|
-
reason:
|
|
445
|
-
marker: '
|
|
2286
|
+
reason: `stopless=${stoplessMode} but responses output completed without reasoning.stop finalized marker`,
|
|
2287
|
+
marker: 'responses_stopless_missing_reasoning_stop_finalization'
|
|
446
2288
|
};
|
|
447
2289
|
}
|
|
448
2290
|
}
|
|
@@ -461,17 +2303,53 @@ export class HubRequestExecutor {
|
|
|
461
2303
|
this.trafficGovernor = createNoopProviderTrafficGovernor();
|
|
462
2304
|
return;
|
|
463
2305
|
}
|
|
2306
|
+
const disableTrafficGovernor = process.env.ROUTECODEX_PROVIDER_TRAFFIC_NOOP === '1'
|
|
2307
|
+
|| process.env.RCC_PROVIDER_TRAFFIC_NOOP === '1';
|
|
2308
|
+
if (disableTrafficGovernor) {
|
|
2309
|
+
this.trafficGovernor = createNoopProviderTrafficGovernor();
|
|
2310
|
+
return;
|
|
2311
|
+
}
|
|
464
2312
|
this.trafficGovernor = getSharedProviderTrafficGovernor();
|
|
465
2313
|
}
|
|
466
2314
|
logProviderRetrySwitch(args) {
|
|
2315
|
+
const now = Date.now();
|
|
467
2316
|
const providerLabel = args.providerKey || 'unknown-provider';
|
|
468
|
-
const
|
|
2317
|
+
const dedupeKey = [
|
|
2318
|
+
providerLabel,
|
|
2319
|
+
args.switchAction,
|
|
2320
|
+
typeof args.statusCode === 'number' ? String(args.statusCode) : 'none',
|
|
2321
|
+
args.errorCode || 'none',
|
|
2322
|
+
args.upstreamCode || 'none',
|
|
2323
|
+
truncateReason(args.reason, 96)
|
|
2324
|
+
].join('|');
|
|
2325
|
+
const prior = providerSwitchLogState.get(dedupeKey);
|
|
2326
|
+
if (prior && now - prior.lastAtMs < PROVIDER_SWITCH_LOG_THROTTLE_MS) {
|
|
2327
|
+
prior.suppressed += 1;
|
|
2328
|
+
prior.lastAtMs = now;
|
|
2329
|
+
providerSwitchLogState.set(dedupeKey, prior);
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
if (prior?.suppressed && prior.suppressed > 0) {
|
|
2333
|
+
console.warn(`[provider-switch] aggregated key=${JSON.stringify(dedupeKey)} suppressed=${prior.suppressed} ` +
|
|
2334
|
+
`windowMs=${PROVIDER_SWITCH_LOG_THROTTLE_MS}`);
|
|
2335
|
+
}
|
|
2336
|
+
providerSwitchLogState.set(dedupeKey, { lastAtMs: now, suppressed: 0 });
|
|
2337
|
+
const boundedNextAttempt = Math.max(args.attempt, Math.min(args.maxAttempts, args.nextAttempt));
|
|
2338
|
+
const retryTag = `[provider-switch] req=${args.requestId} attempt=${args.attempt}/${args.maxAttempts} -> ` +
|
|
2339
|
+
`${boundedNextAttempt}/${args.maxAttempts}`;
|
|
469
2340
|
const details = [
|
|
470
2341
|
`provider=${providerLabel}`,
|
|
471
2342
|
`switch=${args.switchAction}`,
|
|
2343
|
+
...(args.decisionLabel ? [`decision=${args.decisionLabel}`] : []),
|
|
2344
|
+
...(args.backoffScope ? [`backoffScope=${args.backoffScope}`] : []),
|
|
2345
|
+
...(args.stage ? [`stage=${args.stage}`] : []),
|
|
472
2346
|
...(typeof args.statusCode === 'number' ? [`status=${args.statusCode}`] : []),
|
|
473
2347
|
...(args.errorCode ? [`code=${args.errorCode}`] : []),
|
|
474
2348
|
...(args.upstreamCode ? [`upstreamCode=${args.upstreamCode}`] : []),
|
|
2349
|
+
...(typeof args.backoffMs === 'number' ? [`backoff=${Math.max(0, Math.round(args.backoffMs))}ms`] : []),
|
|
2350
|
+
...(typeof args.runtimeScopeExcludedCount === 'number' && args.runtimeScopeExcludedCount > 0
|
|
2351
|
+
? [`runtimeScopeExcluded=${args.runtimeScopeExcludedCount}`]
|
|
2352
|
+
: []),
|
|
475
2353
|
`reason=${JSON.stringify(truncateReason(args.reason))}`
|
|
476
2354
|
];
|
|
477
2355
|
console.warn(`${retryTag} ${details.join(' ')}`);
|
|
@@ -480,6 +2358,15 @@ export class HubRequestExecutor {
|
|
|
480
2358
|
// Stats must remain stable across provider retries and requestId enhancements.
|
|
481
2359
|
const statsRequestId = input.requestId;
|
|
482
2360
|
const executorRequestId = input.requestId;
|
|
2361
|
+
const logicalRequestChainKey = retainLogicalRequestChain(deriveLogicalRequestChainKey(executorRequestId));
|
|
2362
|
+
let logicalRequestChainReleased = false;
|
|
2363
|
+
const releaseLogicalRequestChainIfNeeded = () => {
|
|
2364
|
+
if (logicalRequestChainReleased) {
|
|
2365
|
+
return;
|
|
2366
|
+
}
|
|
2367
|
+
logicalRequestChainReleased = true;
|
|
2368
|
+
releaseLogicalRequestChain(logicalRequestChainKey);
|
|
2369
|
+
};
|
|
483
2370
|
this.deps.stats.recordRequestStart(statsRequestId);
|
|
484
2371
|
const requestStartedAt = Date.now();
|
|
485
2372
|
let recordedAnyAttempt = false;
|
|
@@ -500,6 +2387,21 @@ export class HubRequestExecutor {
|
|
|
500
2387
|
const inboundClientHeaders = cloneClientHeaders(initialMetadata?.clientHeaders);
|
|
501
2388
|
const providerRequestId = input.requestId;
|
|
502
2389
|
const clientRequestId = resolveClientRequestId(initialMetadata, providerRequestId);
|
|
2390
|
+
const sessionStormBackoffScope = resolveSessionStormBackoffScope(initialMetadata);
|
|
2391
|
+
if (sessionStormBackoffScope) {
|
|
2392
|
+
const pendingSessionStormWaitMs = peekSessionStormBackoffWaitMs(sessionStormBackoffScope);
|
|
2393
|
+
if (pendingSessionStormWaitMs > 0) {
|
|
2394
|
+
this.logStage('request.session_storm_backoff_wait', providerRequestId, {
|
|
2395
|
+
scope: sessionStormBackoffScope,
|
|
2396
|
+
waitMs: pendingSessionStormWaitMs
|
|
2397
|
+
});
|
|
2398
|
+
await waitSessionStormBackoffWithGate(sessionStormBackoffScope, pendingSessionStormWaitMs, getClientConnectionAbortSignal(initialMetadata));
|
|
2399
|
+
this.logStage('request.session_storm_backoff_wait.completed', providerRequestId, {
|
|
2400
|
+
scope: sessionStormBackoffScope,
|
|
2401
|
+
waitMs: pendingSessionStormWaitMs
|
|
2402
|
+
});
|
|
2403
|
+
}
|
|
2404
|
+
}
|
|
503
2405
|
this.logStage('request.received', providerRequestId, {
|
|
504
2406
|
endpoint: input.entryEndpoint,
|
|
505
2407
|
stream: initialMetadata.stream === true
|
|
@@ -515,7 +2417,7 @@ export class HubRequestExecutor {
|
|
|
515
2417
|
let aggregatedUsage;
|
|
516
2418
|
const excludedProviderKeys = new Set();
|
|
517
2419
|
let maxAttempts = resolveMaxProviderAttempts();
|
|
518
|
-
const
|
|
2420
|
+
const retryPayloadSeed = prepareRequestPayloadRetrySeed(input.body);
|
|
519
2421
|
let attempt = 0;
|
|
520
2422
|
let lastError;
|
|
521
2423
|
let initialRoutePool = null;
|
|
@@ -523,18 +2425,23 @@ export class HubRequestExecutor {
|
|
|
523
2425
|
let poolCooldownWaitBudgetMs = 60 * 1000;
|
|
524
2426
|
let forcedRouteHint;
|
|
525
2427
|
let contextOverflowRetries = 0;
|
|
526
|
-
|
|
2428
|
+
let cumulativeExternalLatencyMs = 0;
|
|
2429
|
+
let cumulativeTrafficWaitMs = 0;
|
|
2430
|
+
let cumulativeClientInjectWaitMs = 0;
|
|
527
2431
|
while (attempt < maxAttempts) {
|
|
528
2432
|
attempt += 1;
|
|
529
2433
|
// Ensure each attempt starts from the base requestId so pipeline snapshots
|
|
530
2434
|
// don't inherit a provider-specific id from a previous attempt.
|
|
531
2435
|
input.requestId = providerRequestId;
|
|
532
|
-
if (
|
|
533
|
-
const cloned =
|
|
534
|
-
|
|
535
|
-
|
|
2436
|
+
if (attempt > 1 && retryPayloadSeed.mode !== 'none') {
|
|
2437
|
+
const cloned = restoreRequestPayloadFromRetrySeed(retryPayloadSeed);
|
|
2438
|
+
if (cloned && typeof cloned === 'object') {
|
|
2439
|
+
input.body = cloned;
|
|
2440
|
+
}
|
|
536
2441
|
}
|
|
537
2442
|
const metadataForAttempt = decorateMetadataForAttempt(initialMetadata, attempt, excludedProviderKeys);
|
|
2443
|
+
const clientAbortSignal = getClientConnectionAbortSignal(metadataForAttempt);
|
|
2444
|
+
throwIfClientAbortSignalAborted(clientAbortSignal);
|
|
538
2445
|
if (forcedRouteHint) {
|
|
539
2446
|
metadataForAttempt.routeHint = forcedRouteHint;
|
|
540
2447
|
}
|
|
@@ -595,7 +2502,7 @@ export class HubRequestExecutor {
|
|
|
595
2502
|
reason: 'provider_pool_cooling_down'
|
|
596
2503
|
});
|
|
597
2504
|
poolCooldownWaitBudgetMs -= cooldownWaitMs;
|
|
598
|
-
await
|
|
2505
|
+
await waitWithClientAbortSignal(cooldownWaitMs, clientAbortSignal);
|
|
599
2506
|
attempt = Math.max(0, attempt - 1);
|
|
600
2507
|
continue;
|
|
601
2508
|
}
|
|
@@ -607,6 +2514,7 @@ export class HubRequestExecutor {
|
|
|
607
2514
|
}
|
|
608
2515
|
const pipelineMetadata = pipelineResult.metadata ?? {};
|
|
609
2516
|
const mergedMetadata = mergeMetadataPreservingDefined(metadataForAttempt, pipelineMetadata);
|
|
2517
|
+
throwIfClientAbortSignalAborted(clientAbortSignal);
|
|
610
2518
|
registerRequestLogContext(input.requestId, {
|
|
611
2519
|
sessionId: mergedMetadata.sessionId,
|
|
612
2520
|
conversationId: mergedMetadata.conversationId
|
|
@@ -636,6 +2544,36 @@ export class HubRequestExecutor {
|
|
|
636
2544
|
requestId: input.requestId
|
|
637
2545
|
});
|
|
638
2546
|
}
|
|
2547
|
+
if (excludedProviderKeys.has(target.providerKey)) {
|
|
2548
|
+
const reselectedExcludedPlan = resolveExcludedProviderReselectionPlan({
|
|
2549
|
+
providerKey: target.providerKey,
|
|
2550
|
+
routePool: routePoolForAttempt,
|
|
2551
|
+
excludedProviderKeys,
|
|
2552
|
+
lastError
|
|
2553
|
+
});
|
|
2554
|
+
this.logStage('provider.retry.excluded_target_reselected', providerRequestId, {
|
|
2555
|
+
providerKey: target.providerKey,
|
|
2556
|
+
excluded: Array.from(excludedProviderKeys),
|
|
2557
|
+
attempt,
|
|
2558
|
+
hasAlternativeCandidate: reselectedExcludedPlan.hasAlternativeCandidate
|
|
2559
|
+
});
|
|
2560
|
+
if (!reselectedExcludedPlan.keepExcludedForNextAttempt) {
|
|
2561
|
+
excludedProviderKeys.delete(target.providerKey);
|
|
2562
|
+
}
|
|
2563
|
+
else {
|
|
2564
|
+
if (reselectedExcludedPlan.hasAlternativeCandidate) {
|
|
2565
|
+
continue;
|
|
2566
|
+
}
|
|
2567
|
+
if (lastError) {
|
|
2568
|
+
throw lastError;
|
|
2569
|
+
}
|
|
2570
|
+
throw Object.assign(new Error(`Virtual router reselected excluded provider ${target.providerKey}`), {
|
|
2571
|
+
code: 'ERR_EXCLUDED_PROVIDER_RESELECTED',
|
|
2572
|
+
requestId: input.requestId,
|
|
2573
|
+
providerKey: target.providerKey
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
639
2577
|
// Ensure response-side conversion always uses the route-selected target metadata.
|
|
640
2578
|
// ServerTool followups may carry stale metadata from the previous hop; response compat
|
|
641
2579
|
// must follow the current target/provider, not the inherited request profile.
|
|
@@ -646,7 +2584,7 @@ export class HubRequestExecutor {
|
|
|
646
2584
|
else if (Object.prototype.hasOwnProperty.call(mergedMetadata, 'compatibilityProfile')) {
|
|
647
2585
|
delete mergedMetadata.compatibilityProfile;
|
|
648
2586
|
}
|
|
649
|
-
let runtimeKey;
|
|
2587
|
+
let runtimeKey = typeof target.runtimeKey === 'string' ? target.runtimeKey : '';
|
|
650
2588
|
let handle;
|
|
651
2589
|
let providerContext;
|
|
652
2590
|
try {
|
|
@@ -670,7 +2608,6 @@ export class HubRequestExecutor {
|
|
|
670
2608
|
runtimeKey = resolved.runtimeKey;
|
|
671
2609
|
handle = resolved.handle;
|
|
672
2610
|
this.logStage('provider.runtime_resolve.completed', providerRequestId, {
|
|
673
|
-
providerKey: target.providerKey,
|
|
674
2611
|
runtimeKey,
|
|
675
2612
|
providerType: handle.providerType,
|
|
676
2613
|
providerFamily: handle.providerFamily,
|
|
@@ -705,49 +2642,66 @@ export class HubRequestExecutor {
|
|
|
705
2642
|
...(retryError.upstreamCode ? { upstreamCode: retryError.upstreamCode } : {}),
|
|
706
2643
|
attempt
|
|
707
2644
|
});
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
recordAttempt({ error: true });
|
|
712
|
-
throw error;
|
|
713
|
-
}
|
|
714
|
-
recordAttempt({ error: true });
|
|
715
|
-
const retryBackoffMs = await waitBeforeRetry(error, { attempt });
|
|
716
|
-
const singleProviderPool = Boolean(initialRoutePool && initialRoutePool.length === 1 && initialRoutePool[0] === target.providerKey);
|
|
717
|
-
if (!singleProviderPool && target.providerKey) {
|
|
718
|
-
excludedProviderKeys.add(target.providerKey);
|
|
719
|
-
}
|
|
720
|
-
const switchAction = singleProviderPool ? 'retry_same_provider' : 'exclude_and_reroute';
|
|
721
|
-
this.logProviderRetrySwitch({
|
|
2645
|
+
const providerFailurePlan = await resolveRequestExecutorProviderFailurePlan({
|
|
2646
|
+
error,
|
|
2647
|
+
retryError,
|
|
722
2648
|
requestId: providerRequestId,
|
|
723
|
-
attempt,
|
|
724
|
-
maxAttempts,
|
|
725
|
-
providerKey: target.providerKey,
|
|
726
|
-
nextAttempt: attempt + 1,
|
|
727
|
-
reason: retryError.reason,
|
|
728
|
-
statusCode: retryError.statusCode,
|
|
729
|
-
errorCode: retryError.errorCode,
|
|
730
|
-
upstreamCode: retryError.upstreamCode,
|
|
731
|
-
switchAction
|
|
732
|
-
});
|
|
733
|
-
this.logStage('provider.retry', input.requestId, {
|
|
734
2649
|
providerKey: target.providerKey,
|
|
2650
|
+
providerType: typeof target.providerType === 'string'
|
|
2651
|
+
? String(target.providerType)
|
|
2652
|
+
: undefined,
|
|
2653
|
+
providerProtocol: target.outboundProfile,
|
|
2654
|
+
routeName: pipelineResult.routingDecision?.routeName,
|
|
2655
|
+
runtimeKey,
|
|
2656
|
+
target: target,
|
|
2657
|
+
dependencies: this.deps.getModuleDependencies(),
|
|
735
2658
|
attempt,
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
2659
|
+
maxAttempts,
|
|
2660
|
+
stage: 'provider.runtime_resolve',
|
|
2661
|
+
logicalRequestChainKey,
|
|
2662
|
+
logicalChainRetryLimitStageRequestId: providerRequestId,
|
|
2663
|
+
excludedProviderKeys,
|
|
2664
|
+
recordAttempt,
|
|
2665
|
+
logStage: (stage, requestId, details) => this.logStage(stage, requestId, details),
|
|
739
2666
|
routeHint: forcedRouteHint,
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
2667
|
+
forceExcludeCurrentProviderOnRetry: true,
|
|
2668
|
+
abortSignal: clientAbortSignal
|
|
2669
|
+
});
|
|
2670
|
+
lastError = error;
|
|
2671
|
+
const retryExecutionPlan = providerFailurePlan.retryExecutionPlan;
|
|
2672
|
+
if (!retryExecutionPlan.shouldRetry || !retryExecutionPlan.retrySwitchPlan || !retryExecutionPlan.backoffScope) {
|
|
2673
|
+
throw error;
|
|
2674
|
+
}
|
|
2675
|
+
if (!providerFailurePlan.retryTelemetryPlan) {
|
|
2676
|
+
throw error;
|
|
2677
|
+
}
|
|
2678
|
+
emitRequestExecutorProviderRetryTelemetry({
|
|
2679
|
+
requestId: input.requestId,
|
|
2680
|
+
retryTelemetryPlan: providerFailurePlan.retryTelemetryPlan,
|
|
2681
|
+
logStage: (stage, requestId, details) => this.logStage(stage, requestId, details),
|
|
2682
|
+
logProviderRetrySwitch: (switchArgs) => this.logProviderRetrySwitch(switchArgs)
|
|
745
2683
|
});
|
|
746
2684
|
continue;
|
|
747
2685
|
}
|
|
748
2686
|
const previousRequestId = input.requestId;
|
|
749
2687
|
if (providerContext.requestId !== input.requestId) {
|
|
750
2688
|
input.requestId = providerContext.requestId;
|
|
2689
|
+
try {
|
|
2690
|
+
await rebindResponsesConversationRequestId(previousRequestId, input.requestId);
|
|
2691
|
+
}
|
|
2692
|
+
catch (error) {
|
|
2693
|
+
logRequestExecutorNonBlockingError('responsesConversation.rebindRequestId', error, {
|
|
2694
|
+
previousRequestId,
|
|
2695
|
+
requestId: input.requestId,
|
|
2696
|
+
providerKey: target.providerKey,
|
|
2697
|
+
runtimeKey
|
|
2698
|
+
});
|
|
2699
|
+
logRequestExecutorDegraded('responsesConversation.rebindRequestId', input.requestId, {
|
|
2700
|
+
previousRequestId,
|
|
2701
|
+
providerKey: target.providerKey,
|
|
2702
|
+
runtimeKey
|
|
2703
|
+
});
|
|
2704
|
+
}
|
|
751
2705
|
}
|
|
752
2706
|
this.logStage('provider.context_resolve.completed', input.requestId, {
|
|
753
2707
|
providerKey: target.providerKey,
|
|
@@ -782,6 +2736,7 @@ export class HubRequestExecutor {
|
|
|
782
2736
|
providerLabel,
|
|
783
2737
|
attempt
|
|
784
2738
|
});
|
|
2739
|
+
throwIfClientAbortSignalAborted(clientAbortSignal);
|
|
785
2740
|
this.logStage('provider.metadata_attach.start', input.requestId, {
|
|
786
2741
|
providerKey: target.providerKey,
|
|
787
2742
|
runtimeKey,
|
|
@@ -799,60 +2754,111 @@ export class HubRequestExecutor {
|
|
|
799
2754
|
runtimeKey,
|
|
800
2755
|
target,
|
|
801
2756
|
metadata: mergedMetadata,
|
|
802
|
-
compatibilityProfile: target.compatibilityProfile
|
|
2757
|
+
compatibilityProfile: target.compatibilityProfile,
|
|
2758
|
+
abortSignal: getClientConnectionAbortSignal(mergedMetadata)
|
|
803
2759
|
});
|
|
804
2760
|
this.logStage('provider.metadata_attach.completed', input.requestId, {
|
|
805
2761
|
providerKey: target.providerKey,
|
|
806
2762
|
runtimeKey,
|
|
807
2763
|
attempt
|
|
808
2764
|
});
|
|
2765
|
+
const providerTransportBackoffKey = buildProviderTransportBackoffKey({
|
|
2766
|
+
providerKey: target.providerKey,
|
|
2767
|
+
runtimeKey
|
|
2768
|
+
});
|
|
809
2769
|
let trafficPermit = null;
|
|
2770
|
+
let trafficPolicyMaxInFlight = 0;
|
|
2771
|
+
let trafficActiveInFlightAtAcquire = 0;
|
|
810
2772
|
let providerSendStartedAtMs = 0;
|
|
2773
|
+
let providerSendElapsedMs = 0;
|
|
2774
|
+
const stoplessLogState = resolveStoplessLogState(mergedMetadata);
|
|
2775
|
+
const providerRequestedStream = typeof providerPayload?.stream === 'boolean'
|
|
2776
|
+
? Boolean(providerPayload.stream)
|
|
2777
|
+
: undefined;
|
|
2778
|
+
const bypassTrafficGovernor = isServerToolFollowupRequest(metadataForAttempt);
|
|
811
2779
|
try {
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
2780
|
+
throwIfClientAbortSignalAborted(clientAbortSignal);
|
|
2781
|
+
if (providerTransportBackoffKey) {
|
|
2782
|
+
const pendingProviderTransportWaitMs = peekProviderTransportBackoffWaitMs(providerTransportBackoffKey);
|
|
2783
|
+
if (pendingProviderTransportWaitMs > 0) {
|
|
2784
|
+
this.logStage('provider.transport_backoff_wait', input.requestId, {
|
|
2785
|
+
providerKey: target.providerKey,
|
|
2786
|
+
runtimeKey,
|
|
2787
|
+
waitMs: pendingProviderTransportWaitMs,
|
|
2788
|
+
attempt
|
|
2789
|
+
});
|
|
2790
|
+
await waitProviderTransportBackoffWithGate(providerTransportBackoffKey, pendingProviderTransportWaitMs, clientAbortSignal);
|
|
2791
|
+
this.logStage('provider.transport_backoff_wait.completed', input.requestId, {
|
|
2792
|
+
providerKey: target.providerKey,
|
|
2793
|
+
runtimeKey,
|
|
2794
|
+
waitMs: pendingProviderTransportWaitMs,
|
|
2795
|
+
attempt
|
|
2796
|
+
});
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
if (bypassTrafficGovernor) {
|
|
2800
|
+
this.logStage('provider.traffic.acquire.bypassed', input.requestId, {
|
|
829
2801
|
providerKey: target.providerKey,
|
|
830
2802
|
runtimeKey,
|
|
831
|
-
|
|
2803
|
+
reason: 'servertool_followup',
|
|
2804
|
+
attempt
|
|
2805
|
+
});
|
|
2806
|
+
}
|
|
2807
|
+
else {
|
|
2808
|
+
this.logStage('provider.traffic.acquire.start', input.requestId, {
|
|
2809
|
+
providerKey: target.providerKey,
|
|
2810
|
+
runtimeKey,
|
|
2811
|
+
attempt
|
|
2812
|
+
});
|
|
2813
|
+
const trafficAcquired = await this.trafficGovernor.acquire({
|
|
2814
|
+
runtimeKey,
|
|
2815
|
+
providerKey: target.providerKey,
|
|
2816
|
+
requestId: input.requestId,
|
|
2817
|
+
runtime: resolveTrafficRuntimeProfile(runtimeKey, handle, target.providerKey),
|
|
2818
|
+
// Hard rule (2026-04): local traffic saturation must block-wait/backoff instead of
|
|
2819
|
+
// switch storm. Do not use soft timeout here.
|
|
2820
|
+
softWaitTimeoutMs: undefined
|
|
2821
|
+
});
|
|
2822
|
+
trafficPermit = trafficAcquired.permit;
|
|
2823
|
+
trafficPolicyMaxInFlight = trafficAcquired.policy.concurrency.maxInFlight;
|
|
2824
|
+
trafficActiveInFlightAtAcquire = trafficAcquired.activeInFlight;
|
|
2825
|
+
if (trafficAcquired.waitedMs > 0) {
|
|
2826
|
+
cumulativeTrafficWaitMs += trafficAcquired.waitedMs;
|
|
2827
|
+
this.logStage('provider.traffic.acquire.wait', input.requestId, {
|
|
2828
|
+
providerKey: target.providerKey,
|
|
2829
|
+
runtimeKey,
|
|
2830
|
+
waitedMs: trafficAcquired.waitedMs,
|
|
2831
|
+
attempt
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
this.logStage('provider.traffic.acquire.completed', input.requestId, {
|
|
2835
|
+
providerKey: target.providerKey,
|
|
2836
|
+
runtimeKey,
|
|
2837
|
+
maxInFlight: trafficAcquired.policy.concurrency.maxInFlight,
|
|
2838
|
+
requestsPerMinute: trafficAcquired.policy.rpm.requestsPerMinute,
|
|
2839
|
+
activeInFlight: trafficAcquired.activeInFlight,
|
|
2840
|
+
rpmInWindow: trafficAcquired.rpmInWindow,
|
|
832
2841
|
attempt
|
|
833
2842
|
});
|
|
834
2843
|
}
|
|
835
|
-
this.logStage('provider.traffic.acquire.completed', input.requestId, {
|
|
836
|
-
providerKey: target.providerKey,
|
|
837
|
-
runtimeKey,
|
|
838
|
-
maxInFlight: trafficAcquired.policy.concurrency.maxInFlight,
|
|
839
|
-
requestsPerMinute: trafficAcquired.policy.rpm.requestsPerMinute,
|
|
840
|
-
activeInFlight: trafficAcquired.activeInFlight,
|
|
841
|
-
rpmInWindow: trafficAcquired.rpmInWindow,
|
|
842
|
-
attempt
|
|
843
|
-
});
|
|
844
2844
|
const routingDecisionRecord = pipelineResult.routingDecision && typeof pipelineResult.routingDecision === 'object'
|
|
845
2845
|
? pipelineResult.routingDecision
|
|
846
2846
|
: undefined;
|
|
847
2847
|
emitVirtualRouterConcurrencyLog({
|
|
848
2848
|
sessionId: readString(mergedMetadata.sessionId) ?? readString(mergedMetadata.conversationId),
|
|
2849
|
+
projectPath: readString(mergedMetadata.clientWorkdir)
|
|
2850
|
+
?? readString(mergedMetadata.client_workdir)
|
|
2851
|
+
?? readString(mergedMetadata.workdir)
|
|
2852
|
+
?? readString(mergedMetadata.cwd),
|
|
849
2853
|
routeName: pipelineResult.routingDecision?.routeName,
|
|
850
2854
|
poolId: readString(routingDecisionRecord?.poolId),
|
|
851
2855
|
providerKey: target.providerKey,
|
|
852
2856
|
model: providerModel,
|
|
853
2857
|
reason: readString(routingDecisionRecord?.reasoning),
|
|
854
|
-
|
|
855
|
-
|
|
2858
|
+
stoplessMode: stoplessLogState.mode,
|
|
2859
|
+
stoplessArmed: stoplessLogState.armed,
|
|
2860
|
+
activeInFlight: trafficActiveInFlightAtAcquire,
|
|
2861
|
+
maxInFlight: trafficPolicyMaxInFlight
|
|
856
2862
|
});
|
|
857
2863
|
providerSendStartedAtMs = Date.now();
|
|
858
2864
|
this.logStage('provider.send.start', input.requestId, {
|
|
@@ -863,14 +2869,19 @@ export class HubRequestExecutor {
|
|
|
863
2869
|
providerFamily: handle.providerFamily,
|
|
864
2870
|
model: providerModel,
|
|
865
2871
|
providerLabel,
|
|
2872
|
+
providerRequestedStream,
|
|
866
2873
|
attempt
|
|
867
2874
|
});
|
|
2875
|
+
throwIfClientAbortSignalAborted(clientAbortSignal);
|
|
2876
|
+
allowSnapshotLocalDiskWrite(executorRequestId, providerRequestId, input.requestId, clientRequestId);
|
|
868
2877
|
const providerResponse = await handle.instance.processIncoming(providerPayload);
|
|
869
2878
|
const responseStatus = extractResponseStatus(providerResponse);
|
|
2879
|
+
providerSendElapsedMs = Date.now() - providerSendStartedAtMs;
|
|
2880
|
+
cumulativeExternalLatencyMs += providerSendElapsedMs;
|
|
870
2881
|
this.logStage('provider.send.completed', input.requestId, {
|
|
871
2882
|
providerKey: target.providerKey,
|
|
872
2883
|
status: responseStatus,
|
|
873
|
-
elapsedMs:
|
|
2884
|
+
elapsedMs: providerSendElapsedMs,
|
|
874
2885
|
providerType: handle.providerType,
|
|
875
2886
|
providerFamily: handle.providerFamily,
|
|
876
2887
|
model: providerModel,
|
|
@@ -926,34 +2937,58 @@ export class HubRequestExecutor {
|
|
|
926
2937
|
processMode: pipelineResult.processMode,
|
|
927
2938
|
attempt
|
|
928
2939
|
});
|
|
929
|
-
const converted =
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
2940
|
+
const converted = shouldBypassProviderResponseConversion(normalized)
|
|
2941
|
+
? (() => {
|
|
2942
|
+
this.logStage('provider.response_convert.skipped', input.requestId, {
|
|
2943
|
+
providerKey: target.providerKey,
|
|
2944
|
+
status: normalized.status,
|
|
2945
|
+
reason: 'non_success_status_bypass',
|
|
2946
|
+
attempt
|
|
2947
|
+
});
|
|
2948
|
+
return normalized;
|
|
2949
|
+
})()
|
|
2950
|
+
: await this.convertProviderResponseIfNeeded({
|
|
2951
|
+
entryEndpoint: input.entryEndpoint,
|
|
2952
|
+
providerProtocol,
|
|
2953
|
+
providerType: handle.providerType,
|
|
2954
|
+
requestId: input.requestId,
|
|
2955
|
+
serverToolsEnabled,
|
|
2956
|
+
wantsStream: wantsStreamBase,
|
|
2957
|
+
originalRequest: resolveOriginalRequestForResponseConversion(retryPayloadSeed),
|
|
2958
|
+
requestSemantics,
|
|
2959
|
+
processMode: pipelineResult.processMode,
|
|
2960
|
+
response: normalized,
|
|
2961
|
+
pipelineMetadata: mergedMetadata
|
|
2962
|
+
});
|
|
942
2963
|
const clientInjectWaitMsRaw = converted.timingBreakdown?.hubResponseExcludedMs;
|
|
943
2964
|
const clientInjectWaitMs = typeof clientInjectWaitMsRaw === 'number' && Number.isFinite(clientInjectWaitMsRaw)
|
|
944
2965
|
? Math.max(0, Math.floor(clientInjectWaitMsRaw))
|
|
945
2966
|
: 0;
|
|
2967
|
+
if (clientInjectWaitMs > 0) {
|
|
2968
|
+
cumulativeClientInjectWaitMs += clientInjectWaitMs;
|
|
2969
|
+
}
|
|
946
2970
|
const hubResponseElapsedMsRaw = Date.now() - hubResponseStartedAtMs;
|
|
947
2971
|
const hubResponseElapsedMs = Math.max(0, hubResponseElapsedMsRaw - clientInjectWaitMs);
|
|
948
2972
|
const convertedBodyRecord = converted.body && typeof converted.body === 'object'
|
|
949
2973
|
? converted.body
|
|
950
2974
|
: undefined;
|
|
2975
|
+
const normalizedBodyRecord = normalized.body && typeof normalized.body === 'object'
|
|
2976
|
+
? normalized.body
|
|
2977
|
+
: undefined;
|
|
951
2978
|
if (convertedBodyRecord) {
|
|
952
2979
|
backfillResponsesOutputTextIfMissing(convertedBodyRecord);
|
|
953
2980
|
}
|
|
954
|
-
const finishReason =
|
|
955
|
-
|
|
956
|
-
|
|
2981
|
+
const finishReason = (() => {
|
|
2982
|
+
if (convertedBodyRecord
|
|
2983
|
+
&& typeof convertedBodyRecord[STREAM_LOG_FINISH_REASON_KEY] === 'string') {
|
|
2984
|
+
return String(convertedBodyRecord[STREAM_LOG_FINISH_REASON_KEY]);
|
|
2985
|
+
}
|
|
2986
|
+
const fromConverted = deriveFinishReason(convertedBodyRecord);
|
|
2987
|
+
if (fromConverted) {
|
|
2988
|
+
return fromConverted;
|
|
2989
|
+
}
|
|
2990
|
+
return deriveFinishReason(normalizedBodyRecord);
|
|
2991
|
+
})();
|
|
957
2992
|
this.logStage('provider.response_convert.completed', input.requestId, {
|
|
958
2993
|
providerKey: target.providerKey,
|
|
959
2994
|
status: converted.status,
|
|
@@ -1010,41 +3045,7 @@ export class HubRequestExecutor {
|
|
|
1010
3045
|
errorToThrow.statusCode = statusCode;
|
|
1011
3046
|
errorToThrow.status = statusCode;
|
|
1012
3047
|
errorToThrow.response = { data: bodyForError };
|
|
1013
|
-
|
|
1014
|
-
const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
|
|
1015
|
-
emitProviderError({
|
|
1016
|
-
error: errorToThrow,
|
|
1017
|
-
stage: 'provider.http',
|
|
1018
|
-
runtime: {
|
|
1019
|
-
requestId: input.requestId,
|
|
1020
|
-
providerKey: target.providerKey,
|
|
1021
|
-
providerId: handle.providerId,
|
|
1022
|
-
providerType: handle.providerType,
|
|
1023
|
-
providerFamily: handle.providerFamily,
|
|
1024
|
-
providerProtocol,
|
|
1025
|
-
routeName: pipelineResult.routingDecision?.routeName,
|
|
1026
|
-
pipelineId: target.providerKey,
|
|
1027
|
-
target,
|
|
1028
|
-
runtimeKey
|
|
1029
|
-
},
|
|
1030
|
-
dependencies: this.deps.getModuleDependencies(),
|
|
1031
|
-
statusCode,
|
|
1032
|
-
recoverable: statusCode === 401 ||
|
|
1033
|
-
statusCode === 429 ||
|
|
1034
|
-
statusCode === 408 ||
|
|
1035
|
-
statusCode === 425 ||
|
|
1036
|
-
statusCode >= 500,
|
|
1037
|
-
affectsHealth: true,
|
|
1038
|
-
details: {
|
|
1039
|
-
source: 'converted_response_status',
|
|
1040
|
-
convertedStatus: statusCode,
|
|
1041
|
-
wrappedErrorResponse: true
|
|
1042
|
-
}
|
|
1043
|
-
});
|
|
1044
|
-
}
|
|
1045
|
-
catch {
|
|
1046
|
-
// best-effort; never block retry/failover path
|
|
1047
|
-
}
|
|
3048
|
+
errorToThrow.requestExecutorProviderErrorStage = 'provider.http';
|
|
1048
3049
|
throw errorToThrow;
|
|
1049
3050
|
}
|
|
1050
3051
|
this.logStage('provider.response_status_check.completed', input.requestId, {
|
|
@@ -1052,15 +3053,66 @@ export class HubRequestExecutor {
|
|
|
1052
3053
|
convertedStatus,
|
|
1053
3054
|
attempt
|
|
1054
3055
|
});
|
|
3056
|
+
clearProviderTransportBackoff(providerTransportBackoffKey);
|
|
3057
|
+
if (!bypassTrafficGovernor) {
|
|
3058
|
+
try {
|
|
3059
|
+
await this.trafficGovernor.observeOutcome?.({
|
|
3060
|
+
runtimeKey,
|
|
3061
|
+
providerKey: target.providerKey,
|
|
3062
|
+
requestId: input.requestId,
|
|
3063
|
+
success: true,
|
|
3064
|
+
statusCode: convertedStatus,
|
|
3065
|
+
activeInFlight: trafficActiveInFlightAtAcquire,
|
|
3066
|
+
configuredMaxInFlight: trafficPolicyMaxInFlight || undefined
|
|
3067
|
+
});
|
|
3068
|
+
}
|
|
3069
|
+
catch (observeError) {
|
|
3070
|
+
this.logStage('provider.traffic.observe_outcome.error', input.requestId, {
|
|
3071
|
+
providerKey: target.providerKey,
|
|
3072
|
+
runtimeKey,
|
|
3073
|
+
message: observeError instanceof Error
|
|
3074
|
+
? observeError.message
|
|
3075
|
+
: String(observeError ?? 'Unknown observe outcome error'),
|
|
3076
|
+
attempt
|
|
3077
|
+
});
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
1055
3080
|
const emptyAssistantSignal = detectRetryableEmptyAssistantResponse(converted.body);
|
|
1056
3081
|
if (emptyAssistantSignal) {
|
|
1057
3082
|
const bodyForError = converted.body;
|
|
3083
|
+
try {
|
|
3084
|
+
await persistEmptyAssistantProviderSnapshots({
|
|
3085
|
+
requestId: input.requestId,
|
|
3086
|
+
entryEndpoint: input.entryEndpoint,
|
|
3087
|
+
providerKey: target.providerKey,
|
|
3088
|
+
providerId: handle.providerId,
|
|
3089
|
+
providerRequestPayload: providerPayload,
|
|
3090
|
+
providerRequestHeaders: providerPayload && typeof providerPayload === 'object'
|
|
3091
|
+
? providerPayload.headers
|
|
3092
|
+
: undefined,
|
|
3093
|
+
providerRequestUrl: providerPayload && typeof providerPayload === 'object' && typeof providerPayload.url === 'string'
|
|
3094
|
+
? String(providerPayload.url)
|
|
3095
|
+
: undefined,
|
|
3096
|
+
normalizedResponse: normalized,
|
|
3097
|
+
convertedResponse: converted,
|
|
3098
|
+
emptyAssistantSignal
|
|
3099
|
+
});
|
|
3100
|
+
}
|
|
3101
|
+
catch (snapshotError) {
|
|
3102
|
+
logRequestExecutorNonBlockingError('host.response_contract.empty_assistant.snapshot', snapshotError, {
|
|
3103
|
+
requestId: input.requestId,
|
|
3104
|
+
providerKey: target.providerKey,
|
|
3105
|
+
marker: emptyAssistantSignal.marker
|
|
3106
|
+
});
|
|
3107
|
+
}
|
|
1058
3108
|
const errorToThrow = new Error(`Upstream returned empty assistant payload: ${emptyAssistantSignal.reason}`);
|
|
1059
3109
|
errorToThrow.statusCode = 502;
|
|
1060
3110
|
errorToThrow.status = 502;
|
|
1061
3111
|
errorToThrow.code = 'EMPTY_ASSISTANT_RESPONSE';
|
|
3112
|
+
errorToThrow.retryable = true;
|
|
3113
|
+
errorToThrow.requestExecutorProviderErrorStage = 'host.response_contract';
|
|
1062
3114
|
errorToThrow.response = { data: bodyForError };
|
|
1063
|
-
this.logStage('
|
|
3115
|
+
this.logStage('host.response_contract.empty_assistant', input.requestId, {
|
|
1064
3116
|
providerKey: target.providerKey,
|
|
1065
3117
|
marker: emptyAssistantSignal.marker,
|
|
1066
3118
|
reason: emptyAssistantSignal.reason,
|
|
@@ -1068,6 +3120,29 @@ export class HubRequestExecutor {
|
|
|
1068
3120
|
});
|
|
1069
3121
|
throw errorToThrow;
|
|
1070
3122
|
}
|
|
3123
|
+
const stoplessTerminationSignal = detectStoplessTerminationWithoutFinalization(converted.body, stoplessLogState.mode);
|
|
3124
|
+
if (stoplessTerminationSignal) {
|
|
3125
|
+
const bodyForError = converted.body && typeof converted.body === 'object'
|
|
3126
|
+
? converted.body
|
|
3127
|
+
: undefined;
|
|
3128
|
+
const errorToThrow = new Error(`Stopless contract violated: ${stoplessTerminationSignal.reason}`);
|
|
3129
|
+
errorToThrow.statusCode = 502;
|
|
3130
|
+
errorToThrow.status = 502;
|
|
3131
|
+
errorToThrow.code = 'STOPLESS_FINALIZATION_MISSING';
|
|
3132
|
+
errorToThrow.retryable = true;
|
|
3133
|
+
errorToThrow.requestExecutorProviderErrorStage = 'host.stopless_contract';
|
|
3134
|
+
if (bodyForError) {
|
|
3135
|
+
errorToThrow.response = { data: bodyForError };
|
|
3136
|
+
}
|
|
3137
|
+
this.logStage('host.stopless_finalization_missing', input.requestId, {
|
|
3138
|
+
providerKey: target.providerKey,
|
|
3139
|
+
marker: stoplessTerminationSignal.marker,
|
|
3140
|
+
reason: stoplessTerminationSignal.reason,
|
|
3141
|
+
stoplessMode: stoplessLogState.mode,
|
|
3142
|
+
attempt
|
|
3143
|
+
});
|
|
3144
|
+
throw errorToThrow;
|
|
3145
|
+
}
|
|
1071
3146
|
this.logStage('provider.usage_extract.start', input.requestId, {
|
|
1072
3147
|
providerKey: target.providerKey,
|
|
1073
3148
|
source: 'converted_response',
|
|
@@ -1096,24 +3171,92 @@ export class HubRequestExecutor {
|
|
|
1096
3171
|
attempt
|
|
1097
3172
|
});
|
|
1098
3173
|
recordAttempt({ usage: aggregatedUsage, error: false });
|
|
3174
|
+
if (sessionStormBackoffScope) {
|
|
3175
|
+
clearSessionStormBackoff(sessionStormBackoffScope);
|
|
3176
|
+
}
|
|
1099
3177
|
const metadataHubStageTop = readHubStageTop(mergedMetadata);
|
|
3178
|
+
const hubDecodeBreakdown = readHubDecodeBreakdown(metadataHubStageTop);
|
|
1100
3179
|
return {
|
|
1101
3180
|
...converted,
|
|
1102
3181
|
usageLogInfo: {
|
|
1103
3182
|
providerKey: target.providerKey,
|
|
1104
3183
|
model: providerModel,
|
|
3184
|
+
routeName: pipelineResult.routingDecision?.routeName,
|
|
3185
|
+
poolId: readString(routingDecisionRecord?.poolId),
|
|
3186
|
+
finishReason,
|
|
1105
3187
|
usage: aggregatedUsage,
|
|
3188
|
+
externalLatencyMs: cumulativeExternalLatencyMs > 0 ? cumulativeExternalLatencyMs : undefined,
|
|
3189
|
+
trafficWaitMs: cumulativeTrafficWaitMs > 0 ? cumulativeTrafficWaitMs : undefined,
|
|
3190
|
+
clientInjectWaitMs: cumulativeClientInjectWaitMs > 0 ? cumulativeClientInjectWaitMs : undefined,
|
|
3191
|
+
sseDecodeMs: hubDecodeBreakdown.sseDecodeMs > 0 ? hubDecodeBreakdown.sseDecodeMs : undefined,
|
|
3192
|
+
codecDecodeMs: hubDecodeBreakdown.codecDecodeMs > 0 ? hubDecodeBreakdown.codecDecodeMs : undefined,
|
|
3193
|
+
providerAttemptCount: attempt,
|
|
3194
|
+
retryCount: Math.max(0, attempt - 1),
|
|
1106
3195
|
hubStageTop: metadataHubStageTop,
|
|
1107
3196
|
requestStartedAtMs: requestStartedAt,
|
|
1108
3197
|
timingRequestIds: Array.from(new Set([providerRequestId, input.requestId].filter((value) => Boolean(value)))),
|
|
1109
3198
|
sessionId: mergedMetadata.sessionId,
|
|
1110
|
-
conversationId: mergedMetadata.conversationId
|
|
3199
|
+
conversationId: mergedMetadata.conversationId,
|
|
3200
|
+
projectPath: readString(mergedMetadata.clientWorkdir)
|
|
3201
|
+
?? readString(mergedMetadata.client_workdir)
|
|
3202
|
+
?? readString(mergedMetadata.workdir)
|
|
3203
|
+
?? readString(mergedMetadata.cwd)
|
|
1111
3204
|
}
|
|
1112
3205
|
};
|
|
1113
3206
|
}
|
|
1114
3207
|
catch (error) {
|
|
3208
|
+
if (providerSendStartedAtMs > 0 && providerSendElapsedMs <= 0) {
|
|
3209
|
+
const failedSendElapsedMs = Math.max(0, Date.now() - providerSendStartedAtMs);
|
|
3210
|
+
if (failedSendElapsedMs > 0) {
|
|
3211
|
+
providerSendElapsedMs = failedSendElapsedMs;
|
|
3212
|
+
cumulativeExternalLatencyMs += failedSendElapsedMs;
|
|
3213
|
+
this.logStage('provider.send.failed_elapsed', input.requestId, {
|
|
3214
|
+
providerKey: target.providerKey,
|
|
3215
|
+
elapsedMs: failedSendElapsedMs,
|
|
3216
|
+
attempt
|
|
3217
|
+
});
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
1115
3220
|
const errorMessage = error instanceof Error ? error.message : String(error ?? 'Unknown error');
|
|
1116
3221
|
const retryError = extractRetryErrorSnapshot(error);
|
|
3222
|
+
if (sessionStormBackoffScope && isSessionStormBackoffCandidate(error)) {
|
|
3223
|
+
const backoffMs = consumeSessionStormBackoffMs(sessionStormBackoffScope);
|
|
3224
|
+
this.logStage('request.session_storm_backoff.recorded', input.requestId, {
|
|
3225
|
+
scope: sessionStormBackoffScope,
|
|
3226
|
+
backoffMs,
|
|
3227
|
+
consecutive: sessionStormBackoffState.get(sessionStormBackoffScope)?.consecutive ?? 0,
|
|
3228
|
+
reason: retryError.reason,
|
|
3229
|
+
errorCode: retryError.errorCode,
|
|
3230
|
+
upstreamCode: retryError.upstreamCode,
|
|
3231
|
+
statusCode: retryError.statusCode
|
|
3232
|
+
});
|
|
3233
|
+
}
|
|
3234
|
+
if (!bypassTrafficGovernor) {
|
|
3235
|
+
try {
|
|
3236
|
+
await this.trafficGovernor.observeOutcome?.({
|
|
3237
|
+
runtimeKey,
|
|
3238
|
+
providerKey: target.providerKey,
|
|
3239
|
+
requestId: input.requestId,
|
|
3240
|
+
success: false,
|
|
3241
|
+
statusCode: retryError.statusCode,
|
|
3242
|
+
errorCode: retryError.errorCode,
|
|
3243
|
+
upstreamCode: retryError.upstreamCode,
|
|
3244
|
+
reason: retryError.reason,
|
|
3245
|
+
activeInFlight: trafficActiveInFlightAtAcquire,
|
|
3246
|
+
configuredMaxInFlight: trafficPolicyMaxInFlight || undefined
|
|
3247
|
+
});
|
|
3248
|
+
}
|
|
3249
|
+
catch (observeError) {
|
|
3250
|
+
this.logStage('provider.traffic.observe_outcome.error', input.requestId, {
|
|
3251
|
+
providerKey: target.providerKey,
|
|
3252
|
+
runtimeKey,
|
|
3253
|
+
message: observeError instanceof Error
|
|
3254
|
+
? observeError.message
|
|
3255
|
+
: String(observeError ?? 'Unknown observe outcome error'),
|
|
3256
|
+
attempt
|
|
3257
|
+
});
|
|
3258
|
+
}
|
|
3259
|
+
}
|
|
1117
3260
|
this.logStage('provider.send.error', input.requestId, {
|
|
1118
3261
|
providerKey: target.providerKey,
|
|
1119
3262
|
message: errorMessage,
|
|
@@ -1127,85 +3270,41 @@ export class HubRequestExecutor {
|
|
|
1127
3270
|
attempt
|
|
1128
3271
|
});
|
|
1129
3272
|
lastError = error;
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
pipelineId: target.providerKey,
|
|
1156
|
-
target,
|
|
1157
|
-
runtimeKey
|
|
1158
|
-
},
|
|
1159
|
-
dependencies: this.deps.getModuleDependencies(),
|
|
1160
|
-
statusCode: 429,
|
|
1161
|
-
recoverable: true,
|
|
1162
|
-
affectsHealth: true,
|
|
1163
|
-
details: {
|
|
1164
|
-
source: 'sse_decode_rate_limit',
|
|
1165
|
-
errorCode: typeof error?.code === 'string' ? String(error.code) : undefined,
|
|
1166
|
-
upstreamCode: typeof error?.upstreamCode === 'string' ? String(error.upstreamCode) : undefined,
|
|
1167
|
-
message: errorMessage
|
|
1168
|
-
}
|
|
1169
|
-
});
|
|
1170
|
-
}
|
|
1171
|
-
catch {
|
|
1172
|
-
// best-effort; never block retry/failover path
|
|
1173
|
-
}
|
|
1174
|
-
}
|
|
1175
|
-
else if (isSseDecodeRetryableNetworkError(error, status)) {
|
|
1176
|
-
try {
|
|
1177
|
-
const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
|
|
1178
|
-
emitProviderError({
|
|
1179
|
-
error,
|
|
1180
|
-
stage: 'provider.sse_decode',
|
|
1181
|
-
runtime: {
|
|
1182
|
-
requestId: input.requestId,
|
|
1183
|
-
providerKey: target.providerKey,
|
|
1184
|
-
providerId: handle.providerId,
|
|
1185
|
-
providerType: handle.providerType,
|
|
1186
|
-
providerFamily: handle.providerFamily,
|
|
1187
|
-
providerProtocol,
|
|
1188
|
-
routeName: pipelineResult.routingDecision?.routeName,
|
|
1189
|
-
pipelineId: target.providerKey,
|
|
1190
|
-
target,
|
|
1191
|
-
runtimeKey
|
|
1192
|
-
},
|
|
1193
|
-
dependencies: this.deps.getModuleDependencies(),
|
|
1194
|
-
statusCode: 502,
|
|
1195
|
-
recoverable: true,
|
|
1196
|
-
affectsHealth: true,
|
|
1197
|
-
details: {
|
|
1198
|
-
source: 'sse_decode_network_error',
|
|
1199
|
-
errorCode: typeof error?.code === 'string' ? String(error.code) : undefined,
|
|
1200
|
-
upstreamCode: typeof error?.upstreamCode === 'string' ? String(error.upstreamCode) : undefined,
|
|
1201
|
-
message: errorMessage
|
|
1202
|
-
}
|
|
1203
|
-
});
|
|
1204
|
-
}
|
|
1205
|
-
catch {
|
|
1206
|
-
// best-effort; never block retry/failover path
|
|
1207
|
-
}
|
|
3273
|
+
const status = typeof retryError.statusCode === 'number'
|
|
3274
|
+
? retryError.statusCode
|
|
3275
|
+
: extractStatusCodeFromError(error);
|
|
3276
|
+
if (providerTransportBackoffKey
|
|
3277
|
+
&& shouldApplyProviderTransportBackoff({
|
|
3278
|
+
error,
|
|
3279
|
+
retryError,
|
|
3280
|
+
stage: 'provider.send'
|
|
3281
|
+
})) {
|
|
3282
|
+
const providerBackoffMs = consumeProviderTransportBackoffMs(providerTransportBackoffKey, {
|
|
3283
|
+
error,
|
|
3284
|
+
statusCode: status
|
|
3285
|
+
});
|
|
3286
|
+
this.logStage('provider.transport_backoff.recorded', input.requestId, {
|
|
3287
|
+
providerKey: target.providerKey,
|
|
3288
|
+
runtimeKey,
|
|
3289
|
+
backoffKey: providerTransportBackoffKey,
|
|
3290
|
+
backoffMs: providerBackoffMs,
|
|
3291
|
+
consecutive: providerTransportBackoffState.get(providerTransportBackoffKey)?.consecutive ?? 0,
|
|
3292
|
+
reason: retryError.reason,
|
|
3293
|
+
errorCode: retryError.errorCode,
|
|
3294
|
+
upstreamCode: retryError.upstreamCode,
|
|
3295
|
+
statusCode: status,
|
|
3296
|
+
attempt
|
|
3297
|
+
});
|
|
1208
3298
|
}
|
|
3299
|
+
const nextAntigravityRetrySignal = isAntigravityProviderKey(target.providerKey)
|
|
3300
|
+
? (() => {
|
|
3301
|
+
const signature = extractRetryErrorSignature(error);
|
|
3302
|
+
const consecutive = antigravityRetrySignal && antigravityRetrySignal.signature === signature
|
|
3303
|
+
? antigravityRetrySignal.consecutive + 1
|
|
3304
|
+
: 1;
|
|
3305
|
+
return { signature, consecutive };
|
|
3306
|
+
})()
|
|
3307
|
+
: null;
|
|
1209
3308
|
const isVerify = status === 403 && isGoogleAccountVerificationRequiredError(error);
|
|
1210
3309
|
const isReauth = status === 403 && isAntigravityReauthRequired403(error);
|
|
1211
3310
|
const promptTooLong = isPromptTooLongError(error);
|
|
@@ -1215,72 +3314,53 @@ export class HubRequestExecutor {
|
|
|
1215
3314
|
forcedRouteHint = 'longcontext';
|
|
1216
3315
|
}
|
|
1217
3316
|
}
|
|
1218
|
-
const
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
: (shouldRetryProviderError(error) ||
|
|
1222
|
-
(isAntigravityProviderKey(target.providerKey) && (isVerify || isReauth))));
|
|
1223
|
-
if (!shouldRetry) {
|
|
1224
|
-
recordAttempt({ error: true });
|
|
1225
|
-
throw error;
|
|
1226
|
-
}
|
|
1227
|
-
// Record this failed provider attempt even if the overall request succeeds later via failover.
|
|
1228
|
-
recordAttempt({ error: true });
|
|
1229
|
-
const retryBackoffMs = await waitBeforeRetry(error, { attempt });
|
|
1230
|
-
const singleProviderPool = Boolean(initialRoutePool && initialRoutePool.length === 1 && initialRoutePool[0] === target.providerKey);
|
|
1231
|
-
if (promptTooLong && target.providerKey) {
|
|
1232
|
-
excludedProviderKeys.add(target.providerKey);
|
|
1233
|
-
}
|
|
1234
|
-
if (!promptTooLong && !singleProviderPool && target.providerKey) {
|
|
1235
|
-
const is429 = status === 429;
|
|
1236
|
-
if (isAntigravityProviderKey(target.providerKey) && (isVerify || is429)) {
|
|
1237
|
-
// For Antigravity 403 verify / 429 states:
|
|
1238
|
-
// - exclude the current providerKey so we don't immediately retry the same account
|
|
1239
|
-
// - avoid ALL other Antigravity aliases on retry (prefer non-Antigravity fallbacks)
|
|
1240
|
-
excludedProviderKeys.add(target.providerKey);
|
|
1241
|
-
if (antigravityRetrySignal) {
|
|
1242
|
-
antigravityRetrySignal.avoidAllOnRetry = true;
|
|
1243
|
-
}
|
|
1244
|
-
else {
|
|
1245
|
-
antigravityRetrySignal = { signature: extractRetryErrorSignature(error), consecutive: 1, avoidAllOnRetry: true };
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
else if (isAntigravityProviderKey(target.providerKey) && isReauth) {
|
|
1249
|
-
// Antigravity OAuth reauth-required 403:
|
|
1250
|
-
// - exclude the current providerKey so router can pick another alias
|
|
1251
|
-
// - DO NOT avoid all Antigravity on retry; switching aliases is the intended recovery path.
|
|
1252
|
-
excludedProviderKeys.add(target.providerKey);
|
|
1253
|
-
}
|
|
1254
|
-
else if (!isAntigravityProviderKey(target.providerKey) || shouldRotateAntigravityAliasOnRetry(error)) {
|
|
1255
|
-
excludedProviderKeys.add(target.providerKey);
|
|
1256
|
-
}
|
|
1257
|
-
}
|
|
1258
|
-
const switchAction = singleProviderPool ? 'retry_same_provider' : 'exclude_and_reroute';
|
|
1259
|
-
this.logProviderRetrySwitch({
|
|
3317
|
+
const providerFailurePlan = await resolveRequestExecutorProviderFailurePlan({
|
|
3318
|
+
error,
|
|
3319
|
+
retryError,
|
|
1260
3320
|
requestId: input.requestId,
|
|
1261
|
-
attempt,
|
|
1262
|
-
maxAttempts,
|
|
1263
|
-
providerKey: target.providerKey,
|
|
1264
|
-
nextAttempt: attempt + 1,
|
|
1265
|
-
reason: retryError.reason,
|
|
1266
|
-
statusCode: retryError.statusCode,
|
|
1267
|
-
errorCode: retryError.errorCode,
|
|
1268
|
-
upstreamCode: retryError.upstreamCode,
|
|
1269
|
-
switchAction
|
|
1270
|
-
});
|
|
1271
|
-
this.logStage('provider.retry', input.requestId, {
|
|
1272
3321
|
providerKey: target.providerKey,
|
|
3322
|
+
providerId: handle.providerId,
|
|
3323
|
+
providerType: handle.providerType,
|
|
3324
|
+
providerFamily: handle.providerFamily,
|
|
3325
|
+
providerProtocol,
|
|
3326
|
+
routeName: pipelineResult.routingDecision?.routeName,
|
|
3327
|
+
runtimeKey,
|
|
3328
|
+
target: target,
|
|
3329
|
+
dependencies: this.deps.getModuleDependencies(),
|
|
1273
3330
|
attempt,
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
3331
|
+
maxAttempts,
|
|
3332
|
+
stage: 'provider.send',
|
|
3333
|
+
logicalRequestChainKey,
|
|
3334
|
+
logicalChainRetryLimitStageRequestId: input.requestId,
|
|
3335
|
+
routePool: routePoolForAttempt,
|
|
3336
|
+
runtimeManager: this.deps.runtimeManager,
|
|
3337
|
+
excludedProviderKeys,
|
|
3338
|
+
recordAttempt,
|
|
3339
|
+
logStage: (stage, requestId, details) => this.logStage(stage, requestId, details),
|
|
1277
3340
|
routeHint: forcedRouteHint,
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
3341
|
+
promptTooLong,
|
|
3342
|
+
contextOverflowRetries,
|
|
3343
|
+
maxContextOverflowRetries: MAX_CONTEXT_OVERFLOW_RETRIES,
|
|
3344
|
+
isVerify,
|
|
3345
|
+
isReauth,
|
|
3346
|
+
allowAntigravityRecovery: true,
|
|
3347
|
+
antigravityRetrySignal: nextAntigravityRetrySignal,
|
|
3348
|
+
status,
|
|
3349
|
+
abortSignal: clientAbortSignal
|
|
3350
|
+
});
|
|
3351
|
+
const retryExecutionPlan = providerFailurePlan.retryExecutionPlan;
|
|
3352
|
+
if (!retryExecutionPlan.shouldRetry || !retryExecutionPlan.retrySwitchPlan || !retryExecutionPlan.backoffScope) {
|
|
3353
|
+
throw error;
|
|
3354
|
+
}
|
|
3355
|
+
antigravityRetrySignal = retryExecutionPlan.antigravityRetrySignal;
|
|
3356
|
+
if (!providerFailurePlan.retryTelemetryPlan) {
|
|
3357
|
+
throw error;
|
|
3358
|
+
}
|
|
3359
|
+
emitRequestExecutorProviderRetryTelemetry({
|
|
3360
|
+
requestId: input.requestId,
|
|
3361
|
+
retryTelemetryPlan: providerFailurePlan.retryTelemetryPlan,
|
|
3362
|
+
logStage: (stage, requestId, details) => this.logStage(stage, requestId, details),
|
|
3363
|
+
logProviderRetrySwitch: (switchArgs) => this.logProviderRetrySwitch(switchArgs)
|
|
1284
3364
|
});
|
|
1285
3365
|
continue;
|
|
1286
3366
|
}
|
|
@@ -1327,6 +3407,7 @@ export class HubRequestExecutor {
|
|
|
1327
3407
|
}
|
|
1328
3408
|
finally {
|
|
1329
3409
|
await this.deps.onRequestEnd?.({ requestId: executorRequestId });
|
|
3410
|
+
releaseLogicalRequestChainIfNeeded();
|
|
1330
3411
|
}
|
|
1331
3412
|
}
|
|
1332
3413
|
catch (error) {
|
|
@@ -1335,6 +3416,7 @@ export class HubRequestExecutor {
|
|
|
1335
3416
|
if (!recordedAnyAttempt) {
|
|
1336
3417
|
recordAttempt({ error: true });
|
|
1337
3418
|
}
|
|
3419
|
+
releaseLogicalRequestChainIfNeeded();
|
|
1338
3420
|
throw error;
|
|
1339
3421
|
}
|
|
1340
3422
|
}
|
|
@@ -1349,7 +3431,35 @@ export const __requestExecutorTestables = {
|
|
|
1349
3431
|
readString,
|
|
1350
3432
|
extractRetryErrorSnapshot,
|
|
1351
3433
|
truncateReason,
|
|
1352
|
-
|
|
3434
|
+
isHealthNeutralProviderError,
|
|
3435
|
+
isLastAvailableProvider429,
|
|
3436
|
+
shouldApplyProviderTransportBackoff,
|
|
3437
|
+
buildRecoverableErrorBackoffKey,
|
|
3438
|
+
consumeRecoverableErrorBackoffMs,
|
|
3439
|
+
buildProviderTransportBackoffKey,
|
|
3440
|
+
consumeProviderTransportBackoffMs,
|
|
3441
|
+
peekProviderTransportBackoffWaitMs,
|
|
3442
|
+
clearProviderTransportBackoff,
|
|
3443
|
+
detectRetryableEmptyAssistantResponse,
|
|
3444
|
+
deriveLogicalRequestChainKey,
|
|
3445
|
+
resolveSessionStormBackoffScope,
|
|
3446
|
+
isSessionStormBackoffCandidate,
|
|
3447
|
+
consumeSessionStormBackoffMs,
|
|
3448
|
+
peekSessionStormBackoffWaitMs,
|
|
3449
|
+
clearSessionStormBackoff,
|
|
3450
|
+
prepareRequestPayloadRetrySeed,
|
|
3451
|
+
resolveOriginalRequestForResponseConversion,
|
|
3452
|
+
resolveRequestExecutorProviderErrorClassification,
|
|
3453
|
+
resolveRequestExecutorProviderErrorReportPlan,
|
|
3454
|
+
resolveProviderRetryEligibilityPlan,
|
|
3455
|
+
resolveProviderRetryExclusionPlan,
|
|
3456
|
+
resolveExcludedProviderReselectionPlan,
|
|
3457
|
+
resolveProviderRetryExecutionPlan,
|
|
3458
|
+
buildProviderRetryTelemetryPlan,
|
|
3459
|
+
acquireRecoverableRetryWaiterSlotForTests,
|
|
3460
|
+
peekRecoverableRetryWaitersForTests,
|
|
3461
|
+
releaseRecoverableRetryWaiterSlotForTests,
|
|
3462
|
+
resetRequestExecutorInternalStateForTests
|
|
1353
3463
|
};
|
|
1354
3464
|
export function createRequestExecutor(deps) {
|
|
1355
3465
|
return new HubRequestExecutor(deps);
|