@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +943 -51
- package/CHANGELOG.md +273 -1
- package/LICENSE +21 -0
- package/README.md +117 -29
- package/SUBMISSION.md +3 -2
- package/bin/apifuse-check.ts +293 -13
- package/bin/apifuse-dev.ts +42 -19
- package/bin/apifuse-migrate-deployment.ts +200 -0
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-migrate-shape.ts +202 -0
- package/bin/apifuse-pack-check.ts +19 -6
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +736 -38
- package/bin/apifuse-perf.ts +59 -23
- package/bin/apifuse-record.ts +923 -94
- package/bin/apifuse-submit-check.ts +2313 -281
- package/bin/apifuse-test.ts +3 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +30 -0
- package/dist/cli/create.d.ts +1 -1
- package/dist/cli/create.js +8 -2
- package/dist/cli/deployment-intent.d.ts +178 -0
- package/dist/cli/deployment-intent.js +277 -0
- package/dist/cli/local-connection.d.ts +111 -0
- package/dist/cli/local-connection.js +151 -0
- package/dist/cli/migrate-deployment-intent.d.ts +132 -0
- package/dist/cli/migrate-deployment-intent.js +1552 -0
- package/dist/cli/migrate-operation-declaration.d.ts +70 -0
- package/dist/cli/migrate-operation-declaration.js +2206 -0
- package/dist/cli/migrate-operation-shape.d.ts +44 -0
- package/dist/cli/migrate-operation-shape.js +113 -0
- package/dist/cli/migrate-provider-shape.d.ts +56 -0
- package/dist/cli/migrate-provider-shape.js +580 -0
- package/dist/cli/resolver-runtime.d.ts +21 -0
- package/dist/cli/resolver-runtime.js +31 -0
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +141 -23
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +41 -19
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/dist/cli/templates/provider/provider.json.tpl +6 -0
- package/dist/config/loader.d.ts +202 -16
- package/dist/config/loader.js +491 -141
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +12 -5
- package/dist/contract.js +24 -10
- package/dist/declaration-validation.d.ts +34 -0
- package/dist/declaration-validation.js +305 -0
- package/dist/define.d.ts +71 -29
- package/dist/define.js +807 -69
- package/dist/dev.d.ts +5 -0
- package/dist/dev.js +8 -1
- package/dist/engine.d.ts +129 -0
- package/dist/engine.js +209 -0
- package/dist/error-observability.d.ts +7 -0
- package/dist/error-observability.js +61 -0
- package/dist/error-resolution.d.ts +18 -0
- package/dist/error-resolution.js +243 -0
- package/dist/errors.d.ts +74 -0
- package/dist/errors.js +99 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +371 -0
- package/dist/handle-meta.d.ts +90 -0
- package/dist/handle-meta.js +73 -0
- package/dist/handle.d.ts +226 -0
- package/dist/handle.js +345 -0
- package/dist/health-scenario.d.ts +1901 -0
- package/dist/health-scenario.js +653 -0
- package/dist/i18n/catalog.js +21 -1
- package/dist/i18n/error-messages.d.ts +86 -0
- package/dist/i18n/error-messages.js +192 -0
- package/dist/i18n/index.d.ts +1 -0
- package/dist/i18n/index.js +1 -0
- package/dist/i18n/keys.d.ts +12 -0
- package/dist/i18n/keys.js +76 -5
- package/dist/i18n/operation-locale-namespace.d.ts +2 -0
- package/dist/i18n/operation-locale-namespace.js +7 -0
- package/dist/index.d.ts +40 -15
- package/dist/index.js +29 -11
- package/dist/lint.d.ts +43 -11
- package/dist/lint.js +2512 -177
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +18 -8
- package/dist/provider.js +11 -6
- package/dist/runtime/auth-flow.d.ts +6 -1
- package/dist/runtime/auth-flow.js +8 -0
- package/dist/runtime/bounded-expiring-map.d.ts +21 -0
- package/dist/runtime/bounded-expiring-map.js +64 -0
- package/dist/runtime/browser-telemetry-binding.d.ts +5 -0
- package/dist/runtime/browser-telemetry-binding.js +83 -0
- package/dist/runtime/browser-telemetry.d.ts +76 -0
- package/dist/runtime/browser-telemetry.js +165 -0
- package/dist/runtime/browser.d.ts +4 -0
- package/dist/runtime/browser.js +688 -83
- package/dist/runtime/cache-telemetry.d.ts +71 -0
- package/dist/runtime/cache-telemetry.js +177 -0
- package/dist/runtime/cache.d.ts +3 -0
- package/dist/runtime/cache.js +245 -23
- package/dist/runtime/capability-response.d.ts +3 -0
- package/dist/runtime/capability-response.js +56 -0
- package/dist/runtime/capability-telemetry.d.ts +46 -0
- package/dist/runtime/capability-telemetry.js +68 -0
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/diagnostic-env-values.d.ts +5 -0
- package/dist/runtime/diagnostic-env-values.js +57 -0
- package/dist/runtime/diagnostic-env.d.ts +14 -0
- package/dist/runtime/diagnostic-env.js +44 -0
- package/dist/runtime/diagnostic-redactor.d.ts +54 -0
- package/dist/runtime/diagnostic-redactor.js +571 -0
- package/dist/runtime/egress-lease.d.ts +65 -0
- package/dist/runtime/egress-lease.js +236 -0
- package/dist/runtime/engine-mode.d.ts +74 -0
- package/dist/runtime/engine-mode.js +152 -0
- package/dist/runtime/env.js +14 -1
- package/dist/runtime/executor.d.ts +3 -2
- package/dist/runtime/executor.js +23 -26
- package/dist/runtime/handle-wordlist.d.ts +43 -0
- package/dist/runtime/handle-wordlist.js +256 -0
- package/dist/runtime/handle.d.ts +71 -0
- package/dist/runtime/handle.js +908 -0
- package/dist/runtime/http-telemetry-binding.d.ts +10 -0
- package/dist/runtime/http-telemetry-binding.js +152 -0
- package/dist/runtime/http-telemetry-guard.d.ts +16 -0
- package/dist/runtime/http-telemetry-guard.js +202 -0
- package/dist/runtime/http-telemetry.d.ts +104 -0
- package/dist/runtime/http-telemetry.js +190 -0
- package/dist/runtime/http.d.ts +34 -0
- package/dist/runtime/http.js +680 -121
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +462 -19
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +105 -0
- package/dist/runtime/native-network.js +1433 -0
- package/dist/runtime/native-telemetry.d.ts +142 -0
- package/dist/runtime/native-telemetry.js +231 -0
- package/dist/runtime/ocr-telemetry.d.ts +82 -0
- package/dist/runtime/ocr-telemetry.js +192 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +468 -0
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +398 -16
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +147 -0
- package/dist/runtime/proxy-retry-policy.d.ts +4 -1
- package/dist/runtime/proxy-retry-policy.js +11 -2
- package/dist/runtime/proxy-telemetry.d.ts +133 -3
- package/dist/runtime/proxy-telemetry.js +218 -70
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +61 -0
- package/dist/runtime/redirects.js +149 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +75 -1
- package/dist/runtime/request-options.js +588 -0
- package/dist/runtime/request-telemetry.d.ts +70 -0
- package/dist/runtime/request-telemetry.js +229 -0
- package/dist/runtime/resolver-config.d.ts +13 -0
- package/dist/runtime/resolver-config.js +14 -0
- package/dist/runtime/resolver-public.d.ts +3 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-runtime-options.d.ts +14 -0
- package/dist/runtime/resolver-runtime-options.js +33 -0
- package/dist/runtime/resolver-shared.d.ts +24 -0
- package/dist/runtime/resolver-shared.js +69 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-telemetry.d.ts +135 -0
- package/dist/runtime/resolver-telemetry.js +204 -0
- package/dist/runtime/resolver-usage.d.ts +32 -0
- package/dist/runtime/resolver-usage.js +54 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +53 -0
- package/dist/runtime/resolver-vendors/bindings.js +45 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +24 -0
- package/dist/runtime/resolver-vendors/browser.js +423 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +544 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +23 -0
- package/dist/runtime/resolver-vendors/hosts.js +68 -0
- package/dist/runtime/resolver-vendors/hypersolutions.d.ts +27 -0
- package/dist/runtime/resolver-vendors/hypersolutions.js +369 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +425 -0
- package/dist/runtime/resolver-vendors/types.d.ts +125 -0
- package/dist/runtime/resolver-vendors/types.js +106 -0
- package/dist/runtime/resolver.d.ts +79 -0
- package/dist/runtime/resolver.js +1094 -0
- package/dist/runtime/state-telemetry.d.ts +79 -0
- package/dist/runtime/state-telemetry.js +194 -0
- package/dist/runtime/state.d.ts +10 -1
- package/dist/runtime/state.js +322 -82
- package/dist/runtime/stealth-akamai-sbsd.d.ts +54 -0
- package/dist/runtime/stealth-akamai-sbsd.js +142 -0
- package/dist/runtime/stealth-cookies.d.ts +24 -0
- package/dist/runtime/stealth-cookies.js +131 -0
- package/dist/runtime/stealth-owned-headers.d.ts +39 -0
- package/dist/runtime/stealth-owned-headers.js +57 -0
- package/dist/runtime/stealth-telemetry.d.ts +92 -0
- package/dist/runtime/stealth-telemetry.js +179 -0
- package/dist/runtime/stealth.d.ts +45 -6
- package/dist/runtime/stealth.js +1775 -320
- package/dist/runtime/stt-telemetry.d.ts +77 -0
- package/dist/runtime/stt-telemetry.js +176 -0
- package/dist/runtime/stt.js +18 -19
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +62 -0
- package/dist/runtime/trace.d.ts +17 -0
- package/dist/runtime/trace.js +176 -25
- package/dist/runtime-boundary-lint.d.ts +137 -0
- package/dist/runtime-boundary-lint.js +590 -0
- package/dist/schema.d.ts +20 -0
- package/dist/schema.js +72 -2
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/error-observability.d.ts +1 -0
- package/dist/server/error-observability.js +1 -0
- package/dist/server/error-stack-frames.d.ts +35 -0
- package/dist/server/error-stack-frames.js +219 -0
- package/dist/server/index.d.ts +21 -2
- package/dist/server/index.js +11 -2
- package/dist/server/runtime-diagnostics.d.ts +165 -0
- package/dist/server/runtime-diagnostics.js +360 -0
- package/dist/server/sdk-version.d.ts +3 -0
- package/dist/server/sdk-version.js +19 -0
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test-redaction.js +8 -33
- package/dist/server/self-test.d.ts +18 -3
- package/dist/server/self-test.js +139 -76
- package/dist/server/sensitive-values.d.ts +26 -0
- package/dist/server/sensitive-values.js +137 -0
- package/dist/server/serve-implementation.d.ts +285 -0
- package/dist/server/serve-implementation.js +3172 -0
- package/dist/server/serve.d.ts +1 -75
- package/dist/server/serve.js +1 -1170
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/dist/server/trace-output.d.ts +7 -0
- package/dist/server/trace-output.js +111 -0
- package/dist/server/types.d.ts +37 -5
- package/dist/server/types.js +34 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +237 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +35 -0
- package/dist/stateful/stateful-provider-runtime-context.js +61 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +15 -4
- package/dist/stealth/profiles.js +169 -211
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +509 -25
- package/dist/trace-sanitization.d.ts +6 -0
- package/dist/trace-sanitization.js +68 -0
- package/dist/types.d.ts +899 -210
- package/dist/types.js +1 -0
- package/dist/typescript-module.d.ts +39 -0
- package/dist/typescript-module.js +72 -0
- package/package.json +52 -6
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/commands.ts +37 -0
- package/src/cli/create.ts +8 -2
- package/src/cli/deployment-intent.ts +364 -0
- package/src/cli/local-connection.ts +221 -0
- package/src/cli/migrate-deployment-intent.ts +1908 -0
- package/src/cli/migrate-operation-declaration.ts +3026 -0
- package/src/cli/migrate-operation-shape.ts +184 -0
- package/src/cli/migrate-provider-shape.ts +778 -0
- package/src/cli/resolver-runtime.ts +72 -0
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +141 -23
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +41 -19
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/src/cli/templates/provider/provider.json.tpl +6 -0
- package/src/config/loader.ts +774 -181
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +12 -5
- package/src/contract.ts +24 -10
- package/src/declaration-validation.ts +376 -0
- package/src/define.ts +1105 -131
- package/src/dev.ts +13 -1
- package/src/engine.ts +390 -0
- package/src/error-observability.ts +64 -0
- package/src/error-resolution.ts +269 -0
- package/src/errors.ts +157 -0
- package/src/fixture-sanitization.ts +422 -0
- package/src/handle-meta.ts +143 -0
- package/src/handle.ts +694 -0
- package/src/health-scenario.ts +960 -0
- package/src/i18n/catalog.ts +23 -1
- package/src/i18n/error-messages.ts +255 -0
- package/src/i18n/index.ts +1 -0
- package/src/i18n/keys.ts +80 -6
- package/src/i18n/operation-locale-namespace.ts +10 -0
- package/src/index.ts +390 -23
- package/src/lint.ts +2975 -234
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +207 -27
- package/src/runtime/auth-flow.ts +15 -0
- package/src/runtime/bounded-expiring-map.ts +67 -0
- package/src/runtime/browser-telemetry-binding.ts +92 -0
- package/src/runtime/browser-telemetry.ts +289 -0
- package/src/runtime/browser.ts +882 -99
- package/src/runtime/cache-telemetry.ts +247 -0
- package/src/runtime/cache.ts +258 -22
- package/src/runtime/capability-response.ts +53 -0
- package/src/runtime/capability-telemetry.ts +101 -0
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/diagnostic-env-values.ts +56 -0
- package/src/runtime/diagnostic-env.ts +65 -0
- package/src/runtime/diagnostic-redactor.ts +668 -0
- package/src/runtime/egress-lease.ts +341 -0
- package/src/runtime/engine-mode.ts +233 -0
- package/src/runtime/env.ts +15 -1
- package/src/runtime/executor.ts +39 -32
- package/src/runtime/handle-wordlist.ts +266 -0
- package/src/runtime/handle.ts +1180 -0
- package/src/runtime/http-telemetry-binding.ts +162 -0
- package/src/runtime/http-telemetry-guard.ts +208 -0
- package/src/runtime/http-telemetry.ts +292 -0
- package/src/runtime/http.ts +896 -150
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/instrumentation.ts +628 -28
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1853 -0
- package/src/runtime/native-telemetry.ts +366 -0
- package/src/runtime/ocr-telemetry.ts +308 -0
- package/src/runtime/ocr.ts +564 -0
- package/src/runtime/otlp.ts +468 -21
- package/src/runtime/provider.ts +15 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +222 -0
- package/src/runtime/proxy-retry-policy.ts +12 -1
- package/src/runtime/proxy-telemetry.ts +358 -103
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +215 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +728 -1
- package/src/runtime/request-telemetry.ts +375 -0
- package/src/runtime/resolver-config.ts +18 -0
- package/src/runtime/resolver-public.ts +44 -0
- package/src/runtime/resolver-runtime-options.ts +50 -0
- package/src/runtime/resolver-shared.ts +113 -0
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-telemetry.ts +362 -0
- package/src/runtime/resolver-usage.ts +99 -0
- package/src/runtime/resolver-vendors/bindings.ts +61 -0
- package/src/runtime/resolver-vendors/browser.ts +591 -0
- package/src/runtime/resolver-vendors/capsolver.ts +727 -0
- package/src/runtime/resolver-vendors/hosts.ts +81 -0
- package/src/runtime/resolver-vendors/hypersolutions.ts +480 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +555 -0
- package/src/runtime/resolver-vendors/types.ts +261 -0
- package/src/runtime/resolver.ts +1543 -0
- package/src/runtime/state-telemetry.ts +298 -0
- package/src/runtime/state.ts +435 -89
- package/src/runtime/stealth-akamai-sbsd.ts +207 -0
- package/src/runtime/stealth-cookies.ts +155 -0
- package/src/runtime/stealth-owned-headers.ts +67 -0
- package/src/runtime/stealth-telemetry.ts +309 -0
- package/src/runtime/stealth.ts +2275 -372
- package/src/runtime/stt-telemetry.ts +282 -0
- package/src/runtime/stt.ts +96 -91
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +78 -0
- package/src/runtime/trace.ts +257 -35
- package/src/runtime-boundary-lint.ts +707 -0
- package/src/schema.ts +80 -2
- package/src/serve.ts +6 -1
- package/src/server/error-observability.ts +1 -0
- package/src/server/error-stack-frames.ts +214 -0
- package/src/server/index.ts +184 -2
- package/src/server/runtime-diagnostics.ts +522 -0
- package/src/server/sdk-version.ts +22 -0
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test-redaction.ts +10 -40
- package/src/server/self-test.ts +205 -89
- package/src/server/sensitive-values.ts +211 -0
- package/src/server/serve-implementation.ts +4596 -0
- package/src/server/serve.ts +1 -1685
- package/src/server/state-backend-report.ts +185 -0
- package/src/server/trace-output.ts +189 -0
- package/src/server/types.ts +34 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +310 -0
- package/src/stateful/stateful-provider-runtime-context.ts +96 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +212 -231
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +685 -19
- package/src/trace-sanitization.ts +110 -0
- package/src/types.ts +1014 -273
- package/src/typescript-module.ts +100 -0
- package/dist/choice-token.d.ts +0 -24
- package/dist/choice-token.js +0 -74
- package/dist/runtime/choice.d.ts +0 -15
- package/dist/runtime/choice.js +0 -413
- package/src/choice-token.ts +0 -165
- package/src/runtime/choice.ts +0 -686
package/src/config/loader.ts
CHANGED
|
@@ -1,11 +1,33 @@
|
|
|
1
|
+
import { readDiagnosticEnv } from "../runtime/diagnostic-env.js";
|
|
1
2
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
3
|
import { existsSync } from "node:fs";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
3
5
|
import path from "node:path";
|
|
4
6
|
|
|
5
|
-
import { Redis } from "ioredis";
|
|
6
|
-
|
|
7
|
-
import type { ProviderProxyPolicy, TraceConfig } from "../types.js";
|
|
8
|
-
|
|
7
|
+
import type { Redis } from "ioredis";
|
|
8
|
+
|
|
9
|
+
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
10
|
+
import { BoundedExpiringMap } from "../runtime/bounded-expiring-map.js";
|
|
11
|
+
import {
|
|
12
|
+
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
13
|
+
NODEMAVEN_FILTER_ENV,
|
|
14
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
15
|
+
NODEMAVEN_PASSWORD_ENV,
|
|
16
|
+
NODEMAVEN_USERNAME_ENV,
|
|
17
|
+
type ProxyProtocol,
|
|
18
|
+
nodemavenPoolSize,
|
|
19
|
+
synthesizeNodemavenProxy,
|
|
20
|
+
} from "../runtime/proxy-nodemaven.js";
|
|
21
|
+
|
|
22
|
+
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
23
|
+
|
|
24
|
+
/** Proxy vendors with SDK-managed resolution. */
|
|
25
|
+
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
26
|
+
|
|
27
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
28
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
29
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
30
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
9
31
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
10
32
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
11
33
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
@@ -17,15 +39,6 @@ export const PROVIDER_CACHE_REDIS_URL_ENV = "APIFUSE__PROVIDER__CACHE_REDIS_URL"
|
|
|
17
39
|
export const PROVIDER_STATE_REDIS_URL_ENV = "APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
18
40
|
export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
19
41
|
|
|
20
|
-
export type ProxyOptions = {
|
|
21
|
-
url: string;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type ProxyConfig = Partial<ProxyOptions> & {
|
|
25
|
-
provider?: string;
|
|
26
|
-
apiKey?: string;
|
|
27
|
-
};
|
|
28
|
-
|
|
29
42
|
export type BrowserConfig = {
|
|
30
43
|
executablePath?: string;
|
|
31
44
|
headless?: boolean;
|
|
@@ -37,7 +50,6 @@ export type SessionConfig = {
|
|
|
37
50
|
};
|
|
38
51
|
|
|
39
52
|
export type ApiFuseConfig = {
|
|
40
|
-
proxy?: ProxyConfig;
|
|
41
53
|
browser?: BrowserConfig;
|
|
42
54
|
session?: SessionConfig;
|
|
43
55
|
trace?: TraceConfig;
|
|
@@ -47,12 +59,33 @@ export type ApiFuseConfig = {
|
|
|
47
59
|
export type ProxyResolutionOptions = {
|
|
48
60
|
proxy?: string;
|
|
49
61
|
upstream?: { proxy?: boolean | ProviderProxyPolicy };
|
|
50
|
-
apifuseConfig?: Pick<ApiFuseConfig, "proxy">;
|
|
51
62
|
proxyPolicy?: ProviderProxyPolicy;
|
|
52
63
|
affinityKey?: string;
|
|
53
64
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
54
65
|
proxyAttempt?: number;
|
|
66
|
+
/**
|
|
67
|
+
* Tunnelling protocols the calling transport can use. When a resolved
|
|
68
|
+
* protocol is not in this set the resolver fails with
|
|
69
|
+
* `PROXY_PROTOCOL_UNSUPPORTED` instead of silently downgrading. Unset means
|
|
70
|
+
* permissive (both protocols allowed).
|
|
71
|
+
*/
|
|
72
|
+
transportProtocols?: readonly ProxyProtocol[];
|
|
73
|
+
/**
|
|
74
|
+
* Explicit protocol override. Internal — for the verification harness and
|
|
75
|
+
* tests, or an advanced caller. Normal callers omit it and each vendor uses
|
|
76
|
+
* its own benchmarked default protocol (see VENDOR_DEFAULT_PROTOCOL). Not an
|
|
77
|
+
* env var and not a provider-policy field.
|
|
78
|
+
*/
|
|
79
|
+
protocol?: ProxyProtocol;
|
|
80
|
+
/**
|
|
81
|
+
* Gateway pool "refresh" generation. Bumped by transports on pool refresh to
|
|
82
|
+
* derive a fresh gateway session set (ignored by allocation-style vendors,
|
|
83
|
+
* whose refresh is driven by cache invalidation).
|
|
84
|
+
*/
|
|
85
|
+
proxyRefreshEpoch?: number;
|
|
55
86
|
telemetry?: ProxyTelemetrySink;
|
|
87
|
+
/** Engine-owned credential snapshot. Provider code must never populate this field. */
|
|
88
|
+
engineCredentials?: Readonly<Record<string, string>>;
|
|
56
89
|
};
|
|
57
90
|
|
|
58
91
|
export type ProxyCacheStatus =
|
|
@@ -73,8 +106,14 @@ export type SmartproxyAllocatorBodyClass =
|
|
|
73
106
|
| "text_without_proxies"
|
|
74
107
|
| "usable_proxy_endpoints";
|
|
75
108
|
|
|
109
|
+
export type ProxyUserAgentSource = "declared" | "defaulted";
|
|
110
|
+
|
|
76
111
|
export type ProxyResolutionTelemetryEvent = {
|
|
77
|
-
provider:
|
|
112
|
+
provider: ProxyVendorName;
|
|
113
|
+
/** Defaults to `"ok"` when omitted. */
|
|
114
|
+
outcome?: "ok" | "error";
|
|
115
|
+
userAgentSource?: ProxyUserAgentSource;
|
|
116
|
+
protocol?: ProxyProtocol;
|
|
78
117
|
cacheStatus: ProxyCacheStatus;
|
|
79
118
|
cacheHit: boolean;
|
|
80
119
|
resolutionMs: number;
|
|
@@ -92,7 +131,7 @@ export type ProxyResolutionTelemetryEvent = {
|
|
|
92
131
|
};
|
|
93
132
|
|
|
94
133
|
export type ProxyAttemptTelemetryEvent = {
|
|
95
|
-
provider:
|
|
134
|
+
provider: ProxyVendorName;
|
|
96
135
|
attempt: number;
|
|
97
136
|
poolIndex?: number;
|
|
98
137
|
proxyHash?: string;
|
|
@@ -102,31 +141,69 @@ export type ProxyAttemptTelemetryEvent = {
|
|
|
102
141
|
durationMs?: number;
|
|
103
142
|
};
|
|
104
143
|
|
|
144
|
+
export type ProxyVendorFailoverTelemetryEvent = {
|
|
145
|
+
/** Vendor that failed or was skipped. */
|
|
146
|
+
vendor: ProxyVendorName;
|
|
147
|
+
/** Vendor tried next, or undefined when the chain is exhausted. */
|
|
148
|
+
nextVendor?: ProxyVendorName;
|
|
149
|
+
phase: "resolution" | "transport";
|
|
150
|
+
reason: "no_credentials" | "allocation_failed" | "pool_exhausted" | "protocol_unsupported";
|
|
151
|
+
attempt?: number;
|
|
152
|
+
};
|
|
153
|
+
|
|
105
154
|
export type ProxyTelemetrySink = {
|
|
106
155
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
107
156
|
recordProxyAttempt?(event: ProxyAttemptTelemetryEvent): void;
|
|
157
|
+
recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
108
158
|
};
|
|
109
159
|
|
|
160
|
+
export type ProxyResolutionSource =
|
|
161
|
+
| "explicit"
|
|
162
|
+
| "env"
|
|
163
|
+
| "config"
|
|
164
|
+
| "smartproxy-allocator"
|
|
165
|
+
| "nodemaven-gateway";
|
|
166
|
+
|
|
110
167
|
export type ResolvedProxyConfig = {
|
|
111
168
|
shouldWarn: boolean;
|
|
112
169
|
url?: string;
|
|
113
|
-
|
|
170
|
+
/** SDK-native vendor that supplied the URL, when applicable. */
|
|
171
|
+
vendor?: ProxyVendorName;
|
|
172
|
+
source?: ProxyResolutionSource;
|
|
173
|
+
protocol?: ProxyProtocol;
|
|
114
174
|
diagnostics?: Record<string, string | number | boolean>;
|
|
115
175
|
};
|
|
116
176
|
|
|
177
|
+
export type ProxyResolutionErrorCode =
|
|
178
|
+
| "PROXY_REQUIRED"
|
|
179
|
+
| "PROXY_ALLOCATION_FAILED"
|
|
180
|
+
| "PROXY_PROTOCOL_UNSUPPORTED";
|
|
181
|
+
|
|
117
182
|
export class ProxyResolutionError extends Error {
|
|
118
|
-
readonly code:
|
|
183
|
+
readonly code: ProxyResolutionErrorCode;
|
|
119
184
|
readonly telemetry?: ProxyResolutionTelemetryEvent;
|
|
185
|
+
readonly vendor?: ProxyVendorName;
|
|
186
|
+
readonly vendorChain?: ProxyVendorName[];
|
|
187
|
+
readonly protocol?: ProxyProtocol;
|
|
120
188
|
|
|
121
189
|
constructor(
|
|
122
|
-
code:
|
|
190
|
+
code: ProxyResolutionErrorCode,
|
|
123
191
|
message: string,
|
|
124
|
-
options?: {
|
|
192
|
+
options?: {
|
|
193
|
+
cause?: unknown;
|
|
194
|
+
telemetry?: ProxyResolutionTelemetryEvent;
|
|
195
|
+
vendor?: ProxyVendorName;
|
|
196
|
+
vendorChain?: ProxyVendorName[];
|
|
197
|
+
protocol?: ProxyProtocol;
|
|
198
|
+
},
|
|
125
199
|
) {
|
|
126
200
|
super(message, options);
|
|
127
201
|
this.name = "ProxyResolutionError";
|
|
128
202
|
this.code = code;
|
|
129
203
|
this.telemetry = options?.telemetry;
|
|
204
|
+
this.vendor = options?.vendor;
|
|
205
|
+
this.vendorChain = options?.vendorChain;
|
|
206
|
+
this.protocol = options?.protocol;
|
|
130
207
|
}
|
|
131
208
|
}
|
|
132
209
|
|
|
@@ -148,7 +225,14 @@ type ProxyRedisClient = Pick<
|
|
|
148
225
|
"connect" | "del" | "eval" | "get" | "on" | "pttl" | "set" | "status"
|
|
149
226
|
>;
|
|
150
227
|
|
|
151
|
-
|
|
228
|
+
// Keyed per (policy, affinity). Connection affinities are minted by callers, so
|
|
229
|
+
// without a cap a long-lived process accumulates one pool per affinity it ever
|
|
230
|
+
// saw. Evicting a still-fresh pool forces a fresh allocation (a different
|
|
231
|
+
// egress endpoint) for that affinity, so the cap is generous and every
|
|
232
|
+
// capacity eviction is reported as `diagnostics.poolCacheEvictions`.
|
|
233
|
+
const SMARTPROXY_POOL_CACHE_MAX_ENTRIES = 10_000;
|
|
234
|
+
let proxyCache = createSmartproxyPoolCache(SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
|
|
235
|
+
const require = createRequire(import.meta.url);
|
|
152
236
|
const proxyInflight = new Map<string, Promise<SmartproxyAllocationResult>>();
|
|
153
237
|
const invalidatedProxyKeys = new Map<string, number>();
|
|
154
238
|
const redisClients = new Map<string, ProxyRedisClient>();
|
|
@@ -171,25 +255,62 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
|
|
|
171
255
|
const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
|
|
172
256
|
const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
|
|
173
257
|
|
|
258
|
+
/** Env names that can supply a provider store's Redis URL. */
|
|
259
|
+
export type ProviderRedisUrlEnvName =
|
|
260
|
+
| typeof PROVIDER_STATE_REDIS_URL_ENV
|
|
261
|
+
| typeof PROVIDER_CACHE_REDIS_URL_ENV
|
|
262
|
+
| typeof REDIS_URL_ENV;
|
|
263
|
+
|
|
264
|
+
/** Which env name supplied a store's Redis URL, alongside the URL itself. */
|
|
265
|
+
export type ProviderRedisUrlResolution = {
|
|
266
|
+
readonly url: string;
|
|
267
|
+
/** The env name that actually supplied `url`. */
|
|
268
|
+
readonly envName: ProviderRedisUrlEnvName;
|
|
269
|
+
/**
|
|
270
|
+
* True when `url` came from a shared fallback env rather than the store's
|
|
271
|
+
* own. A state store on `fallback: true` is sharing an instance that was
|
|
272
|
+
* provisioned for something else (apifuse#2144).
|
|
273
|
+
*/
|
|
274
|
+
readonly fallback: boolean;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
// Precedence is declared once here so the URL accessors, the boot report and
|
|
278
|
+
// any future consumer can never disagree about which env wins.
|
|
279
|
+
const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV] as const;
|
|
280
|
+
const STATE_REDIS_URL_PRECEDENCE = [
|
|
281
|
+
PROVIDER_STATE_REDIS_URL_ENV,
|
|
282
|
+
PROVIDER_CACHE_REDIS_URL_ENV,
|
|
283
|
+
REDIS_URL_ENV,
|
|
284
|
+
] as const;
|
|
285
|
+
|
|
286
|
+
function resolveRedisUrlFromEnv(
|
|
287
|
+
precedence: readonly ProviderRedisUrlEnvName[],
|
|
288
|
+
): ProviderRedisUrlResolution | undefined {
|
|
289
|
+
for (const [index, envName] of precedence.entries()) {
|
|
290
|
+
const url = readDiagnosticEnv(envName)?.trim();
|
|
291
|
+
if (url) return { url, envName, fallback: index > 0 };
|
|
292
|
+
}
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
|
|
174
296
|
function redisUrlFromEnv(): string | undefined {
|
|
175
|
-
return (
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
);
|
|
297
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
export function providerCacheRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
|
|
301
|
+
return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export function providerStateRedisResolutionFromEnv(): ProviderRedisUrlResolution | undefined {
|
|
305
|
+
return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
|
|
180
306
|
}
|
|
181
307
|
|
|
182
308
|
export function providerCacheRedisUrlFromEnv(): string | undefined {
|
|
183
|
-
return
|
|
309
|
+
return providerCacheRedisResolutionFromEnv()?.url;
|
|
184
310
|
}
|
|
185
311
|
|
|
186
312
|
export function providerStateRedisUrlFromEnv(): string | undefined {
|
|
187
|
-
return (
|
|
188
|
-
process.env[PROVIDER_STATE_REDIS_URL_ENV]?.trim() ||
|
|
189
|
-
process.env[PROVIDER_CACHE_REDIS_URL_ENV]?.trim() ||
|
|
190
|
-
process.env[REDIS_URL_ENV]?.trim() ||
|
|
191
|
-
undefined
|
|
192
|
-
);
|
|
313
|
+
return providerStateRedisResolutionFromEnv()?.url;
|
|
193
314
|
}
|
|
194
315
|
|
|
195
316
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|
|
@@ -201,6 +322,22 @@ export function __setSmartproxyAllocatorDeadlineMsForTests(deadlineMs: number |
|
|
|
201
322
|
smartproxyAllocatorDeadlineMsForTests = deadlineMs;
|
|
202
323
|
}
|
|
203
324
|
|
|
325
|
+
/** @internal Test-only hook for exercising the pool-cache capacity bound at a small size. */
|
|
326
|
+
export function __setSmartproxyPoolCacheMaxEntriesForTests(maxEntries: number | undefined): void {
|
|
327
|
+
proxyCache = createSmartproxyPoolCache(maxEntries ?? SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** @internal Test-only view of the in-memory Smartproxy cache sizes. */
|
|
331
|
+
export function __getSmartproxyCacheSizesForTests(): { pools: number; tombstones: number } {
|
|
332
|
+
return { pools: proxyCache.size, tombstones: invalidatedProxyKeys.size };
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function createSmartproxyPoolCache(
|
|
336
|
+
maxEntries: number,
|
|
337
|
+
): BoundedExpiringMap<string, CachedProxyPool> {
|
|
338
|
+
return new BoundedExpiringMap(maxEntries, (pool) => pool.expiresAt);
|
|
339
|
+
}
|
|
340
|
+
|
|
204
341
|
function getProxyRedis(): ProxyRedisClient | undefined {
|
|
205
342
|
if (proxyRedisForTests) return proxyRedisForTests;
|
|
206
343
|
const redisUrl = redisUrlFromEnv();
|
|
@@ -208,7 +345,8 @@ function getProxyRedis(): ProxyRedisClient | undefined {
|
|
|
208
345
|
const existing = redisClients.get(redisUrl);
|
|
209
346
|
if (existing) return existing;
|
|
210
347
|
|
|
211
|
-
const
|
|
348
|
+
const { Redis: RedisClient } = require("ioredis") as typeof import("ioredis");
|
|
349
|
+
const redis = new RedisClient(redisUrl, {
|
|
212
350
|
connectTimeout: REDIS_TIMEOUT_MS,
|
|
213
351
|
enableOfflineQueue: false,
|
|
214
352
|
lazyConnect: true,
|
|
@@ -296,6 +434,7 @@ function telemetryForFailure(
|
|
|
296
434
|
): ProxyResolutionTelemetryEvent {
|
|
297
435
|
return {
|
|
298
436
|
provider: "smartproxy",
|
|
437
|
+
outcome: "error",
|
|
299
438
|
cacheStatus,
|
|
300
439
|
cacheHit: false,
|
|
301
440
|
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
@@ -365,70 +504,7 @@ function serializeSmartproxyPool(pool: CachedProxyPool): string {
|
|
|
365
504
|
|
|
366
505
|
function normalizeProxyUrl(url?: string): string | undefined {
|
|
367
506
|
const normalized = url?.trim();
|
|
368
|
-
return normalized
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
function readPositiveIntegerEnv(name: string): string | undefined {
|
|
372
|
-
const raw = process.env[name]?.trim();
|
|
373
|
-
if (!raw) return undefined;
|
|
374
|
-
if (!/^[1-9]\d*$/.test(raw)) {
|
|
375
|
-
throw new Error(`${name} must be a positive integer`);
|
|
376
|
-
}
|
|
377
|
-
return raw;
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
function applyStickyProxySession(proxyUrl: string): string {
|
|
381
|
-
let parsed: URL;
|
|
382
|
-
try {
|
|
383
|
-
parsed = new URL(proxyUrl);
|
|
384
|
-
} catch {
|
|
385
|
-
return proxyUrl;
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
389
|
-
return proxyUrl;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
const host = parsed.hostname.toLowerCase();
|
|
393
|
-
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
394
|
-
return proxyUrl;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
const username = decodeURIComponent(parsed.username);
|
|
398
|
-
const sessionId = process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
|
|
399
|
-
const sessionDuration = readPositiveIntegerEnv("APIFUSE__PROXY__SESSION_DURATION");
|
|
400
|
-
const stickyUsername = host.includes("smartproxy")
|
|
401
|
-
? buildSmartproxyUsername(username, sessionId, sessionDuration)
|
|
402
|
-
: buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
|
|
403
|
-
|
|
404
|
-
parsed.username = stickyUsername;
|
|
405
|
-
return parsed.toString();
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
function buildSmartproxyUsername(
|
|
409
|
-
username: string,
|
|
410
|
-
sessionId: string,
|
|
411
|
-
sessionDuration?: string,
|
|
412
|
-
): string {
|
|
413
|
-
const parts = username.split("_");
|
|
414
|
-
const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
|
|
415
|
-
const baseUsername = parts
|
|
416
|
-
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
417
|
-
.join("_");
|
|
418
|
-
return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
function buildDecodoUsername(username: string, sessionId: string, sessionDuration: string): string {
|
|
422
|
-
const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
|
|
423
|
-
const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
|
|
424
|
-
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
function syncProxyEnv(config: ApiFuseConfig): void {
|
|
428
|
-
const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
|
|
429
|
-
if (!process.env.APIFUSE__PROXY__URL && configProxyUrl) {
|
|
430
|
-
process.env.APIFUSE__PROXY__URL = configProxyUrl;
|
|
431
|
-
}
|
|
507
|
+
return normalized || undefined;
|
|
432
508
|
}
|
|
433
509
|
|
|
434
510
|
export function resolveProxyConfig(options: ProxyResolutionOptions = {}): ResolvedProxyConfig {
|
|
@@ -448,16 +524,6 @@ export function resolveProxyConfig(options: ProxyResolutionOptions = {}): Resolv
|
|
|
448
524
|
return { shouldWarn: false };
|
|
449
525
|
}
|
|
450
526
|
|
|
451
|
-
const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE__PROXY__URL);
|
|
452
|
-
if (envProxyUrl) {
|
|
453
|
-
return { shouldWarn: false, url: envProxyUrl };
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
const configuredProxyUrl = normalizeProxyUrl(options.apifuseConfig?.proxy?.url);
|
|
457
|
-
if (configuredProxyUrl) {
|
|
458
|
-
return { shouldWarn: false, url: configuredProxyUrl };
|
|
459
|
-
}
|
|
460
|
-
|
|
461
527
|
return { shouldWarn: true };
|
|
462
528
|
}
|
|
463
529
|
|
|
@@ -477,60 +543,287 @@ export async function resolveProxyConfigAsync(
|
|
|
477
543
|
return { shouldWarn: false };
|
|
478
544
|
}
|
|
479
545
|
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
546
|
+
const chain = resolveVendorChain(policy);
|
|
547
|
+
if (chain.length === 0) {
|
|
548
|
+
const declared = declaredVendorChain(policy);
|
|
549
|
+
const deprecated = declared.filter((vendor) => vendor === "decodo" || vendor === "custom");
|
|
550
|
+
if (policy.mode === "required") {
|
|
551
|
+
const providerIds =
|
|
552
|
+
declared.length > 0 ? declared.map((vendor) => `"${vendor}"`).join(", ") : "none";
|
|
553
|
+
const deprecatedDetail =
|
|
554
|
+
deprecated.length > 0
|
|
555
|
+
? ` Deprecated vendor(s): ${deprecated.map((vendor) => `"${vendor}"`).join(", ")}.`
|
|
556
|
+
: "";
|
|
557
|
+
throw new ProxyResolutionError(
|
|
558
|
+
"PROXY_REQUIRED",
|
|
559
|
+
`Required proxy policy has no SDK-managed adapter for provider id(s): ${providerIds}.${deprecatedDetail} Use "smartproxy" or "nodemaven".`,
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
// Deprecated decodo/custom providers have no SDK-managed adapter. Optional
|
|
563
|
+
// policies preserve the warning-only behavior and may continue directly.
|
|
482
564
|
return resolveProxyConfig({
|
|
483
565
|
...options,
|
|
484
566
|
upstream: { proxy: true },
|
|
485
567
|
});
|
|
486
568
|
}
|
|
487
569
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
570
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
571
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
572
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
573
|
+
const transportProtocols = options.transportProtocols ?? (["http", "socks5"] as const);
|
|
574
|
+
|
|
575
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
576
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
577
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
578
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(
|
|
579
|
+
total > 0 ? normalizedAttempt % total : 0,
|
|
580
|
+
sizes,
|
|
581
|
+
);
|
|
582
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
583
|
+
|
|
584
|
+
let lastError: unknown;
|
|
585
|
+
let blockedProtocol: ProxyProtocol | undefined;
|
|
586
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
587
|
+
const vendor = chain[vendorIndex] as ProxyVendorName;
|
|
588
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
589
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
590
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
591
|
+
|
|
592
|
+
if (!vendorHasCredentials(vendor, options.engineCredentials)) {
|
|
593
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
594
|
+
vendor,
|
|
595
|
+
nextVendor,
|
|
596
|
+
phase: "resolution",
|
|
597
|
+
reason: "no_credentials",
|
|
598
|
+
});
|
|
599
|
+
continue;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
603
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
604
|
+
if (!transportProtocols.includes(protocol)) {
|
|
605
|
+
blockedProtocol = protocol;
|
|
606
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
607
|
+
vendor,
|
|
608
|
+
nextVendor,
|
|
609
|
+
phase: "resolution",
|
|
610
|
+
reason: "protocol_unsupported",
|
|
611
|
+
});
|
|
612
|
+
continue;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
try {
|
|
616
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
617
|
+
protocol,
|
|
618
|
+
poolIndex,
|
|
619
|
+
refreshEpoch,
|
|
620
|
+
credentials: options.engineCredentials,
|
|
621
|
+
});
|
|
622
|
+
} catch (error) {
|
|
623
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
624
|
+
// outages — propagate them rather than failing over.
|
|
625
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
626
|
+
throw error;
|
|
627
|
+
}
|
|
628
|
+
if (error.telemetry) {
|
|
629
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
630
|
+
}
|
|
631
|
+
lastError = error;
|
|
632
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
633
|
+
vendor,
|
|
634
|
+
nextVendor,
|
|
635
|
+
phase: "resolution",
|
|
636
|
+
reason: "allocation_failed",
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
if (policy.mode === "required") {
|
|
642
|
+
if (lastError) {
|
|
643
|
+
throw lastError instanceof ProxyResolutionError
|
|
644
|
+
? lastError
|
|
645
|
+
: new ProxyResolutionError(
|
|
646
|
+
"PROXY_ALLOCATION_FAILED",
|
|
647
|
+
`All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`,
|
|
648
|
+
{ cause: lastError, vendorChain: chain },
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
if (blockedProtocol) {
|
|
491
652
|
throw new ProxyResolutionError(
|
|
492
|
-
"
|
|
493
|
-
`
|
|
653
|
+
"PROXY_PROTOCOL_UNSUPPORTED",
|
|
654
|
+
`No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`,
|
|
655
|
+
{ protocol: blockedProtocol, vendorChain: chain },
|
|
494
656
|
);
|
|
495
657
|
}
|
|
496
|
-
|
|
658
|
+
throw new ProxyResolutionError(
|
|
659
|
+
"PROXY_REQUIRED",
|
|
660
|
+
`Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
661
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
662
|
+
.join(", ")}.`,
|
|
663
|
+
{ vendorChain: chain },
|
|
664
|
+
);
|
|
497
665
|
}
|
|
498
|
-
|
|
666
|
+
return { shouldWarn: true };
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/**
|
|
670
|
+
* Resolve the proxy URL for a provider-owned consumer such as a CAPTCHA solver.
|
|
671
|
+
* Vendor allocation and failover remain owned by the SDK.
|
|
672
|
+
*/
|
|
673
|
+
export async function resolveProxy(
|
|
674
|
+
options: ProxyResolutionOptions = {},
|
|
675
|
+
): Promise<ResolvedProxyConfig> {
|
|
676
|
+
const resolved = await resolveProxyConfigAsync(options);
|
|
677
|
+
const vendor = vendorFromResolvedSource(resolved.source);
|
|
678
|
+
return vendor ? { ...resolved, vendor } : resolved;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
683
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
684
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
685
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
686
|
+
*/
|
|
687
|
+
export const VENDOR_DEFAULT_PROTOCOL: Readonly<Record<ProxyVendorName, ProxyProtocol>> = {
|
|
688
|
+
smartproxy: "http",
|
|
689
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
690
|
+
};
|
|
499
691
|
|
|
692
|
+
/**
|
|
693
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
694
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
695
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
696
|
+
*/
|
|
697
|
+
export function assertTunnelingScheme(url: string): void {
|
|
698
|
+
let scheme: string;
|
|
500
699
|
try {
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
700
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
701
|
+
} catch {
|
|
702
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
703
|
+
}
|
|
704
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
705
|
+
throw new ProxyResolutionError(
|
|
706
|
+
"PROXY_ALLOCATION_FAILED",
|
|
707
|
+
`Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`,
|
|
506
708
|
);
|
|
507
|
-
|
|
508
|
-
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
export type ProxyVendorResolutionContext = {
|
|
713
|
+
readonly protocol: ProxyProtocol;
|
|
714
|
+
readonly poolIndex: number;
|
|
715
|
+
readonly refreshEpoch: number;
|
|
716
|
+
/** Explicit vendor credentials. Omit only on the legacy ambient-env path. */
|
|
717
|
+
readonly credentials?: Readonly<Record<string, string>>;
|
|
718
|
+
/** Disable non-policy env defaults for deterministic injected adapters. */
|
|
719
|
+
readonly ambientDefaults?: boolean;
|
|
720
|
+
/** Disable env-discovered Redis sharing for deterministic injected adapters. */
|
|
721
|
+
readonly sharedCache?: boolean;
|
|
722
|
+
};
|
|
723
|
+
|
|
724
|
+
export async function resolveWithVendor(
|
|
725
|
+
vendor: ProxyVendorName,
|
|
726
|
+
policy: ProviderProxyPolicy,
|
|
727
|
+
options: ProxyResolutionOptions,
|
|
728
|
+
context: ProxyVendorResolutionContext,
|
|
729
|
+
): Promise<ResolvedProxyConfig> {
|
|
730
|
+
if (vendor === "nodemaven") {
|
|
731
|
+
const startedAt = Date.now();
|
|
732
|
+
const username = (
|
|
733
|
+
context.credentials === undefined
|
|
734
|
+
? readDiagnosticEnv(NODEMAVEN_USERNAME_ENV)
|
|
735
|
+
: context.credentials[NODEMAVEN_USERNAME_ENV]
|
|
736
|
+
)?.trim();
|
|
737
|
+
const password = (
|
|
738
|
+
context.credentials === undefined
|
|
739
|
+
? readDiagnosticEnv(NODEMAVEN_PASSWORD_ENV)
|
|
740
|
+
: context.credentials[NODEMAVEN_PASSWORD_ENV]
|
|
741
|
+
)?.trim();
|
|
742
|
+
const filter =
|
|
743
|
+
context.credentials === undefined
|
|
744
|
+
? process.env[NODEMAVEN_FILTER_ENV]
|
|
745
|
+
: context.credentials[NODEMAVEN_FILTER_ENV];
|
|
746
|
+
if (!username || !password) {
|
|
747
|
+
throw new ProxyResolutionError(
|
|
748
|
+
"PROXY_ALLOCATION_FAILED",
|
|
749
|
+
`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`,
|
|
750
|
+
{ vendor: "nodemaven" },
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
754
|
+
policy,
|
|
755
|
+
credentials: {
|
|
756
|
+
username,
|
|
757
|
+
password,
|
|
758
|
+
...(filter ? { filter } : {}),
|
|
759
|
+
},
|
|
760
|
+
affinityKey: options.affinityKey,
|
|
761
|
+
protocol: context.protocol,
|
|
762
|
+
poolIndex: context.poolIndex,
|
|
763
|
+
refreshEpoch: context.refreshEpoch,
|
|
764
|
+
country: resolveSmartproxyCountry(policy, context.ambientDefaults !== false),
|
|
765
|
+
});
|
|
766
|
+
options.telemetry?.recordProxyResolution({
|
|
767
|
+
provider: "nodemaven",
|
|
768
|
+
protocol: synthesized.protocol,
|
|
769
|
+
cacheStatus: "disabled",
|
|
770
|
+
cacheHit: false,
|
|
771
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
772
|
+
attempts: 1,
|
|
773
|
+
});
|
|
774
|
+
assertTunnelingScheme(synthesized.url);
|
|
509
775
|
return {
|
|
510
776
|
shouldWarn: false,
|
|
511
|
-
url:
|
|
512
|
-
source: "
|
|
777
|
+
url: synthesized.url,
|
|
778
|
+
source: "nodemaven-gateway",
|
|
779
|
+
protocol: synthesized.protocol,
|
|
513
780
|
diagnostics: {
|
|
514
|
-
...
|
|
515
|
-
|
|
516
|
-
poolIndex,
|
|
781
|
+
...synthesized.diagnostics,
|
|
782
|
+
poolIndex: context.poolIndex,
|
|
517
783
|
},
|
|
518
784
|
};
|
|
519
|
-
} catch (error) {
|
|
520
|
-
if (error instanceof ProxyResolutionError && error.telemetry) {
|
|
521
|
-
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
522
|
-
}
|
|
523
|
-
if (policy.mode === "required") {
|
|
524
|
-
throw error instanceof ProxyResolutionError
|
|
525
|
-
? error
|
|
526
|
-
: new ProxyResolutionError(
|
|
527
|
-
"PROXY_ALLOCATION_FAILED",
|
|
528
|
-
"Smartproxy allocator failed for required proxy egress.",
|
|
529
|
-
{ cause: error },
|
|
530
|
-
);
|
|
531
|
-
}
|
|
532
|
-
return { shouldWarn: true };
|
|
533
785
|
}
|
|
786
|
+
|
|
787
|
+
// smartproxy allocation-style vendor.
|
|
788
|
+
const appKey = (
|
|
789
|
+
context.credentials === undefined
|
|
790
|
+
? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
|
|
791
|
+
: context.credentials[SMARTPROXY_APP_KEY_ENV]
|
|
792
|
+
)?.trim();
|
|
793
|
+
if (!appKey) {
|
|
794
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
795
|
+
throw new ProxyResolutionError(
|
|
796
|
+
"PROXY_ALLOCATION_FAILED",
|
|
797
|
+
`${SMARTPROXY_APP_KEY_ENV} is not configured.`,
|
|
798
|
+
{ vendor: "smartproxy" },
|
|
799
|
+
);
|
|
800
|
+
}
|
|
801
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy, context.ambientDefaults !== false);
|
|
802
|
+
const allocated = await allocateSmartproxy(
|
|
803
|
+
policy,
|
|
804
|
+
appKey,
|
|
805
|
+
lifetimeMinutes,
|
|
806
|
+
options.affinityKey,
|
|
807
|
+
context.protocol,
|
|
808
|
+
context.ambientDefaults !== false,
|
|
809
|
+
context.sharedCache !== false,
|
|
810
|
+
);
|
|
811
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
812
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
813
|
+
const url = allocated.pool.urls[poolIndex];
|
|
814
|
+
if (url) assertTunnelingScheme(url);
|
|
815
|
+
return {
|
|
816
|
+
shouldWarn: false,
|
|
817
|
+
url,
|
|
818
|
+
source: "smartproxy-allocator",
|
|
819
|
+
protocol: context.protocol,
|
|
820
|
+
diagnostics: {
|
|
821
|
+
...allocated.pool.diagnostics,
|
|
822
|
+
poolSize: allocated.pool.urls.length,
|
|
823
|
+
poolIndex,
|
|
824
|
+
poolCacheEvictions: proxyCache.capacityEvictions,
|
|
825
|
+
},
|
|
826
|
+
};
|
|
534
827
|
}
|
|
535
828
|
|
|
536
829
|
function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | undefined {
|
|
@@ -544,21 +837,240 @@ function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | u
|
|
|
544
837
|
return undefined;
|
|
545
838
|
}
|
|
546
839
|
|
|
547
|
-
function
|
|
548
|
-
return
|
|
549
|
-
|
|
840
|
+
function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
841
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
function declaredVendorChain(policy: ProviderProxyPolicy): ProviderProxyProvider[] {
|
|
845
|
+
const declared = policy.providers?.length
|
|
846
|
+
? policy.providers
|
|
847
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
848
|
+
return declared.filter((vendor): vendor is ProviderProxyProvider => vendor !== undefined);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
853
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
854
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
855
|
+
* an all-deprecated chain has no managed adapter.
|
|
856
|
+
*/
|
|
857
|
+
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
858
|
+
const chain: ProxyVendorName[] = [];
|
|
859
|
+
for (const name of declaredVendorChain(policy)) {
|
|
860
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
861
|
+
chain.push(name);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
return chain;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
function envDefaultProvider(): ProviderProxyProvider | undefined {
|
|
868
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
869
|
+
return (raw as ProviderProxyProvider | undefined) ?? undefined;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
function vendorHasCredentials(
|
|
873
|
+
vendor: ProxyVendorName,
|
|
874
|
+
credentials?: Readonly<Record<string, string>>,
|
|
875
|
+
): boolean {
|
|
876
|
+
if (vendor === "nodemaven") {
|
|
877
|
+
return Boolean(
|
|
878
|
+
(credentials === undefined
|
|
879
|
+
? readDiagnosticEnv(NODEMAVEN_USERNAME_ENV)
|
|
880
|
+
: credentials[NODEMAVEN_USERNAME_ENV]
|
|
881
|
+
)?.trim() &&
|
|
882
|
+
(credentials === undefined
|
|
883
|
+
? readDiagnosticEnv(NODEMAVEN_PASSWORD_ENV)
|
|
884
|
+
: credentials[NODEMAVEN_PASSWORD_ENV]
|
|
885
|
+
)?.trim(),
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
return Boolean(
|
|
889
|
+
(credentials === undefined
|
|
890
|
+
? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
|
|
891
|
+
: credentials[SMARTPROXY_APP_KEY_ENV]
|
|
892
|
+
)?.trim(),
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function missingCredentialEnv(vendor: ProxyVendorName): string {
|
|
897
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
function vendorPoolSize(vendor: ProxyVendorName, policy: ProviderProxyPolicy): number {
|
|
901
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
/**
|
|
905
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
906
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
907
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
908
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
909
|
+
*/
|
|
910
|
+
export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
911
|
+
const chain = resolveVendorChain(policy);
|
|
912
|
+
if (chain.length === 0) return resolveSmartproxyPoolSize(policy);
|
|
913
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
function vendorMaxPoolSize(vendor: ProxyVendorName): number {
|
|
917
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
922
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
923
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
924
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
925
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
926
|
+
*/
|
|
927
|
+
export function maxPolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
928
|
+
const chain = resolveVendorChain(policy);
|
|
929
|
+
if (chain.length === 0) return SMARTPROXY_MAX_POOL_SIZE;
|
|
930
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
937
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
938
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
939
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
940
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
941
|
+
*
|
|
942
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
943
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
944
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
945
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
946
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
947
|
+
* across the pool even if some framework default would allow it;
|
|
948
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
949
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed pool,
|
|
950
|
+
* so there is no possible endpoint crossover — they keep the retry budget.
|
|
951
|
+
*
|
|
952
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
953
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
954
|
+
* a pathological chain can never spin unbounded.
|
|
955
|
+
*/
|
|
956
|
+
/**
|
|
957
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
958
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
959
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
960
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
961
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
962
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
963
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
964
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
965
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
966
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
967
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
968
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
969
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed
|
|
970
|
+
* endpoint, so there is nothing to rotate or de-duplicate.
|
|
971
|
+
*/
|
|
972
|
+
export function policyRotatesTransportVendorChain(input: {
|
|
973
|
+
policy: ProviderProxyPolicy | undefined;
|
|
974
|
+
usesPolicyAllocator: boolean;
|
|
975
|
+
explicitRetry: boolean;
|
|
976
|
+
method: string;
|
|
977
|
+
}): boolean {
|
|
978
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
979
|
+
return false;
|
|
980
|
+
}
|
|
981
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
982
|
+
return false;
|
|
983
|
+
}
|
|
984
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
export function resolvePolicyTransportAttemptCap(input: {
|
|
988
|
+
policy: ProviderProxyPolicy | undefined;
|
|
989
|
+
usesPolicyAllocator: boolean;
|
|
990
|
+
retryAttempts: number;
|
|
991
|
+
explicitRetry: boolean;
|
|
992
|
+
method: string;
|
|
993
|
+
}): number {
|
|
994
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
995
|
+
if (
|
|
996
|
+
!policyRotatesTransportVendorChain({
|
|
997
|
+
policy: input.policy,
|
|
998
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
999
|
+
explicitRetry: input.explicitRetry,
|
|
1000
|
+
method: input.method,
|
|
1001
|
+
})
|
|
1002
|
+
) {
|
|
1003
|
+
return budget;
|
|
1004
|
+
}
|
|
1005
|
+
const span = Math.min(
|
|
1006
|
+
maxPolicyProxyPoolSpan(input.policy as ProviderProxyPolicy),
|
|
1007
|
+
resolvePolicyProxyPoolSpan(input.policy as ProviderProxyPolicy),
|
|
550
1008
|
);
|
|
1009
|
+
return Math.max(budget, span);
|
|
551
1010
|
}
|
|
552
1011
|
|
|
553
|
-
|
|
1012
|
+
/**
|
|
1013
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
1014
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
1015
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
1016
|
+
* ones. Deprecated custom/decodo policies have an empty registry chain and no
|
|
1017
|
+
* managed endpoint, so the transport loop has nothing to rotate or de-duplicate.
|
|
1018
|
+
*/
|
|
1019
|
+
export function policyResolvesRegistryVendorChain(
|
|
1020
|
+
policy: ProviderProxyPolicy | undefined,
|
|
1021
|
+
): boolean {
|
|
1022
|
+
return Boolean(policy) && resolveVendorChain(policy as ProviderProxyPolicy).length > 0;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
1026
|
+
export function vendorFromResolvedSource(
|
|
1027
|
+
source: ResolvedProxyConfig["source"],
|
|
1028
|
+
): ProxyVendorName | undefined {
|
|
1029
|
+
if (source === "nodemaven-gateway") return "nodemaven";
|
|
1030
|
+
if (source === "smartproxy-allocator") return "smartproxy";
|
|
1031
|
+
return undefined;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
function normalizeAttemptIndex(attempt: number | undefined): number {
|
|
1035
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt as number)) : 0;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
/**
|
|
1039
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
1040
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
1041
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
1042
|
+
*/
|
|
1043
|
+
export function mapFlatAttempt(
|
|
1044
|
+
flat: number,
|
|
1045
|
+
sizes: readonly number[],
|
|
1046
|
+
): { vendorIndex: number; poolIndex: number } {
|
|
1047
|
+
let cursor = flat;
|
|
1048
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
1049
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
1050
|
+
if (cursor < size) {
|
|
1051
|
+
return { vendorIndex, poolIndex: cursor };
|
|
1052
|
+
}
|
|
1053
|
+
cursor -= size;
|
|
1054
|
+
}
|
|
1055
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
function resolveSmartproxyCountry(
|
|
1059
|
+
policy: ProviderProxyPolicy,
|
|
1060
|
+
ambientDefaults = true,
|
|
1061
|
+
): string | undefined {
|
|
554
1062
|
return (
|
|
555
|
-
policy.geo?.country ??
|
|
1063
|
+
policy.geo?.country ??
|
|
1064
|
+
(ambientDefaults
|
|
1065
|
+
? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() || undefined
|
|
1066
|
+
: undefined)
|
|
556
1067
|
);
|
|
557
1068
|
}
|
|
558
1069
|
|
|
559
|
-
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy): number {
|
|
1070
|
+
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy, ambientDefaults = true): number {
|
|
560
1071
|
const configuredLifetime =
|
|
561
|
-
policy.session?.lifetimeMinutes ??
|
|
1072
|
+
policy.session?.lifetimeMinutes ??
|
|
1073
|
+
(ambientDefaults ? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30) : 30);
|
|
562
1074
|
return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
|
|
563
1075
|
}
|
|
564
1076
|
|
|
@@ -589,13 +1101,21 @@ function selectProxyPoolIndex(poolSize: number, attempt = 0): number {
|
|
|
589
1101
|
|
|
590
1102
|
function buildSmartproxyCacheKey(
|
|
591
1103
|
policy: ProviderProxyPolicy,
|
|
1104
|
+
appKey: string,
|
|
592
1105
|
affinityKey: string | undefined,
|
|
593
1106
|
lifetimeMinutes: number,
|
|
1107
|
+
protocol: ProxyProtocol,
|
|
1108
|
+
ambientDefaults = true,
|
|
594
1109
|
): string {
|
|
595
1110
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
596
1111
|
return JSON.stringify({
|
|
597
1112
|
provider: "smartproxy",
|
|
598
|
-
|
|
1113
|
+
credentialHash: createHash("sha256")
|
|
1114
|
+
.update("apifuse-smartproxy-credential:v1\0")
|
|
1115
|
+
.update(appKey)
|
|
1116
|
+
.digest("hex"),
|
|
1117
|
+
protocol,
|
|
1118
|
+
country: resolveSmartproxyCountry(policy, ambientDefaults),
|
|
599
1119
|
affinity: policy.session?.affinity ?? "request",
|
|
600
1120
|
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
601
1121
|
lifetimeMinutes,
|
|
@@ -608,16 +1128,34 @@ async function allocateSmartproxy(
|
|
|
608
1128
|
appKey: string,
|
|
609
1129
|
lifetimeMinutes: number,
|
|
610
1130
|
affinityKey: string | undefined,
|
|
1131
|
+
protocol: ProxyProtocol,
|
|
1132
|
+
ambientDefaults = true,
|
|
1133
|
+
sharedCache = true,
|
|
611
1134
|
): Promise<SmartproxyAllocationResult> {
|
|
612
|
-
const cacheKey = buildSmartproxyCacheKey(
|
|
1135
|
+
const cacheKey = buildSmartproxyCacheKey(
|
|
1136
|
+
policy,
|
|
1137
|
+
appKey,
|
|
1138
|
+
affinityKey,
|
|
1139
|
+
lifetimeMinutes,
|
|
1140
|
+
protocol,
|
|
1141
|
+
ambientDefaults,
|
|
1142
|
+
);
|
|
613
1143
|
const startedAt = Date.now();
|
|
614
1144
|
const now = startedAt;
|
|
615
1145
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
616
1146
|
const skipCached = invalidatedUntil > now;
|
|
617
|
-
const cached = proxyCache.get(cacheKey);
|
|
618
|
-
if (
|
|
1147
|
+
const cached = skipCached ? undefined : proxyCache.get(cacheKey, now);
|
|
1148
|
+
if (cached) {
|
|
619
1149
|
if (shouldSoftRefresh(cached, now)) {
|
|
620
|
-
void refreshSmartproxyPool(
|
|
1150
|
+
void refreshSmartproxyPool(
|
|
1151
|
+
cacheKey,
|
|
1152
|
+
policy,
|
|
1153
|
+
appKey,
|
|
1154
|
+
lifetimeMinutes,
|
|
1155
|
+
protocol,
|
|
1156
|
+
ambientDefaults,
|
|
1157
|
+
sharedCache,
|
|
1158
|
+
);
|
|
621
1159
|
return {
|
|
622
1160
|
pool: cached,
|
|
623
1161
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -631,7 +1169,7 @@ async function allocateSmartproxy(
|
|
|
631
1169
|
};
|
|
632
1170
|
}
|
|
633
1171
|
|
|
634
|
-
if (!skipCached) {
|
|
1172
|
+
if (!skipCached && sharedCache) {
|
|
635
1173
|
const redisResult = await readSmartproxyRedisPool(cacheKey, startedAt);
|
|
636
1174
|
if (redisResult) return redisResult;
|
|
637
1175
|
}
|
|
@@ -653,6 +1191,9 @@ async function allocateSmartproxy(
|
|
|
653
1191
|
appKey,
|
|
654
1192
|
lifetimeMinutes,
|
|
655
1193
|
startedAt,
|
|
1194
|
+
protocol,
|
|
1195
|
+
ambientDefaults,
|
|
1196
|
+
sharedCache,
|
|
656
1197
|
).finally(() => {
|
|
657
1198
|
proxyInflight.delete(cacheKey);
|
|
658
1199
|
});
|
|
@@ -679,7 +1220,7 @@ async function readSmartproxyRedisPool(
|
|
|
679
1220
|
}
|
|
680
1221
|
const now = Date.now();
|
|
681
1222
|
if (!isFresh(pool, now)) return null;
|
|
682
|
-
proxyCache.set(cacheKey, pool);
|
|
1223
|
+
proxyCache.set(cacheKey, pool, now);
|
|
683
1224
|
return {
|
|
684
1225
|
pool,
|
|
685
1226
|
telemetry: telemetryForPool(pool, "redis_hit", startedAt, { redisReadMs }),
|
|
@@ -691,11 +1232,24 @@ async function refreshSmartproxyPool(
|
|
|
691
1232
|
policy: ProviderProxyPolicy,
|
|
692
1233
|
appKey: string,
|
|
693
1234
|
lifetimeMinutes: number,
|
|
1235
|
+
protocol: ProxyProtocol,
|
|
1236
|
+
ambientDefaults: boolean,
|
|
1237
|
+
sharedCache: boolean,
|
|
694
1238
|
): Promise<void> {
|
|
695
1239
|
try {
|
|
696
|
-
await allocateSmartproxyShared(
|
|
697
|
-
|
|
698
|
-
|
|
1240
|
+
await allocateSmartproxyShared(
|
|
1241
|
+
cacheKey,
|
|
1242
|
+
policy,
|
|
1243
|
+
appKey,
|
|
1244
|
+
lifetimeMinutes,
|
|
1245
|
+
Date.now(),
|
|
1246
|
+
protocol,
|
|
1247
|
+
ambientDefaults,
|
|
1248
|
+
sharedCache,
|
|
1249
|
+
{
|
|
1250
|
+
background: true,
|
|
1251
|
+
},
|
|
1252
|
+
);
|
|
699
1253
|
} catch {
|
|
700
1254
|
// Soft refresh is opportunistic; current fresh pool remains usable.
|
|
701
1255
|
}
|
|
@@ -707,9 +1261,12 @@ async function allocateSmartproxyShared(
|
|
|
707
1261
|
appKey: string,
|
|
708
1262
|
lifetimeMinutes: number,
|
|
709
1263
|
startedAt: number,
|
|
1264
|
+
protocol: ProxyProtocol,
|
|
1265
|
+
ambientDefaults: boolean,
|
|
1266
|
+
sharedCache: boolean,
|
|
710
1267
|
options: { background?: boolean } = {},
|
|
711
1268
|
): Promise<SmartproxyAllocationResult> {
|
|
712
|
-
const redis = getProxyRedis();
|
|
1269
|
+
const redis = sharedCache ? getProxyRedis() : undefined;
|
|
713
1270
|
if (!redis || !(await ensureRedisReady(redis))) {
|
|
714
1271
|
return await allocateAndStoreSmartproxyPool(
|
|
715
1272
|
cacheKey,
|
|
@@ -717,7 +1274,7 @@ async function allocateSmartproxyShared(
|
|
|
717
1274
|
appKey,
|
|
718
1275
|
lifetimeMinutes,
|
|
719
1276
|
startedAt,
|
|
720
|
-
{ cacheStatus: "allocator" },
|
|
1277
|
+
{ cacheStatus: "allocator", protocol, ambientDefaults },
|
|
721
1278
|
);
|
|
722
1279
|
}
|
|
723
1280
|
|
|
@@ -742,6 +1299,8 @@ async function allocateSmartproxyShared(
|
|
|
742
1299
|
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
743
1300
|
redis,
|
|
744
1301
|
poolKey,
|
|
1302
|
+
protocol,
|
|
1303
|
+
ambientDefaults,
|
|
745
1304
|
},
|
|
746
1305
|
);
|
|
747
1306
|
} finally {
|
|
@@ -897,10 +1456,19 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
897
1456
|
cacheStatus: ProxyCacheStatus;
|
|
898
1457
|
redis?: ProxyRedisClient;
|
|
899
1458
|
poolKey?: string;
|
|
1459
|
+
protocol: ProxyProtocol;
|
|
1460
|
+
ambientDefaults: boolean;
|
|
900
1461
|
},
|
|
901
1462
|
): Promise<SmartproxyAllocationResult> {
|
|
902
1463
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
903
|
-
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1464
|
+
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1465
|
+
policy,
|
|
1466
|
+
appKey,
|
|
1467
|
+
lifetimeMinutes,
|
|
1468
|
+
poolSize,
|
|
1469
|
+
options.protocol,
|
|
1470
|
+
options.ambientDefaults,
|
|
1471
|
+
);
|
|
904
1472
|
const allocatorStartedAt = Date.now();
|
|
905
1473
|
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
906
1474
|
let allocation: SmartproxyAllocatorSuccess | undefined;
|
|
@@ -914,6 +1482,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
914
1482
|
allocatorUrl,
|
|
915
1483
|
attempt,
|
|
916
1484
|
allocatorDeadlineAt,
|
|
1485
|
+
options.protocol,
|
|
917
1486
|
);
|
|
918
1487
|
if (attemptResult.ok) {
|
|
919
1488
|
allocation = attemptResult;
|
|
@@ -969,13 +1538,13 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
969
1538
|
expiresAt: allocatedAt + ttlMs,
|
|
970
1539
|
diagnostics: {
|
|
971
1540
|
provider: "smartproxy",
|
|
972
|
-
country: resolveSmartproxyCountry(policy) ?? "default",
|
|
1541
|
+
country: resolveSmartproxyCountry(policy, options.ambientDefaults) ?? "default",
|
|
973
1542
|
lifetimeMinutes,
|
|
974
1543
|
affinity: policy.session?.affinity ?? "request",
|
|
975
1544
|
rawConnect: true,
|
|
976
1545
|
},
|
|
977
1546
|
};
|
|
978
|
-
proxyCache.set(cacheKey, result);
|
|
1547
|
+
proxyCache.set(cacheKey, result, allocatedAt);
|
|
979
1548
|
if (options.redis && options.poolKey) {
|
|
980
1549
|
const redis = options.redis;
|
|
981
1550
|
const poolKey = options.poolKey;
|
|
@@ -1034,6 +1603,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1034
1603
|
allocatorUrl: string,
|
|
1035
1604
|
attempt: number,
|
|
1036
1605
|
deadlineAt: number,
|
|
1606
|
+
protocol: ProxyProtocol,
|
|
1037
1607
|
): Promise<SmartproxyAllocatorAttemptResult> {
|
|
1038
1608
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
1039
1609
|
let response: Response;
|
|
@@ -1077,7 +1647,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1077
1647
|
};
|
|
1078
1648
|
}
|
|
1079
1649
|
|
|
1080
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1650
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
1081
1651
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
1082
1652
|
if (urls.length === 0) {
|
|
1083
1653
|
return {
|
|
@@ -1117,22 +1687,32 @@ function smartproxyAllocatorFailureMessage(
|
|
|
1117
1687
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
1118
1688
|
}
|
|
1119
1689
|
|
|
1690
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1691
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1692
|
+
// the only value exercised in production today.
|
|
1693
|
+
const SMARTPROXY_PROTOCOL_PARAM: Record<ProxyProtocol, string> = {
|
|
1694
|
+
http: "1",
|
|
1695
|
+
socks5: "2",
|
|
1696
|
+
};
|
|
1697
|
+
|
|
1120
1698
|
function buildSmartproxyAllocatorUrl(
|
|
1121
1699
|
policy: ProviderProxyPolicy,
|
|
1122
1700
|
appKey: string,
|
|
1123
1701
|
lifetimeMinutes: number,
|
|
1124
1702
|
poolSize: number,
|
|
1703
|
+
protocol: ProxyProtocol,
|
|
1704
|
+
ambientDefaults = true,
|
|
1125
1705
|
): string {
|
|
1126
1706
|
const params = new URLSearchParams({
|
|
1127
1707
|
app_key: appKey,
|
|
1128
1708
|
pt: "9",
|
|
1129
1709
|
num: String(poolSize),
|
|
1130
1710
|
life: String(lifetimeMinutes),
|
|
1131
|
-
protocol:
|
|
1711
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
1132
1712
|
format: "txt",
|
|
1133
1713
|
lb: "\\n",
|
|
1134
1714
|
});
|
|
1135
|
-
const country = resolveSmartproxyCountry(policy);
|
|
1715
|
+
const country = resolveSmartproxyCountry(policy, ambientDefaults);
|
|
1136
1716
|
if (country) {
|
|
1137
1717
|
params.set("cc", country);
|
|
1138
1718
|
}
|
|
@@ -1141,7 +1721,8 @@ function buildSmartproxyAllocatorUrl(
|
|
|
1141
1721
|
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
1142
1722
|
}
|
|
1143
1723
|
|
|
1144
|
-
function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
1724
|
+
function parseSmartproxyAllocatorProxies(body: string, protocol: ProxyProtocol): string[] {
|
|
1725
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
1145
1726
|
const trimmed = body.trim();
|
|
1146
1727
|
if (!trimmed) {
|
|
1147
1728
|
return [];
|
|
@@ -1162,7 +1743,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1162
1743
|
"port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
1163
1744
|
? item.port
|
|
1164
1745
|
: "";
|
|
1165
|
-
return ip && port ?
|
|
1746
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
1166
1747
|
})
|
|
1167
1748
|
.filter((url): url is string => url !== null);
|
|
1168
1749
|
}
|
|
@@ -1174,7 +1755,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1174
1755
|
.split(/\r?\n/)
|
|
1175
1756
|
.map((item) => item.trim())
|
|
1176
1757
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
1177
|
-
.map((line) =>
|
|
1758
|
+
.map((line) => `${scheme}://${line}`);
|
|
1178
1759
|
}
|
|
1179
1760
|
|
|
1180
1761
|
function classifySmartproxyAllocatorBody(
|
|
@@ -1207,13 +1788,29 @@ function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): s
|
|
|
1207
1788
|
if (!policy || policy.mode === "disabled") {
|
|
1208
1789
|
return undefined;
|
|
1209
1790
|
}
|
|
1210
|
-
if (
|
|
1791
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
1211
1792
|
return undefined;
|
|
1212
1793
|
}
|
|
1213
1794
|
|
|
1214
1795
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
1215
|
-
const
|
|
1216
|
-
|
|
1796
|
+
const appKey = (
|
|
1797
|
+
options.engineCredentials === undefined
|
|
1798
|
+
? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
|
|
1799
|
+
: options.engineCredentials[SMARTPROXY_APP_KEY_ENV]
|
|
1800
|
+
)?.trim();
|
|
1801
|
+
if (!appKey) return undefined;
|
|
1802
|
+
const cacheKey = buildSmartproxyCacheKey(
|
|
1803
|
+
policy,
|
|
1804
|
+
appKey,
|
|
1805
|
+
options.affinityKey,
|
|
1806
|
+
lifetimeMinutes,
|
|
1807
|
+
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1808
|
+
);
|
|
1809
|
+
const now = Date.now();
|
|
1810
|
+
for (const [key, invalidatedUntil] of invalidatedProxyKeys) {
|
|
1811
|
+
if (invalidatedUntil <= now) invalidatedProxyKeys.delete(key);
|
|
1812
|
+
}
|
|
1813
|
+
invalidatedProxyKeys.set(cacheKey, now + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1217
1814
|
proxyCache.delete(cacheKey);
|
|
1218
1815
|
proxyInflight.delete(cacheKey);
|
|
1219
1816
|
return cacheKey;
|
|
@@ -1262,17 +1859,13 @@ export async function loadApiFuseConfig(dir: string = process.cwd()): Promise<Ap
|
|
|
1262
1859
|
const tsPath = path.resolve(dir, "apifuse.config.ts");
|
|
1263
1860
|
if (existsSync(tsPath)) {
|
|
1264
1861
|
const config = await importConfig(tsPath);
|
|
1265
|
-
|
|
1266
|
-
syncProxyEnv(resolvedConfig);
|
|
1267
|
-
return resolvedConfig;
|
|
1862
|
+
return config ?? {};
|
|
1268
1863
|
}
|
|
1269
1864
|
|
|
1270
1865
|
const jsPath = path.resolve(dir, "apifuse.config.js");
|
|
1271
1866
|
if (existsSync(jsPath)) {
|
|
1272
1867
|
const config = await importConfig(jsPath);
|
|
1273
|
-
|
|
1274
|
-
syncProxyEnv(resolvedConfig);
|
|
1275
|
-
return resolvedConfig;
|
|
1868
|
+
return config ?? {};
|
|
1276
1869
|
}
|
|
1277
1870
|
|
|
1278
1871
|
return {};
|