@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +943 -51
- package/CHANGELOG.md +273 -1
- package/LICENSE +21 -0
- package/README.md +117 -29
- package/SUBMISSION.md +3 -2
- package/bin/apifuse-check.ts +293 -13
- package/bin/apifuse-dev.ts +42 -19
- package/bin/apifuse-migrate-deployment.ts +200 -0
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-migrate-shape.ts +202 -0
- package/bin/apifuse-pack-check.ts +19 -6
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +736 -38
- package/bin/apifuse-perf.ts +59 -23
- package/bin/apifuse-record.ts +923 -94
- package/bin/apifuse-submit-check.ts +2313 -281
- package/bin/apifuse-test.ts +3 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +30 -0
- package/dist/cli/create.d.ts +1 -1
- package/dist/cli/create.js +8 -2
- package/dist/cli/deployment-intent.d.ts +178 -0
- package/dist/cli/deployment-intent.js +277 -0
- package/dist/cli/local-connection.d.ts +111 -0
- package/dist/cli/local-connection.js +151 -0
- package/dist/cli/migrate-deployment-intent.d.ts +132 -0
- package/dist/cli/migrate-deployment-intent.js +1552 -0
- package/dist/cli/migrate-operation-declaration.d.ts +70 -0
- package/dist/cli/migrate-operation-declaration.js +2206 -0
- package/dist/cli/migrate-operation-shape.d.ts +44 -0
- package/dist/cli/migrate-operation-shape.js +113 -0
- package/dist/cli/migrate-provider-shape.d.ts +56 -0
- package/dist/cli/migrate-provider-shape.js +580 -0
- package/dist/cli/resolver-runtime.d.ts +21 -0
- package/dist/cli/resolver-runtime.js +31 -0
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +141 -23
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +41 -19
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/dist/cli/templates/provider/provider.json.tpl +6 -0
- package/dist/config/loader.d.ts +202 -16
- package/dist/config/loader.js +491 -141
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +12 -5
- package/dist/contract.js +24 -10
- package/dist/declaration-validation.d.ts +34 -0
- package/dist/declaration-validation.js +305 -0
- package/dist/define.d.ts +71 -29
- package/dist/define.js +807 -69
- package/dist/dev.d.ts +5 -0
- package/dist/dev.js +8 -1
- package/dist/engine.d.ts +129 -0
- package/dist/engine.js +209 -0
- package/dist/error-observability.d.ts +7 -0
- package/dist/error-observability.js +61 -0
- package/dist/error-resolution.d.ts +18 -0
- package/dist/error-resolution.js +243 -0
- package/dist/errors.d.ts +74 -0
- package/dist/errors.js +99 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +371 -0
- package/dist/handle-meta.d.ts +90 -0
- package/dist/handle-meta.js +73 -0
- package/dist/handle.d.ts +226 -0
- package/dist/handle.js +345 -0
- package/dist/health-scenario.d.ts +1901 -0
- package/dist/health-scenario.js +653 -0
- package/dist/i18n/catalog.js +21 -1
- package/dist/i18n/error-messages.d.ts +86 -0
- package/dist/i18n/error-messages.js +192 -0
- package/dist/i18n/index.d.ts +1 -0
- package/dist/i18n/index.js +1 -0
- package/dist/i18n/keys.d.ts +12 -0
- package/dist/i18n/keys.js +76 -5
- package/dist/i18n/operation-locale-namespace.d.ts +2 -0
- package/dist/i18n/operation-locale-namespace.js +7 -0
- package/dist/index.d.ts +40 -15
- package/dist/index.js +29 -11
- package/dist/lint.d.ts +43 -11
- package/dist/lint.js +2512 -177
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +18 -8
- package/dist/provider.js +11 -6
- package/dist/runtime/auth-flow.d.ts +6 -1
- package/dist/runtime/auth-flow.js +8 -0
- package/dist/runtime/bounded-expiring-map.d.ts +21 -0
- package/dist/runtime/bounded-expiring-map.js +64 -0
- package/dist/runtime/browser-telemetry-binding.d.ts +5 -0
- package/dist/runtime/browser-telemetry-binding.js +83 -0
- package/dist/runtime/browser-telemetry.d.ts +76 -0
- package/dist/runtime/browser-telemetry.js +165 -0
- package/dist/runtime/browser.d.ts +4 -0
- package/dist/runtime/browser.js +688 -83
- package/dist/runtime/cache-telemetry.d.ts +71 -0
- package/dist/runtime/cache-telemetry.js +177 -0
- package/dist/runtime/cache.d.ts +3 -0
- package/dist/runtime/cache.js +245 -23
- package/dist/runtime/capability-response.d.ts +3 -0
- package/dist/runtime/capability-response.js +56 -0
- package/dist/runtime/capability-telemetry.d.ts +46 -0
- package/dist/runtime/capability-telemetry.js +68 -0
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/diagnostic-env-values.d.ts +5 -0
- package/dist/runtime/diagnostic-env-values.js +57 -0
- package/dist/runtime/diagnostic-env.d.ts +14 -0
- package/dist/runtime/diagnostic-env.js +44 -0
- package/dist/runtime/diagnostic-redactor.d.ts +54 -0
- package/dist/runtime/diagnostic-redactor.js +571 -0
- package/dist/runtime/egress-lease.d.ts +65 -0
- package/dist/runtime/egress-lease.js +236 -0
- package/dist/runtime/engine-mode.d.ts +74 -0
- package/dist/runtime/engine-mode.js +152 -0
- package/dist/runtime/env.js +14 -1
- package/dist/runtime/executor.d.ts +3 -2
- package/dist/runtime/executor.js +23 -26
- package/dist/runtime/handle-wordlist.d.ts +43 -0
- package/dist/runtime/handle-wordlist.js +256 -0
- package/dist/runtime/handle.d.ts +71 -0
- package/dist/runtime/handle.js +908 -0
- package/dist/runtime/http-telemetry-binding.d.ts +10 -0
- package/dist/runtime/http-telemetry-binding.js +152 -0
- package/dist/runtime/http-telemetry-guard.d.ts +16 -0
- package/dist/runtime/http-telemetry-guard.js +202 -0
- package/dist/runtime/http-telemetry.d.ts +104 -0
- package/dist/runtime/http-telemetry.js +190 -0
- package/dist/runtime/http.d.ts +34 -0
- package/dist/runtime/http.js +680 -121
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +462 -19
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +105 -0
- package/dist/runtime/native-network.js +1433 -0
- package/dist/runtime/native-telemetry.d.ts +142 -0
- package/dist/runtime/native-telemetry.js +231 -0
- package/dist/runtime/ocr-telemetry.d.ts +82 -0
- package/dist/runtime/ocr-telemetry.js +192 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +468 -0
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +398 -16
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +147 -0
- package/dist/runtime/proxy-retry-policy.d.ts +4 -1
- package/dist/runtime/proxy-retry-policy.js +11 -2
- package/dist/runtime/proxy-telemetry.d.ts +133 -3
- package/dist/runtime/proxy-telemetry.js +218 -70
- package/dist/runtime/redirect-hop-headers.d.ts +19 -0
- package/dist/runtime/redirect-hop-headers.js +58 -0
- package/dist/runtime/redirects.d.ts +61 -0
- package/dist/runtime/redirects.js +149 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +75 -1
- package/dist/runtime/request-options.js +588 -0
- package/dist/runtime/request-telemetry.d.ts +70 -0
- package/dist/runtime/request-telemetry.js +229 -0
- package/dist/runtime/resolver-config.d.ts +13 -0
- package/dist/runtime/resolver-config.js +14 -0
- package/dist/runtime/resolver-public.d.ts +3 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-runtime-options.d.ts +14 -0
- package/dist/runtime/resolver-runtime-options.js +33 -0
- package/dist/runtime/resolver-shared.d.ts +24 -0
- package/dist/runtime/resolver-shared.js +69 -0
- package/dist/runtime/resolver-span-attributes.d.ts +34 -0
- package/dist/runtime/resolver-span-attributes.js +93 -0
- package/dist/runtime/resolver-telemetry.d.ts +135 -0
- package/dist/runtime/resolver-telemetry.js +204 -0
- package/dist/runtime/resolver-usage.d.ts +32 -0
- package/dist/runtime/resolver-usage.js +54 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +53 -0
- package/dist/runtime/resolver-vendors/bindings.js +45 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +24 -0
- package/dist/runtime/resolver-vendors/browser.js +423 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +544 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +23 -0
- package/dist/runtime/resolver-vendors/hosts.js +68 -0
- package/dist/runtime/resolver-vendors/hypersolutions.d.ts +27 -0
- package/dist/runtime/resolver-vendors/hypersolutions.js +369 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +425 -0
- package/dist/runtime/resolver-vendors/types.d.ts +125 -0
- package/dist/runtime/resolver-vendors/types.js +106 -0
- package/dist/runtime/resolver.d.ts +79 -0
- package/dist/runtime/resolver.js +1094 -0
- package/dist/runtime/state-telemetry.d.ts +79 -0
- package/dist/runtime/state-telemetry.js +194 -0
- package/dist/runtime/state.d.ts +10 -1
- package/dist/runtime/state.js +322 -82
- package/dist/runtime/stealth-akamai-sbsd.d.ts +54 -0
- package/dist/runtime/stealth-akamai-sbsd.js +142 -0
- package/dist/runtime/stealth-cookies.d.ts +24 -0
- package/dist/runtime/stealth-cookies.js +131 -0
- package/dist/runtime/stealth-owned-headers.d.ts +39 -0
- package/dist/runtime/stealth-owned-headers.js +57 -0
- package/dist/runtime/stealth-telemetry.d.ts +92 -0
- package/dist/runtime/stealth-telemetry.js +179 -0
- package/dist/runtime/stealth.d.ts +45 -6
- package/dist/runtime/stealth.js +1775 -320
- package/dist/runtime/stt-telemetry.d.ts +77 -0
- package/dist/runtime/stt-telemetry.js +176 -0
- package/dist/runtime/stt.js +18 -19
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +62 -0
- package/dist/runtime/trace.d.ts +17 -0
- package/dist/runtime/trace.js +176 -25
- package/dist/runtime-boundary-lint.d.ts +137 -0
- package/dist/runtime-boundary-lint.js +590 -0
- package/dist/schema.d.ts +20 -0
- package/dist/schema.js +72 -2
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/error-observability.d.ts +1 -0
- package/dist/server/error-observability.js +1 -0
- package/dist/server/error-stack-frames.d.ts +35 -0
- package/dist/server/error-stack-frames.js +219 -0
- package/dist/server/index.d.ts +21 -2
- package/dist/server/index.js +11 -2
- package/dist/server/runtime-diagnostics.d.ts +165 -0
- package/dist/server/runtime-diagnostics.js +360 -0
- package/dist/server/sdk-version.d.ts +3 -0
- package/dist/server/sdk-version.js +19 -0
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test-redaction.js +8 -33
- package/dist/server/self-test.d.ts +18 -3
- package/dist/server/self-test.js +139 -76
- package/dist/server/sensitive-values.d.ts +26 -0
- package/dist/server/sensitive-values.js +137 -0
- package/dist/server/serve-implementation.d.ts +285 -0
- package/dist/server/serve-implementation.js +3172 -0
- package/dist/server/serve.d.ts +1 -75
- package/dist/server/serve.js +1 -1170
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/dist/server/trace-output.d.ts +7 -0
- package/dist/server/trace-output.js +111 -0
- package/dist/server/types.d.ts +37 -5
- package/dist/server/types.js +34 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +237 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +35 -0
- package/dist/stateful/stateful-provider-runtime-context.js +61 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +15 -4
- package/dist/stealth/profiles.js +169 -211
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +509 -25
- package/dist/trace-sanitization.d.ts +6 -0
- package/dist/trace-sanitization.js +68 -0
- package/dist/types.d.ts +899 -210
- package/dist/types.js +1 -0
- package/dist/typescript-module.d.ts +39 -0
- package/dist/typescript-module.js +72 -0
- package/package.json +52 -6
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/commands.ts +37 -0
- package/src/cli/create.ts +8 -2
- package/src/cli/deployment-intent.ts +364 -0
- package/src/cli/local-connection.ts +221 -0
- package/src/cli/migrate-deployment-intent.ts +1908 -0
- package/src/cli/migrate-operation-declaration.ts +3026 -0
- package/src/cli/migrate-operation-shape.ts +184 -0
- package/src/cli/migrate-provider-shape.ts +778 -0
- package/src/cli/resolver-runtime.ts +72 -0
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +141 -23
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +41 -19
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/src/cli/templates/provider/provider.json.tpl +6 -0
- package/src/config/loader.ts +774 -181
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +12 -5
- package/src/contract.ts +24 -10
- package/src/declaration-validation.ts +376 -0
- package/src/define.ts +1105 -131
- package/src/dev.ts +13 -1
- package/src/engine.ts +390 -0
- package/src/error-observability.ts +64 -0
- package/src/error-resolution.ts +269 -0
- package/src/errors.ts +157 -0
- package/src/fixture-sanitization.ts +422 -0
- package/src/handle-meta.ts +143 -0
- package/src/handle.ts +694 -0
- package/src/health-scenario.ts +960 -0
- package/src/i18n/catalog.ts +23 -1
- package/src/i18n/error-messages.ts +255 -0
- package/src/i18n/index.ts +1 -0
- package/src/i18n/keys.ts +80 -6
- package/src/i18n/operation-locale-namespace.ts +10 -0
- package/src/index.ts +390 -23
- package/src/lint.ts +2975 -234
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +207 -27
- package/src/runtime/auth-flow.ts +15 -0
- package/src/runtime/bounded-expiring-map.ts +67 -0
- package/src/runtime/browser-telemetry-binding.ts +92 -0
- package/src/runtime/browser-telemetry.ts +289 -0
- package/src/runtime/browser.ts +882 -99
- package/src/runtime/cache-telemetry.ts +247 -0
- package/src/runtime/cache.ts +258 -22
- package/src/runtime/capability-response.ts +53 -0
- package/src/runtime/capability-telemetry.ts +101 -0
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/diagnostic-env-values.ts +56 -0
- package/src/runtime/diagnostic-env.ts +65 -0
- package/src/runtime/diagnostic-redactor.ts +668 -0
- package/src/runtime/egress-lease.ts +341 -0
- package/src/runtime/engine-mode.ts +233 -0
- package/src/runtime/env.ts +15 -1
- package/src/runtime/executor.ts +39 -32
- package/src/runtime/handle-wordlist.ts +266 -0
- package/src/runtime/handle.ts +1180 -0
- package/src/runtime/http-telemetry-binding.ts +162 -0
- package/src/runtime/http-telemetry-guard.ts +208 -0
- package/src/runtime/http-telemetry.ts +292 -0
- package/src/runtime/http.ts +896 -150
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/instrumentation.ts +628 -28
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1853 -0
- package/src/runtime/native-telemetry.ts +366 -0
- package/src/runtime/ocr-telemetry.ts +308 -0
- package/src/runtime/ocr.ts +564 -0
- package/src/runtime/otlp.ts +468 -21
- package/src/runtime/provider.ts +15 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +222 -0
- package/src/runtime/proxy-retry-policy.ts +12 -1
- package/src/runtime/proxy-telemetry.ts +358 -103
- package/src/runtime/redirect-hop-headers.ts +71 -0
- package/src/runtime/redirects.ts +215 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +728 -1
- package/src/runtime/request-telemetry.ts +375 -0
- package/src/runtime/resolver-config.ts +18 -0
- package/src/runtime/resolver-public.ts +44 -0
- package/src/runtime/resolver-runtime-options.ts +50 -0
- package/src/runtime/resolver-shared.ts +113 -0
- package/src/runtime/resolver-span-attributes.ts +139 -0
- package/src/runtime/resolver-telemetry.ts +362 -0
- package/src/runtime/resolver-usage.ts +99 -0
- package/src/runtime/resolver-vendors/bindings.ts +61 -0
- package/src/runtime/resolver-vendors/browser.ts +591 -0
- package/src/runtime/resolver-vendors/capsolver.ts +727 -0
- package/src/runtime/resolver-vendors/hosts.ts +81 -0
- package/src/runtime/resolver-vendors/hypersolutions.ts +480 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +555 -0
- package/src/runtime/resolver-vendors/types.ts +261 -0
- package/src/runtime/resolver.ts +1543 -0
- package/src/runtime/state-telemetry.ts +298 -0
- package/src/runtime/state.ts +435 -89
- package/src/runtime/stealth-akamai-sbsd.ts +207 -0
- package/src/runtime/stealth-cookies.ts +155 -0
- package/src/runtime/stealth-owned-headers.ts +67 -0
- package/src/runtime/stealth-telemetry.ts +309 -0
- package/src/runtime/stealth.ts +2275 -372
- package/src/runtime/stt-telemetry.ts +282 -0
- package/src/runtime/stt.ts +96 -91
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +78 -0
- package/src/runtime/trace.ts +257 -35
- package/src/runtime-boundary-lint.ts +707 -0
- package/src/schema.ts +80 -2
- package/src/serve.ts +6 -1
- package/src/server/error-observability.ts +1 -0
- package/src/server/error-stack-frames.ts +214 -0
- package/src/server/index.ts +184 -2
- package/src/server/runtime-diagnostics.ts +522 -0
- package/src/server/sdk-version.ts +22 -0
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test-redaction.ts +10 -40
- package/src/server/self-test.ts +205 -89
- package/src/server/sensitive-values.ts +211 -0
- package/src/server/serve-implementation.ts +4596 -0
- package/src/server/serve.ts +1 -1685
- package/src/server/state-backend-report.ts +185 -0
- package/src/server/trace-output.ts +189 -0
- package/src/server/types.ts +34 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +310 -0
- package/src/stateful/stateful-provider-runtime-context.ts +96 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +212 -231
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +685 -19
- package/src/trace-sanitization.ts +110 -0
- package/src/types.ts +1014 -273
- package/src/typescript-module.ts +100 -0
- package/dist/choice-token.d.ts +0 -24
- package/dist/choice-token.js +0 -74
- package/dist/runtime/choice.d.ts +0 -15
- package/dist/runtime/choice.js +0 -413
- package/src/choice-token.ts +0 -165
- package/src/runtime/choice.ts +0 -686
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,191 +1,494 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, resolveProxyConfigAsync,
|
|
2
|
+
import { Cookie } from "tough-cookie";
|
|
3
|
+
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
4
4
|
import { SDKError, TransportError } from "../errors.js";
|
|
5
|
-
import { getStealthProfile } from "../stealth/profiles.js";
|
|
5
|
+
import { DEFAULT_STEALTH_BROWSER, DEFAULT_STEALTH_OS, getStealthProfile, resolverClientProfileFamily, resolveStealthProfileSelection, } from "../stealth/profiles.js";
|
|
6
|
+
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
7
|
+
import { registerDiagnosticValue } from "./diagnostic-env.js";
|
|
8
|
+
import { assertCallerHeadersSupported } from "./stealth-owned-headers.js";
|
|
9
|
+
import { ENGINE_CEREMONY_EGRESS_LEASE, } from "./egress-lease.js";
|
|
6
10
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
|
-
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
|
-
import {
|
|
9
|
-
|
|
11
|
+
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, isProxyTransportRetryMethod, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
12
|
+
import { redirectHopHeaders } from "./redirect-hop-headers.js";
|
|
13
|
+
import { evaluateRedirectHop, invalidRedirectPolicy, isRedirectStatus, nextRedirectMethod, nextSameOriginRedirectHop, normalizeRedirectPolicy, resolveRedirectUrl, } from "./redirects.js";
|
|
14
|
+
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
15
|
+
import { akamaiSbsdChallengeKey, detectAkamaiSbsdChallenge, } from "./stealth-akamai-sbsd.js";
|
|
16
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
17
|
+
export const DEFAULT_STEALTH_PROFILE = Object.freeze({
|
|
18
|
+
browser: DEFAULT_STEALTH_BROWSER,
|
|
19
|
+
os: DEFAULT_STEALTH_OS,
|
|
20
|
+
});
|
|
10
21
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
11
|
-
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
12
22
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
23
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
14
24
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
15
25
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
16
26
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
17
27
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE];
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"chrome-144",
|
|
26
|
-
"chrome-146-psk",
|
|
27
|
-
"chrome-131-psk",
|
|
28
|
-
"chrome-130-psk",
|
|
29
|
-
"edge-131",
|
|
28
|
+
// Failures that mean the bound ceremony endpoint did not carry the request to the origin.
|
|
29
|
+
// Proxy-level refusals are covered by isProxyPoolRefreshableError; PROXY_AUTH_IP_DENIED is
|
|
30
|
+
// deliberately absent because another endpoint of the same vendor is denied just the same.
|
|
31
|
+
const BOUND_EGRESS_TRANSPORT_FAILURE_CODES = new Set([
|
|
32
|
+
PROXY_CONNECT_FAILURE_CODE,
|
|
33
|
+
"transport_network_error",
|
|
34
|
+
"transport_timeout",
|
|
30
35
|
]);
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
125: "chrome125",
|
|
40
|
-
131: "chrome131",
|
|
41
|
-
136: "chrome136",
|
|
42
|
-
142: "chrome142",
|
|
43
|
-
};
|
|
44
|
-
const FIREFOX_IMPIT_BY_MAJOR = {
|
|
45
|
-
128: "firefox128",
|
|
46
|
-
133: "firefox133",
|
|
47
|
-
135: "firefox135",
|
|
48
|
-
144: "firefox144",
|
|
36
|
+
function isBoundEgressTransportFailure(error) {
|
|
37
|
+
return (isProxyPoolRefreshableError(error) ||
|
|
38
|
+
(error.code !== undefined && BOUND_EGRESS_TRANSPORT_FAILURE_CODES.has(error.code)));
|
|
39
|
+
}
|
|
40
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
41
|
+
const STEALTH_REDIRECT_POLICY_TRANSPORT = {
|
|
42
|
+
label: "ctx.stealth",
|
|
43
|
+
maxHops: MAX_STEALTH_REDIRECT_HOPS,
|
|
49
44
|
};
|
|
50
45
|
function isRecord(value) {
|
|
51
46
|
return typeof value === "object" && value !== null;
|
|
52
47
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
getAll() {
|
|
79
|
-
return { ...this.cookies };
|
|
80
|
-
}
|
|
81
|
-
has(name) {
|
|
82
|
-
return Object.hasOwn(this.cookies, name);
|
|
83
|
-
}
|
|
84
|
-
toString() {
|
|
85
|
-
return Object.entries(this.cookies)
|
|
86
|
-
.map(([name, value]) => `${name}=${value}`)
|
|
87
|
-
.join("; ");
|
|
88
|
-
}
|
|
89
|
-
toHeader() {
|
|
90
|
-
return this.toString();
|
|
48
|
+
function sensitiveQueryParamNames(url) {
|
|
49
|
+
const queryStart = url.indexOf("?");
|
|
50
|
+
if (queryStart === -1)
|
|
51
|
+
return [];
|
|
52
|
+
const fragmentStart = url.indexOf("#", queryStart);
|
|
53
|
+
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
54
|
+
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
55
|
+
}
|
|
56
|
+
function assertAkamaiSbsdClientProfile(declared, session) {
|
|
57
|
+
if (declared === undefined || resolverClientProfileFamily(declared) === session.browser)
|
|
58
|
+
return;
|
|
59
|
+
throw new SDKError(`Resolver client profile "${declared}" does not match the initiating stealth session browser "${session.browser}"`, {
|
|
60
|
+
code: "RESOLVER_CLIENT_PROFILE_MISMATCH",
|
|
61
|
+
fix: "Make resolver.clientProfile name the provider stealth browser family (chrome, firefox, or safari).",
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
let wreqModulePromise;
|
|
65
|
+
function getWreqModule() {
|
|
66
|
+
if (!wreqModulePromise) {
|
|
67
|
+
wreqModulePromise = import("wreq-js").catch((error) => {
|
|
68
|
+
throw new SDKError(`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`, {
|
|
69
|
+
code: "stealth_transport_unavailable",
|
|
70
|
+
cause: error instanceof Error ? error : undefined,
|
|
71
|
+
});
|
|
72
|
+
});
|
|
91
73
|
}
|
|
92
|
-
|
|
93
|
-
|
|
74
|
+
return wreqModulePromise;
|
|
75
|
+
}
|
|
76
|
+
function parseProfileIdentifier(identifier) {
|
|
77
|
+
const match = /^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(identifier.toLowerCase());
|
|
78
|
+
if (!match?.[1] || !match[2])
|
|
79
|
+
return null;
|
|
80
|
+
return {
|
|
81
|
+
family: match[1],
|
|
82
|
+
version: match[2].split(/[._]/).map(Number),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function compareVersionDistance(target, left, right) {
|
|
86
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
87
|
+
for (let index = 0; index < width; index += 1) {
|
|
88
|
+
const targetPart = target[index] ?? 0;
|
|
89
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
90
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
91
|
+
if (leftDistance !== rightDistance)
|
|
92
|
+
return leftDistance - rightDistance;
|
|
94
93
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
94
|
+
return 0;
|
|
95
|
+
}
|
|
96
|
+
function closestWreqProfile(identifier, wreqProfiles) {
|
|
97
|
+
const requested = parseProfileIdentifier(identifier);
|
|
98
|
+
if (!requested)
|
|
99
|
+
return undefined;
|
|
100
|
+
let closest;
|
|
101
|
+
for (const candidateName of wreqProfiles) {
|
|
102
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
103
|
+
if (!candidate || candidate.family !== requested.family)
|
|
104
|
+
continue;
|
|
105
|
+
if (!closest ||
|
|
106
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0) {
|
|
107
|
+
closest = { name: candidateName, version: candidate.version };
|
|
100
108
|
}
|
|
101
109
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
110
|
+
return closest?.name;
|
|
111
|
+
}
|
|
112
|
+
export function resolveWreqProfile(selection, wreqProfiles) {
|
|
113
|
+
const profile = getStealthProfile(selection);
|
|
114
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
115
|
+
const os = profile.os;
|
|
116
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
117
|
+
if (!browser) {
|
|
118
|
+
throw new SDKError(`Stealth profile ${profile.browser}/${profile.os} cannot be mapped to a wreq-js browser profile.`);
|
|
106
119
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
return
|
|
120
|
+
return { browser, os };
|
|
121
|
+
}
|
|
122
|
+
function resolveUrl(baseUrl, url) {
|
|
123
|
+
return new URL(url, baseUrl).toString();
|
|
124
|
+
}
|
|
125
|
+
function resolveRequestUrl(baseUrl, url) {
|
|
126
|
+
try {
|
|
127
|
+
return resolveUrl(baseUrl, url);
|
|
115
128
|
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
// Both inputs are caller-owned, so an unparsable URL is a programming error, never a
|
|
131
|
+
// transport condition: not retryable, not the upstream's fault. Classify it here rather
|
|
132
|
+
// than in normalizeStealthTransportError, which also sees ERR_INVALID_URL from malformed
|
|
133
|
+
// upstream redirect targets. The fixed message keeps the URL (and any inline query
|
|
134
|
+
// values) out of the diagnostic surface; the runtime TypeError stays in `cause`.
|
|
135
|
+
throw new TransportError("Invalid request URL", {
|
|
136
|
+
code: "transport_invalid_url",
|
|
137
|
+
status: 0,
|
|
138
|
+
category: "provider_error",
|
|
139
|
+
retryable: false,
|
|
140
|
+
...(error instanceof Error ? { cause: error } : {}),
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function headerEntriesFromHeaders(headers) {
|
|
145
|
+
return Array.from(headers.entries());
|
|
146
|
+
}
|
|
147
|
+
function normalizeHeaders(headers) {
|
|
148
|
+
return Object.fromEntries(normalizedHeaderEntries(headers));
|
|
116
149
|
}
|
|
117
|
-
function
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
150
|
+
function trimOuterHttpWhitespace(value) {
|
|
151
|
+
return value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, "");
|
|
152
|
+
}
|
|
153
|
+
function normalizedHeaderEntries(headers) {
|
|
154
|
+
const entries = [];
|
|
155
|
+
const indices = new Map();
|
|
156
|
+
for (const [originalName, originalValue] of Object.entries(headers)) {
|
|
157
|
+
if (originalValue === undefined)
|
|
158
|
+
continue;
|
|
159
|
+
const name = originalName.toLowerCase();
|
|
160
|
+
const value = (Array.isArray(originalValue) ? originalValue : [originalValue])
|
|
161
|
+
.map(trimOuterHttpWhitespace)
|
|
162
|
+
.join(", ");
|
|
163
|
+
const existingIndex = indices.get(name);
|
|
164
|
+
if (existingIndex === undefined) {
|
|
165
|
+
indices.set(name, entries.length);
|
|
166
|
+
entries.push([name, value]);
|
|
167
|
+
continue;
|
|
126
168
|
}
|
|
169
|
+
const existing = entries[existingIndex];
|
|
170
|
+
if (existing)
|
|
171
|
+
existing[1] = `${existing[1]}, ${value}`;
|
|
127
172
|
}
|
|
128
|
-
return
|
|
173
|
+
return entries;
|
|
129
174
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
175
|
+
// Non-pseudo-header order of real Chrome 149. The fixture captures
|
|
176
|
+
// (chrome-ground-truth-capture.json, chrome-extended-capture.json,
|
|
177
|
+
// h1-casing-capture.json) were taken through Playwright's `locale` option, which
|
|
178
|
+
// installs Accept-Language via DevTools next to User-Agent; real Chrome only
|
|
179
|
+
// receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
|
|
180
|
+
// Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
|
|
181
|
+
// chrome-form-post-capture.json was taken without that option and needs no
|
|
182
|
+
// adjustment.
|
|
183
|
+
const CHROME_HEADER_ORDERS = {
|
|
184
|
+
navigation: [
|
|
185
|
+
"sec-ch-ua",
|
|
186
|
+
"sec-ch-ua-mobile",
|
|
187
|
+
"sec-ch-ua-platform",
|
|
188
|
+
"upgrade-insecure-requests",
|
|
189
|
+
"user-agent",
|
|
190
|
+
"accept",
|
|
191
|
+
"sec-fetch-site",
|
|
192
|
+
"sec-fetch-mode",
|
|
193
|
+
"sec-fetch-user",
|
|
194
|
+
"sec-fetch-dest",
|
|
195
|
+
"accept-encoding",
|
|
196
|
+
"accept-language",
|
|
197
|
+
"cookie",
|
|
198
|
+
"priority",
|
|
199
|
+
],
|
|
200
|
+
// xhr has no fixed table: it is emulated by chrome149HeaderOrder.
|
|
201
|
+
post: [
|
|
202
|
+
"content-length",
|
|
203
|
+
"sec-ch-ua-platform",
|
|
204
|
+
"user-agent",
|
|
205
|
+
"sec-ch-ua",
|
|
206
|
+
"content-type",
|
|
207
|
+
"sec-ch-ua-mobile",
|
|
208
|
+
"accept",
|
|
209
|
+
"origin",
|
|
210
|
+
"sec-fetch-site",
|
|
211
|
+
"sec-fetch-mode",
|
|
212
|
+
"sec-fetch-dest",
|
|
213
|
+
"referer",
|
|
214
|
+
"accept-encoding",
|
|
215
|
+
"accept-language",
|
|
216
|
+
"cookie",
|
|
217
|
+
"priority",
|
|
218
|
+
],
|
|
219
|
+
"form-post": [
|
|
220
|
+
"content-length",
|
|
221
|
+
"cache-control",
|
|
222
|
+
"sec-ch-ua",
|
|
223
|
+
"sec-ch-ua-mobile",
|
|
224
|
+
"sec-ch-ua-platform",
|
|
225
|
+
"upgrade-insecure-requests",
|
|
226
|
+
"content-type",
|
|
227
|
+
"user-agent",
|
|
228
|
+
"origin",
|
|
229
|
+
"accept",
|
|
230
|
+
"sec-fetch-site",
|
|
231
|
+
"sec-fetch-mode",
|
|
232
|
+
"sec-fetch-user",
|
|
233
|
+
"sec-fetch-dest",
|
|
234
|
+
"referer",
|
|
235
|
+
"accept-encoding",
|
|
236
|
+
"accept-language",
|
|
237
|
+
"cookie",
|
|
238
|
+
"priority",
|
|
239
|
+
],
|
|
240
|
+
};
|
|
241
|
+
const CHROME_H1_HEADER_ORDERS = {
|
|
242
|
+
navigation: [
|
|
243
|
+
"host",
|
|
244
|
+
"connection",
|
|
245
|
+
"sec-ch-ua",
|
|
246
|
+
"sec-ch-ua-mobile",
|
|
247
|
+
"sec-ch-ua-platform",
|
|
248
|
+
"upgrade-insecure-requests",
|
|
249
|
+
"user-agent",
|
|
250
|
+
"accept",
|
|
251
|
+
"sec-fetch-site",
|
|
252
|
+
"sec-fetch-mode",
|
|
253
|
+
"sec-fetch-user",
|
|
254
|
+
"sec-fetch-dest",
|
|
255
|
+
"accept-encoding",
|
|
256
|
+
"accept-language",
|
|
257
|
+
"cookie",
|
|
258
|
+
],
|
|
259
|
+
xhr: [
|
|
260
|
+
"host",
|
|
261
|
+
"connection",
|
|
262
|
+
"sec-ch-ua-platform",
|
|
263
|
+
"cache-control",
|
|
264
|
+
"x-requested-with",
|
|
265
|
+
"user-agent",
|
|
266
|
+
"sec-ch-ua",
|
|
267
|
+
"sec-ch-ua-mobile",
|
|
268
|
+
"accept",
|
|
269
|
+
"sec-fetch-site",
|
|
270
|
+
"sec-fetch-mode",
|
|
271
|
+
"sec-fetch-dest",
|
|
272
|
+
"referer",
|
|
273
|
+
"accept-encoding",
|
|
274
|
+
"accept-language",
|
|
275
|
+
"cookie",
|
|
276
|
+
],
|
|
277
|
+
post: [
|
|
278
|
+
"host",
|
|
279
|
+
"connection",
|
|
280
|
+
"content-length",
|
|
281
|
+
"sec-ch-ua-platform",
|
|
282
|
+
"user-agent",
|
|
283
|
+
"sec-ch-ua",
|
|
284
|
+
"content-type",
|
|
285
|
+
"sec-ch-ua-mobile",
|
|
286
|
+
"accept",
|
|
287
|
+
"origin",
|
|
288
|
+
"sec-fetch-site",
|
|
289
|
+
"sec-fetch-mode",
|
|
290
|
+
"sec-fetch-dest",
|
|
291
|
+
"referer",
|
|
292
|
+
"accept-encoding",
|
|
293
|
+
"accept-language",
|
|
294
|
+
"cookie",
|
|
295
|
+
],
|
|
296
|
+
"form-post": [
|
|
297
|
+
"host",
|
|
298
|
+
"connection",
|
|
299
|
+
"content-length",
|
|
300
|
+
"cache-control",
|
|
301
|
+
"sec-ch-ua",
|
|
302
|
+
"sec-ch-ua-mobile",
|
|
303
|
+
"sec-ch-ua-platform",
|
|
304
|
+
"upgrade-insecure-requests",
|
|
305
|
+
"content-type",
|
|
306
|
+
"user-agent",
|
|
307
|
+
"origin",
|
|
308
|
+
"accept",
|
|
309
|
+
"sec-fetch-site",
|
|
310
|
+
"sec-fetch-mode",
|
|
311
|
+
"sec-fetch-user",
|
|
312
|
+
"sec-fetch-dest",
|
|
313
|
+
"referer",
|
|
314
|
+
"accept-encoding",
|
|
315
|
+
"accept-language",
|
|
316
|
+
"cookie",
|
|
317
|
+
],
|
|
318
|
+
};
|
|
319
|
+
const CHROME_H1_HEADER_NAMES = {
|
|
320
|
+
host: "Host",
|
|
321
|
+
connection: "Connection",
|
|
322
|
+
"upgrade-insecure-requests": "Upgrade-Insecure-Requests",
|
|
323
|
+
"user-agent": "User-Agent",
|
|
324
|
+
"accept-language": "Accept-Language",
|
|
325
|
+
accept: "Accept",
|
|
326
|
+
"sec-fetch-site": "Sec-Fetch-Site",
|
|
327
|
+
"sec-fetch-mode": "Sec-Fetch-Mode",
|
|
328
|
+
"sec-fetch-user": "Sec-Fetch-User",
|
|
329
|
+
"sec-fetch-dest": "Sec-Fetch-Dest",
|
|
330
|
+
"accept-encoding": "Accept-Encoding",
|
|
331
|
+
cookie: "Cookie",
|
|
332
|
+
"cache-control": "Cache-Control",
|
|
333
|
+
"x-requested-with": "X-Requested-With",
|
|
334
|
+
referer: "Referer",
|
|
335
|
+
"content-length": "Content-Length",
|
|
336
|
+
"content-type": "Content-Type",
|
|
337
|
+
origin: "Origin",
|
|
338
|
+
range: "Range",
|
|
339
|
+
};
|
|
340
|
+
/**
|
|
341
|
+
* What each request class sends besides its header order, one row per public
|
|
342
|
+
* `stealth.requestClass`:
|
|
343
|
+
*
|
|
344
|
+
* - `fetchMetadata: "navigate"` is a document navigation: `Sec-Fetch-Mode:
|
|
345
|
+
* navigate`, `Sec-Fetch-Dest: document`, `Sec-Fetch-User: ?1` unless
|
|
346
|
+
* `userActivation` is false, `Upgrade-Insecure-Requests: 1`, and the
|
|
347
|
+
* profile's `Accept` and `Priority`. `"cors"` is a script fetch: `cors` /
|
|
348
|
+
* `empty`, `Accept: *\/*`, `Priority: u=1, i`.
|
|
349
|
+
* - `bodyful` sends `Content-Length`.
|
|
350
|
+
* - `validateCache` sends `Cache-Control: max-age=0`, which Chrome adds to a
|
|
351
|
+
* form submission (chrome-form-post-capture.json).
|
|
352
|
+
*/
|
|
353
|
+
const CHROME_REQUEST_CLASSES = {
|
|
354
|
+
navigation: { fetchMetadata: "navigate", bodyful: false, validateCache: false },
|
|
355
|
+
"form-post": { fetchMetadata: "navigate", bodyful: true, validateCache: true },
|
|
356
|
+
xhr: { fetchMetadata: "cors", bodyful: false, validateCache: false },
|
|
357
|
+
post: { fetchMetadata: "cors", bodyful: true, validateCache: false },
|
|
358
|
+
};
|
|
359
|
+
function chromeHeaderOrder(requestClass, isHttp1, caller) {
|
|
360
|
+
if (isHttp1)
|
|
361
|
+
return CHROME_H1_HEADER_ORDERS[requestClass];
|
|
362
|
+
if (requestClass !== "xhr")
|
|
363
|
+
return CHROME_HEADER_ORDERS[requestClass];
|
|
364
|
+
// Cookie and Referer are forbidden Fetch headers, so they never enter the page
|
|
365
|
+
// Fetch Headers map the emulator models. Range does occupy a map bucket
|
|
366
|
+
// (m1-capture.json range_*), but HttpCache::Transaction removes it from the
|
|
367
|
+
// request headers and PartialData re-adds it at the tail.
|
|
368
|
+
const order = chrome149HeaderOrder([...caller.keys()].filter((name) => name !== "cookie" && name !== "referer"));
|
|
369
|
+
if (caller.has("range"))
|
|
370
|
+
order.splice(order.indexOf("range"), 1);
|
|
371
|
+
// //net appends Accept-Encoding and, unless the caller supplied one,
|
|
372
|
+
// Accept-Language (URLRequestHttpJob::AddExtraHeaders), then writes the Cookie
|
|
373
|
+
// header (SetCookieHeaderAndStart); the HTTP cache re-adds a caller Range after
|
|
374
|
+
// that, and the HTTP/2 Priority header stays last.
|
|
375
|
+
let insertAt = order.indexOf("accept-encoding") + 1;
|
|
376
|
+
if (order[insertAt] === "accept-language")
|
|
377
|
+
insertAt += 1;
|
|
378
|
+
for (const name of ["cookie", "range"]) {
|
|
379
|
+
if (!caller.has(name))
|
|
380
|
+
continue;
|
|
381
|
+
order.splice(insertAt, 0, name);
|
|
382
|
+
insertAt += 1;
|
|
133
383
|
}
|
|
134
|
-
|
|
384
|
+
return order;
|
|
385
|
+
}
|
|
386
|
+
function normalizedCallerHeaderEntriesFromRecord(headers) {
|
|
387
|
+
// The record has been through normalizeHeaders (lowercase, merged names); the
|
|
388
|
+
// lowercase here only guards later insertions such as the cookie jar's header.
|
|
389
|
+
assertCallerHeadersSupported(headers);
|
|
390
|
+
return Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]);
|
|
391
|
+
}
|
|
392
|
+
function chromeRequestClass(method, requestedClass) {
|
|
393
|
+
if (requestedClass)
|
|
394
|
+
return requestedClass;
|
|
395
|
+
if (method === "POST")
|
|
396
|
+
return "post";
|
|
397
|
+
return "navigation";
|
|
398
|
+
}
|
|
399
|
+
function secFetchSite(requestUrl, referer) {
|
|
400
|
+
if (!referer)
|
|
401
|
+
return "none";
|
|
135
402
|
try {
|
|
136
|
-
|
|
403
|
+
return new URL(referer).origin === new URL(requestUrl).origin ? "same-origin" : "cross-site";
|
|
137
404
|
}
|
|
138
405
|
catch {
|
|
139
|
-
|
|
140
|
-
// profile strings still run with the transport default instead of failing
|
|
141
|
-
// before the request starts. Removed built-in profile aliases above remain
|
|
142
|
-
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
143
|
-
return "chrome142";
|
|
406
|
+
return "cross-site";
|
|
144
407
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
408
|
+
}
|
|
409
|
+
function requiredEmulationHeader(headers, name) {
|
|
410
|
+
const value = headers.get(name);
|
|
411
|
+
if (value !== undefined)
|
|
412
|
+
return value;
|
|
413
|
+
throw new SDKError(`wreq-js Chrome emulation exposes no ${name} header.`, {
|
|
414
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
function buildChromeHeaderTuples(options) {
|
|
418
|
+
const callerEntries = normalizedCallerHeaderEntriesFromRecord(options.headers);
|
|
419
|
+
const caller = new Map(callerEntries);
|
|
420
|
+
const emulation = new Map(Array.from(options.emulationHeaders, ([name, value]) => [name.toLowerCase(), value]));
|
|
421
|
+
const requestClass = chromeRequestClass(options.method, options.requestClass);
|
|
422
|
+
const shape = CHROME_REQUEST_CLASSES[requestClass];
|
|
423
|
+
const referer = caller.get("referer");
|
|
424
|
+
const fetchSite = secFetchSite(options.requestUrl, referer);
|
|
425
|
+
const isNavigation = shape.fetchMetadata === "navigate";
|
|
426
|
+
const values = new Map([
|
|
427
|
+
["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
|
|
428
|
+
["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
|
|
429
|
+
["sec-ch-ua-platform", requiredEmulationHeader(emulation, "sec-ch-ua-platform")],
|
|
430
|
+
["user-agent", requiredEmulationHeader(emulation, "user-agent")],
|
|
431
|
+
[
|
|
432
|
+
"accept-language",
|
|
433
|
+
caller.get("accept-language") ??
|
|
434
|
+
options.acceptLanguage ??
|
|
435
|
+
requiredEmulationHeader(emulation, "accept-language"),
|
|
436
|
+
],
|
|
437
|
+
["accept", isNavigation ? requiredEmulationHeader(emulation, "accept") : "*/*"],
|
|
438
|
+
[
|
|
439
|
+
"accept-encoding",
|
|
440
|
+
caller.has("range") ? "identity" : requiredEmulationHeader(emulation, "accept-encoding"),
|
|
441
|
+
],
|
|
442
|
+
["priority", isNavigation ? requiredEmulationHeader(emulation, "priority") : "u=1, i"],
|
|
443
|
+
["sec-fetch-site", fetchSite],
|
|
444
|
+
["sec-fetch-mode", isNavigation ? "navigate" : "cors"],
|
|
445
|
+
["sec-fetch-dest", isNavigation ? "document" : "empty"],
|
|
446
|
+
]);
|
|
447
|
+
if (isNavigation) {
|
|
448
|
+
values.set("upgrade-insecure-requests", "1");
|
|
449
|
+
if (options.userActivation !== false)
|
|
450
|
+
values.set("sec-fetch-user", "?1");
|
|
149
451
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
452
|
+
if (shape.validateCache)
|
|
453
|
+
values.set("cache-control", "max-age=0");
|
|
454
|
+
for (const [name, value] of callerEntries)
|
|
455
|
+
values.set(name, value);
|
|
456
|
+
for (const name of ["content-type", "origin", "referer"]) {
|
|
457
|
+
const value = caller.get(name);
|
|
458
|
+
if (value !== undefined)
|
|
459
|
+
values.set(name, value);
|
|
153
460
|
}
|
|
154
|
-
|
|
155
|
-
|
|
461
|
+
// A bodyful class keeps its class across a 301/302/303 that rewrites the method
|
|
462
|
+
// to GET (a form submission's redirect stays a validated navigation), but the
|
|
463
|
+
// rewritten request has no body and Chrome sends no Content-Length for it.
|
|
464
|
+
if (shape.bodyful && options.method !== "GET" && options.method !== "HEAD") {
|
|
465
|
+
values.set("content-length", caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")));
|
|
156
466
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
for (const [name, value] of Object.entries(headers)) {
|
|
467
|
+
const isHttp1 = new URL(options.requestUrl).protocol === "http:";
|
|
468
|
+
if (isHttp1) {
|
|
469
|
+
values.set("host", new URL(options.requestUrl).host);
|
|
470
|
+
values.set("connection", "keep-alive");
|
|
471
|
+
}
|
|
472
|
+
const order = chromeHeaderOrder(requestClass, isHttp1, caller);
|
|
473
|
+
const tuples = [];
|
|
474
|
+
const placed = new Set();
|
|
475
|
+
for (const name of order) {
|
|
476
|
+
const value = values.get(name);
|
|
168
477
|
if (value === undefined)
|
|
169
478
|
continue;
|
|
170
|
-
|
|
479
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
480
|
+
placed.add(name);
|
|
171
481
|
}
|
|
172
|
-
|
|
482
|
+
for (const [name, value] of callerEntries) {
|
|
483
|
+
if (placed.has(name))
|
|
484
|
+
continue;
|
|
485
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
486
|
+
}
|
|
487
|
+
return tuples;
|
|
173
488
|
}
|
|
174
489
|
function hasOwn(object, key) {
|
|
175
490
|
return Object.hasOwn(object, key);
|
|
176
491
|
}
|
|
177
|
-
function toImpitCookieJar(cookieJar) {
|
|
178
|
-
return {
|
|
179
|
-
setCookie(cookie, _url, cb) {
|
|
180
|
-
cookieJar.setFromCookieStrings([cookie]);
|
|
181
|
-
if (typeof cb === "function")
|
|
182
|
-
cb();
|
|
183
|
-
},
|
|
184
|
-
getCookieString(_url) {
|
|
185
|
-
return cookieJar.toString();
|
|
186
|
-
},
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
492
|
function assertNoUnsupportedFingerprintOverrides(options) {
|
|
190
493
|
if (!isRecord(options))
|
|
191
494
|
return;
|
|
@@ -199,7 +502,7 @@ function assertNoUnsupportedFingerprintOverrides(options) {
|
|
|
199
502
|
unsupported.push("stealth.h2");
|
|
200
503
|
if (unsupported.length === 0)
|
|
201
504
|
return;
|
|
202
|
-
throw new SDKError(`ctx.stealth.fetch uses
|
|
505
|
+
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use stealth.browser and stealth.os instead.`);
|
|
203
506
|
}
|
|
204
507
|
function responseHeadersToRecord(headers) {
|
|
205
508
|
const record = {};
|
|
@@ -214,6 +517,29 @@ function setCookieHeadersFromResponse(headers) {
|
|
|
214
517
|
const setCookie = headers.get("set-cookie");
|
|
215
518
|
return setCookie ? splitCombinedSetCookieHeader(setCookie) : [];
|
|
216
519
|
}
|
|
520
|
+
function resolverCookiesFromResponse(response, requestUrl) {
|
|
521
|
+
const responseUrl = response.url ?? requestUrl;
|
|
522
|
+
return setCookieHeadersFromResponse(response.headers).flatMap((cookieString) => {
|
|
523
|
+
const cookie = Cookie.parse(cookieString);
|
|
524
|
+
if (!cookie?.key)
|
|
525
|
+
return [];
|
|
526
|
+
const expires = cookie.expires instanceof Date && Number.isFinite(cookie.expires.getTime())
|
|
527
|
+
? cookie.expires.getTime() / 1_000
|
|
528
|
+
: undefined;
|
|
529
|
+
return [
|
|
530
|
+
{
|
|
531
|
+
name: cookie.key,
|
|
532
|
+
value: cookie.value,
|
|
533
|
+
...(expires === undefined ? {} : { expires }),
|
|
534
|
+
httpOnly: cookie.httpOnly,
|
|
535
|
+
secure: cookie.secure,
|
|
536
|
+
...(cookie.domain ? { domain: cookie.domain } : { domain: new URL(responseUrl).hostname }),
|
|
537
|
+
...(cookie.path ? { path: cookie.path } : {}),
|
|
538
|
+
...(cookie.sameSite ? { sameSite: cookie.sameSite } : {}),
|
|
539
|
+
},
|
|
540
|
+
];
|
|
541
|
+
});
|
|
542
|
+
}
|
|
217
543
|
function splitCombinedSetCookieHeader(headerValue) {
|
|
218
544
|
const cookieNamePattern = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+=/;
|
|
219
545
|
const cookieStrings = [];
|
|
@@ -234,10 +560,15 @@ function splitCombinedSetCookieHeader(headerValue) {
|
|
|
234
560
|
cookieStrings.push(finalCookie);
|
|
235
561
|
return cookieStrings;
|
|
236
562
|
}
|
|
237
|
-
export async function normalizeResponse(response, requestUrl) {
|
|
563
|
+
export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
564
|
+
return normalizeResponseWithSignal(response, requestUrl, maxBodyBytes);
|
|
565
|
+
}
|
|
566
|
+
async function normalizeResponseWithSignal(response, requestUrl, maxBodyBytes, signal) {
|
|
238
567
|
const headers = Object.fromEntries(response.headers.entries());
|
|
239
|
-
const cookies = new
|
|
240
|
-
const bodyBytes =
|
|
568
|
+
const cookies = new StealthCookieJar(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
569
|
+
const bodyBytes = maxBodyBytes === undefined
|
|
570
|
+
? await readResponseArrayBuffer(response, signal)
|
|
571
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes, signal);
|
|
241
572
|
const body = new TextDecoder().decode(bodyBytes);
|
|
242
573
|
return {
|
|
243
574
|
status: response.status,
|
|
@@ -263,17 +594,141 @@ export async function normalizeResponse(response, requestUrl) {
|
|
|
263
594
|
},
|
|
264
595
|
};
|
|
265
596
|
}
|
|
597
|
+
async function readResponseArrayBuffer(response, signal) {
|
|
598
|
+
if (!signal)
|
|
599
|
+
return response.arrayBuffer();
|
|
600
|
+
throwIfAmbientAborted(signal);
|
|
601
|
+
return new Promise((resolve, reject) => {
|
|
602
|
+
let settled = false;
|
|
603
|
+
const settle = (operation) => {
|
|
604
|
+
if (settled)
|
|
605
|
+
return;
|
|
606
|
+
settled = true;
|
|
607
|
+
signal.removeEventListener("abort", onAbort);
|
|
608
|
+
operation();
|
|
609
|
+
};
|
|
610
|
+
const onAbort = () => {
|
|
611
|
+
const error = toAmbientCancellationError(signal);
|
|
612
|
+
try {
|
|
613
|
+
void response.body?.cancel().catch(() => undefined);
|
|
614
|
+
}
|
|
615
|
+
catch {
|
|
616
|
+
// Preserve the cancellation error if accessing or cancelling the body fails.
|
|
617
|
+
}
|
|
618
|
+
settle(() => reject(error));
|
|
619
|
+
};
|
|
620
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
621
|
+
try {
|
|
622
|
+
void response.arrayBuffer().then((body) => settle(() => resolve(body)), (error) => settle(() => reject(error)));
|
|
623
|
+
}
|
|
624
|
+
catch (error) {
|
|
625
|
+
settle(() => reject(error));
|
|
626
|
+
}
|
|
627
|
+
if (signal.aborted)
|
|
628
|
+
onAbort();
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
function responseTooLargeError(maxBodyBytes, observedBytes) {
|
|
632
|
+
return new TransportError(`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`, {
|
|
633
|
+
code: "response_too_large",
|
|
634
|
+
category: "upstream_http",
|
|
635
|
+
retryable: false,
|
|
636
|
+
status: 0,
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
function declaredContentLength(headers) {
|
|
640
|
+
const contentLength = headers.get("content-length")?.trim();
|
|
641
|
+
if (!contentLength || !/^\d+$/.test(contentLength))
|
|
642
|
+
return undefined;
|
|
643
|
+
const parsed = Number(contentLength);
|
|
644
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
645
|
+
}
|
|
646
|
+
async function readResponseBodyWithLimit(response, maxBodyBytes, signal) {
|
|
647
|
+
throwIfAmbientAborted(signal);
|
|
648
|
+
const contentLength = declaredContentLength(response.headers);
|
|
649
|
+
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
650
|
+
await response.body?.cancel().catch(() => undefined);
|
|
651
|
+
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
652
|
+
}
|
|
653
|
+
if (!response.body) {
|
|
654
|
+
throw new TransportError("Response body stream is unavailable", {
|
|
655
|
+
code: "transport_stream_unavailable",
|
|
656
|
+
category: "upstream_http",
|
|
657
|
+
status: 0,
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
const reader = response.body.getReader();
|
|
661
|
+
const chunks = [];
|
|
662
|
+
let receivedBytes = 0;
|
|
663
|
+
try {
|
|
664
|
+
while (true) {
|
|
665
|
+
const { done, value } = await readResponseBodyChunk(reader, signal);
|
|
666
|
+
if (done)
|
|
667
|
+
break;
|
|
668
|
+
if (!value)
|
|
669
|
+
continue;
|
|
670
|
+
receivedBytes += value.byteLength;
|
|
671
|
+
if (receivedBytes > maxBodyBytes) {
|
|
672
|
+
await reader.cancel().catch(() => undefined);
|
|
673
|
+
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
674
|
+
}
|
|
675
|
+
chunks.push(value);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
finally {
|
|
679
|
+
reader.releaseLock();
|
|
680
|
+
}
|
|
681
|
+
return concatenateResponseBodyChunks(chunks, receivedBytes);
|
|
682
|
+
}
|
|
683
|
+
function concatenateResponseBodyChunks(chunks, receivedBytes) {
|
|
684
|
+
const bodyBytes = new Uint8Array(receivedBytes);
|
|
685
|
+
let offset = 0;
|
|
686
|
+
for (const chunk of chunks) {
|
|
687
|
+
bodyBytes.set(chunk, offset);
|
|
688
|
+
offset += chunk.byteLength;
|
|
689
|
+
}
|
|
690
|
+
return bodyBytes.buffer;
|
|
691
|
+
}
|
|
692
|
+
function readResponseBodyChunk(reader, signal) {
|
|
693
|
+
if (!signal)
|
|
694
|
+
return reader.read();
|
|
695
|
+
throwIfAmbientAborted(signal);
|
|
696
|
+
return new Promise((resolve, reject) => {
|
|
697
|
+
let settled = false;
|
|
698
|
+
const settle = (operation) => {
|
|
699
|
+
if (settled)
|
|
700
|
+
return;
|
|
701
|
+
settled = true;
|
|
702
|
+
signal.removeEventListener("abort", onAbort);
|
|
703
|
+
operation();
|
|
704
|
+
};
|
|
705
|
+
const onAbort = () => {
|
|
706
|
+
const error = toAmbientCancellationError(signal);
|
|
707
|
+
void reader.cancel().catch(() => undefined);
|
|
708
|
+
settle(() => reject(error));
|
|
709
|
+
};
|
|
710
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
711
|
+
void reader.read().then((chunk) => settle(() => resolve(chunk)), (error) => settle(() => reject(error)));
|
|
712
|
+
if (signal.aborted)
|
|
713
|
+
onAbort();
|
|
714
|
+
});
|
|
715
|
+
}
|
|
266
716
|
function normalizeBody(body) {
|
|
267
717
|
if (body === undefined) {
|
|
268
|
-
return
|
|
718
|
+
return undefined;
|
|
269
719
|
}
|
|
270
720
|
if (typeof body === "string") {
|
|
271
721
|
return body;
|
|
272
722
|
}
|
|
273
723
|
if (Buffer.isBuffer(body)) {
|
|
274
|
-
return
|
|
724
|
+
return Buffer.from(body);
|
|
275
725
|
}
|
|
276
|
-
|
|
726
|
+
// `StealthFetchOptions.body` is `string | Buffer`; anything else is a provider fault,
|
|
727
|
+
// caught before the first request so a later replay can send the same bytes.
|
|
728
|
+
throw new SDKError("Stealth request bodies must be a string or Buffer", {
|
|
729
|
+
code: "STEALTH_BODY_UNSUPPORTED",
|
|
730
|
+
fix: "Supply a string or Buffer body; streams and typed arrays are not snapshotted.",
|
|
731
|
+
});
|
|
277
732
|
}
|
|
278
733
|
function isPolicyManagedProxy(options) {
|
|
279
734
|
const policy = options.proxyPolicy ?? options.upstream?.proxy;
|
|
@@ -315,6 +770,10 @@ function proxyPoolIndexFromDiagnostics(diagnostics) {
|
|
|
315
770
|
}
|
|
316
771
|
return Math.floor(value);
|
|
317
772
|
}
|
|
773
|
+
function positiveDiagnosticNumber(diagnostics, name) {
|
|
774
|
+
const value = diagnostics?.[name];
|
|
775
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
|
776
|
+
}
|
|
318
777
|
function proxyEndpointHash(proxyUrl) {
|
|
319
778
|
if (!proxyUrl)
|
|
320
779
|
return undefined;
|
|
@@ -395,8 +854,38 @@ function normalizeStealthTransportError(error) {
|
|
|
395
854
|
cause: error instanceof Error ? error : undefined,
|
|
396
855
|
});
|
|
397
856
|
}
|
|
398
|
-
function
|
|
399
|
-
|
|
857
|
+
function toAmbientCancellationError(signal, error = signal.reason) {
|
|
858
|
+
if (error instanceof TransportError && error.code === "transport_cancelled") {
|
|
859
|
+
return error;
|
|
860
|
+
}
|
|
861
|
+
return new TransportError("Request cancelled", {
|
|
862
|
+
code: "transport_cancelled",
|
|
863
|
+
status: 0,
|
|
864
|
+
retryable: false,
|
|
865
|
+
...(error !== undefined
|
|
866
|
+
? { cause: error instanceof Error ? error : new Error(String(error)) }
|
|
867
|
+
: {}),
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
function throwIfAmbientAborted(signal) {
|
|
871
|
+
if (signal?.aborted)
|
|
872
|
+
throw toAmbientCancellationError(signal);
|
|
873
|
+
}
|
|
874
|
+
function sleep(ms, signal) {
|
|
875
|
+
if (!signal)
|
|
876
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
877
|
+
throwIfAmbientAborted(signal);
|
|
878
|
+
return new Promise((resolve, reject) => {
|
|
879
|
+
const onAbort = () => {
|
|
880
|
+
clearTimeout(timer);
|
|
881
|
+
reject(toAmbientCancellationError(signal));
|
|
882
|
+
};
|
|
883
|
+
const timer = setTimeout(() => {
|
|
884
|
+
signal.removeEventListener("abort", onAbort);
|
|
885
|
+
resolve();
|
|
886
|
+
}, ms);
|
|
887
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
888
|
+
});
|
|
400
889
|
}
|
|
401
890
|
function normalizeMethod(method) {
|
|
402
891
|
switch (method.toUpperCase()) {
|
|
@@ -420,16 +909,6 @@ function normalizeMethod(method) {
|
|
|
420
909
|
throw new SDKError(`Unsupported stealth method: ${method}`);
|
|
421
910
|
}
|
|
422
911
|
}
|
|
423
|
-
function isRedirectStatus(status) {
|
|
424
|
-
return [301, 302, 303, 307, 308].includes(status);
|
|
425
|
-
}
|
|
426
|
-
function nextRedirectMethod(status, method) {
|
|
427
|
-
if (status === 303 && method !== "HEAD")
|
|
428
|
-
return "GET";
|
|
429
|
-
if ((status === 301 || status === 302) && method === "POST")
|
|
430
|
-
return "GET";
|
|
431
|
-
return method;
|
|
432
|
-
}
|
|
433
912
|
function locationHeader(headers) {
|
|
434
913
|
for (const [name, value] of Object.entries(headers)) {
|
|
435
914
|
if (name.toLowerCase() === "location")
|
|
@@ -437,44 +916,441 @@ function locationHeader(headers) {
|
|
|
437
916
|
}
|
|
438
917
|
return undefined;
|
|
439
918
|
}
|
|
919
|
+
/**
|
|
920
|
+
* Snapshot a caller's `redirectPolicy` before the first send. The policy acts
|
|
921
|
+
* on redirects the transport follows, so it requires the default
|
|
922
|
+
* `redirect: "follow"`.
|
|
923
|
+
*/
|
|
924
|
+
function snapshotStealthRedirectPolicy(options) {
|
|
925
|
+
let policy;
|
|
926
|
+
try {
|
|
927
|
+
policy = normalizeRedirectPolicy(options.redirectPolicy, STEALTH_REDIRECT_POLICY_TRANSPORT);
|
|
928
|
+
}
|
|
929
|
+
catch (error) {
|
|
930
|
+
if (error instanceof TransportError)
|
|
931
|
+
throw error;
|
|
932
|
+
throw invalidRedirectPolicy(STEALTH_REDIRECT_POLICY_TRANSPORT, "could not be read safely", error instanceof Error ? error : undefined);
|
|
933
|
+
}
|
|
934
|
+
if (policy && options.redirect !== undefined && options.redirect !== "follow") {
|
|
935
|
+
throw invalidRedirectPolicy(STEALTH_REDIRECT_POLICY_TRANSPORT, `it applies to followed redirects and cannot be combined with redirect: "${options.redirect}"`);
|
|
936
|
+
}
|
|
937
|
+
return policy;
|
|
938
|
+
}
|
|
939
|
+
function assertStealthRedirectUrl(url) {
|
|
940
|
+
const protocol = new URL(url).protocol;
|
|
941
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
942
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
943
|
+
code: "transport_invalid_url",
|
|
944
|
+
status: 0,
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
function discardStealthRedirectBody(response) {
|
|
949
|
+
try {
|
|
950
|
+
const cancellation = response.body?.cancel();
|
|
951
|
+
if (cancellation)
|
|
952
|
+
void cancellation.catch(() => undefined);
|
|
953
|
+
}
|
|
954
|
+
catch {
|
|
955
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
956
|
+
// failure must not replace or delay that decision.
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
/** Telemetry must never replace a transport result, including with caller-provided sinks. */
|
|
960
|
+
function emitStealthTelemetry(sink, record) {
|
|
961
|
+
if (!sink)
|
|
962
|
+
return;
|
|
963
|
+
try {
|
|
964
|
+
record(sink);
|
|
965
|
+
}
|
|
966
|
+
catch {
|
|
967
|
+
/* The request owns its result. */
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
function stripStealthCookieMaterial(text, cookieNames) {
|
|
971
|
+
// Header-shaped Cookie/Set-Cookie material is redacted wholesale, including
|
|
972
|
+
// foreign names. Bare pairs outside a header are limited to jar-known names:
|
|
973
|
+
// a foreign bare name is indistinguishable from benign key=value prose and is
|
|
974
|
+
// therefore covered only when layer 1 registered its value.
|
|
975
|
+
let result = text.replace(/(^|[^\p{L}\p{N}_-])[\t ]*(?:cookie|set-cookie)[\t ]*(?::|=)[^\r\n]*/giu, "$1[REDACTED]");
|
|
976
|
+
result = result.replace(/"(?:cookie|set-cookie)"[\t ]*:[\t ]*"(?:\\.|[^"\\])*"/giu, "[REDACTED]");
|
|
977
|
+
for (const name of cookieNames) {
|
|
978
|
+
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
979
|
+
result = result.replace(new RegExp(`(^|[;,&?\\s])${escaped}\\s*=\\s*[^;,\\s&\\r\\n]+`, "giu"), "$1[REDACTED]");
|
|
980
|
+
}
|
|
981
|
+
return result;
|
|
982
|
+
}
|
|
983
|
+
function startStealthExchange(telemetry, method, options, signal) {
|
|
984
|
+
if (!telemetry)
|
|
985
|
+
return () => { };
|
|
986
|
+
const startedAt = Date.now();
|
|
987
|
+
const requestClass = chromeRequestClass(method, options.stealth?.requestClass);
|
|
988
|
+
let recorded = false;
|
|
989
|
+
return (status, error) => {
|
|
990
|
+
if (recorded)
|
|
991
|
+
return;
|
|
992
|
+
recorded = true;
|
|
993
|
+
emitStealthTelemetry(telemetry?.sink, (sink) => {
|
|
994
|
+
let code;
|
|
995
|
+
if (error !== undefined) {
|
|
996
|
+
try {
|
|
997
|
+
code = signal?.aborted
|
|
998
|
+
? "transport_cancelled"
|
|
999
|
+
: normalizeStealthTransportError(error).code;
|
|
1000
|
+
}
|
|
1001
|
+
catch {
|
|
1002
|
+
code = "other";
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
const knownCodes = [
|
|
1006
|
+
"transport_network_error",
|
|
1007
|
+
"transport_timeout",
|
|
1008
|
+
"transport_cancelled",
|
|
1009
|
+
"upstream_http_error",
|
|
1010
|
+
"response_too_large",
|
|
1011
|
+
"proxy_connect_failed",
|
|
1012
|
+
"PROXY_POOL_STALE",
|
|
1013
|
+
"PROXY_EDGE_AUTH_REJECTED",
|
|
1014
|
+
"PROXY_AUTH_IP_DENIED",
|
|
1015
|
+
"PROXY_EDGE_TLS_REJECTED",
|
|
1016
|
+
"PROXY_REQUIRED",
|
|
1017
|
+
];
|
|
1018
|
+
const errorCode = code
|
|
1019
|
+
? knownCodes.includes(code)
|
|
1020
|
+
? code
|
|
1021
|
+
: "other"
|
|
1022
|
+
: status !== undefined && status >= 400
|
|
1023
|
+
? "upstream_http_error"
|
|
1024
|
+
: undefined;
|
|
1025
|
+
const strip = (text) => stripStealthCookieMaterial(text, telemetry.cookieJar?.names() ?? []);
|
|
1026
|
+
sink.recordAttempt({
|
|
1027
|
+
ms: Date.now() - startedAt,
|
|
1028
|
+
profileId: telemetry.profileId,
|
|
1029
|
+
proxyUsed: telemetry.proxyUsed,
|
|
1030
|
+
kind: telemetry.kind,
|
|
1031
|
+
requestClass: requestClass === "navigation" && options.stealth?.userActivation === false
|
|
1032
|
+
? "script_navigation"
|
|
1033
|
+
: requestClass === "form-post"
|
|
1034
|
+
? "form_post"
|
|
1035
|
+
: requestClass,
|
|
1036
|
+
...(status === undefined ? {} : { status }),
|
|
1037
|
+
...(errorCode ? { errorCode } : {}),
|
|
1038
|
+
...(error === undefined
|
|
1039
|
+
? {}
|
|
1040
|
+
: {
|
|
1041
|
+
diagnostics: {
|
|
1042
|
+
name: strip(error instanceof Error ? error.name : "Error"),
|
|
1043
|
+
message: strip(error instanceof Error ? error.message : String(error)),
|
|
1044
|
+
...(error instanceof Error && "code" in error && typeof error.code === "string"
|
|
1045
|
+
? { code: strip(error.code) }
|
|
1046
|
+
: {}),
|
|
1047
|
+
},
|
|
1048
|
+
}),
|
|
1049
|
+
});
|
|
1050
|
+
});
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, signal, buildHeaders, telemetry) {
|
|
1054
|
+
let currentUrl = requestUrl;
|
|
1055
|
+
let currentMethod = method;
|
|
1056
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
1057
|
+
let currentHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1058
|
+
let followedHops = 0;
|
|
1059
|
+
let response;
|
|
1060
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
1061
|
+
// Snapshotted and validated by session.fetch before the first send.
|
|
1062
|
+
const redirectPolicy = options.redirectPolicy;
|
|
1063
|
+
const initialOrigin = new URL(requestUrl).origin;
|
|
1064
|
+
const visitedRequests = new Set([`${method} ${requestUrl}`]);
|
|
1065
|
+
let finishExchange;
|
|
1066
|
+
let status;
|
|
1067
|
+
let exchangeError;
|
|
1068
|
+
try {
|
|
1069
|
+
while (true) {
|
|
1070
|
+
throwIfAmbientAborted(signal);
|
|
1071
|
+
const headers = { ...currentHeaders };
|
|
1072
|
+
if (!hasHeader(headers, "cookie")) {
|
|
1073
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
1074
|
+
if (cookieHeader)
|
|
1075
|
+
headers.cookie = cookieHeader;
|
|
1076
|
+
}
|
|
1077
|
+
const requestInit = {
|
|
1078
|
+
headers: buildHeaders
|
|
1079
|
+
? buildHeaders(currentUrl, currentMethod, currentBody, headers)
|
|
1080
|
+
: headers,
|
|
1081
|
+
method: currentMethod,
|
|
1082
|
+
redirect: "manual",
|
|
1083
|
+
...(new URL(currentUrl).protocol === "http:" ? { disableDefaultHeaders: true } : {}),
|
|
1084
|
+
...(signal ? { signal } : {}),
|
|
1085
|
+
};
|
|
1086
|
+
if (currentBody !== undefined)
|
|
1087
|
+
requestInit.body = currentBody;
|
|
1088
|
+
await transport.clearCookies();
|
|
1089
|
+
throwIfAmbientAborted(signal);
|
|
1090
|
+
const remainingTimeout = deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
1091
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
1092
|
+
throw new TransportError("Request timed out", {
|
|
1093
|
+
code: "transport_timeout",
|
|
1094
|
+
status: 0,
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
if (remainingTimeout !== undefined)
|
|
1098
|
+
requestInit.timeout = remainingTimeout;
|
|
1099
|
+
status = undefined;
|
|
1100
|
+
finishExchange = startStealthExchange(telemetry, currentMethod, options, signal);
|
|
1101
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
1102
|
+
status = response.status;
|
|
1103
|
+
if (signal?.aborted) {
|
|
1104
|
+
discardStealthRedirectBody(response);
|
|
1105
|
+
throw toAmbientCancellationError(signal);
|
|
1106
|
+
}
|
|
1107
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? currentUrl);
|
|
1108
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual")
|
|
1109
|
+
break;
|
|
1110
|
+
if (options.redirect === "error") {
|
|
1111
|
+
discardStealthRedirectBody(response);
|
|
1112
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
1113
|
+
code: "transport_network_error",
|
|
1114
|
+
status: 0,
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
let nextUrl;
|
|
1118
|
+
let nextMethod;
|
|
1119
|
+
if (redirectPolicy) {
|
|
1120
|
+
discardStealthRedirectBody(response);
|
|
1121
|
+
({ nextUrl, nextMethod } = nextSameOriginRedirectHop({
|
|
1122
|
+
policy: redirectPolicy,
|
|
1123
|
+
initialOrigin,
|
|
1124
|
+
currentUrl,
|
|
1125
|
+
responseUrl: response.url ?? currentUrl,
|
|
1126
|
+
status: response.status,
|
|
1127
|
+
method: currentMethod,
|
|
1128
|
+
location: response.headers.get("location") ?? undefined,
|
|
1129
|
+
followedHops,
|
|
1130
|
+
visitedRequests,
|
|
1131
|
+
}));
|
|
1132
|
+
}
|
|
1133
|
+
else {
|
|
1134
|
+
const location = resolveRedirectUrl(response.headers.get("location") ?? undefined, response.url ?? currentUrl);
|
|
1135
|
+
if (!location)
|
|
1136
|
+
break;
|
|
1137
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
1138
|
+
discardStealthRedirectBody(response);
|
|
1139
|
+
throw new TransportError(`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`, { code: "transport_network_error", status: 0 });
|
|
1140
|
+
}
|
|
1141
|
+
discardStealthRedirectBody(response);
|
|
1142
|
+
nextUrl = location;
|
|
1143
|
+
nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
1144
|
+
}
|
|
1145
|
+
assertStealthRedirectUrl(nextUrl);
|
|
1146
|
+
currentHeaders = redirectHopHeaders({ from: currentUrl, to: nextUrl, methodChanged: nextMethod !== currentMethod }, currentHeaders);
|
|
1147
|
+
if (nextMethod !== currentMethod)
|
|
1148
|
+
currentBody = undefined;
|
|
1149
|
+
currentMethod = nextMethod;
|
|
1150
|
+
currentUrl = nextUrl;
|
|
1151
|
+
followedHops += 1;
|
|
1152
|
+
visitedRequests.add(`${currentMethod} ${currentUrl}`);
|
|
1153
|
+
emitStealthTelemetry(telemetry?.sink, (sink) => sink.recordRedirectHop());
|
|
1154
|
+
finishExchange(status);
|
|
1155
|
+
finishExchange = undefined;
|
|
1156
|
+
}
|
|
1157
|
+
const normalized = await normalizeResponseWithSignal(response, currentUrl, options.maxBodyBytes, signal);
|
|
1158
|
+
if (followedHops > 0)
|
|
1159
|
+
normalized.redirected = true;
|
|
1160
|
+
return { normalized, response };
|
|
1161
|
+
}
|
|
1162
|
+
catch (error) {
|
|
1163
|
+
exchangeError = error;
|
|
1164
|
+
throw error;
|
|
1165
|
+
}
|
|
1166
|
+
finally {
|
|
1167
|
+
finishExchange?.(status, exchangeError);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
440
1170
|
function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
441
1171
|
const clients = new Map();
|
|
442
1172
|
let closed = false;
|
|
443
1173
|
let hasWarnedMissingProxy = false;
|
|
444
1174
|
const warn = clientOptions.warn ?? console.warn;
|
|
445
|
-
const cookieJar = new
|
|
446
|
-
const
|
|
447
|
-
|
|
1175
|
+
const cookieJar = new StealthCookieJar([], baseUrl, registerDiagnosticValue);
|
|
1176
|
+
const akamaiSbsdState = { transactions: new Map() };
|
|
1177
|
+
const challengedReplays = new WeakMap();
|
|
1178
|
+
const ceremonyEgressLease = clientOptions[ENGINE_CEREMONY_EGRESS_LEASE];
|
|
1179
|
+
// The lease is shared by every session of this client; this is the lease generation this
|
|
1180
|
+
// session's challenge state and replay records were built against.
|
|
1181
|
+
let observedEgressGeneration = ceremonyEgressLease?.generation ?? 0;
|
|
1182
|
+
const automaticChallengeRefetchPolicy = {
|
|
1183
|
+
...createDefaultProxyTransportRetryOptions({ label: "Stealth" }),
|
|
1184
|
+
methods: ["GET"],
|
|
1185
|
+
};
|
|
1186
|
+
function clearAkamaiSbsdState() {
|
|
1187
|
+
akamaiSbsdState.rememberedScript = undefined;
|
|
1188
|
+
akamaiSbsdState.transactions.clear();
|
|
1189
|
+
akamaiSbsdState.completedSuccessKey = undefined;
|
|
1190
|
+
}
|
|
1191
|
+
function currentEgressGeneration() {
|
|
1192
|
+
return ceremonyEgressLease?.generation ?? 0;
|
|
1193
|
+
}
|
|
1194
|
+
/**
|
|
1195
|
+
* Whoever dropped the binding (this session by expiry or release, or another session of
|
|
1196
|
+
* the same client), the challenge state that was only valid on the retired endpoint goes
|
|
1197
|
+
* with it, and replay records made against the old generation stop matching.
|
|
1198
|
+
*/
|
|
1199
|
+
function syncCeremonyEgress() {
|
|
1200
|
+
const generation = currentEgressGeneration();
|
|
1201
|
+
if (generation === observedEgressGeneration)
|
|
1202
|
+
return;
|
|
1203
|
+
observedEgressGeneration = generation;
|
|
1204
|
+
clearAkamaiSbsdState();
|
|
1205
|
+
}
|
|
1206
|
+
/**
|
|
1207
|
+
* A binding past its vendor session lifetime is dropped, together with the challenge
|
|
1208
|
+
* state that was only valid on that endpoint, so this request selects and binds afresh.
|
|
1209
|
+
*/
|
|
1210
|
+
function expireCeremonyEgressLease() {
|
|
1211
|
+
ceremonyEgressLease?.dropExpiredBinding();
|
|
1212
|
+
syncCeremonyEgress();
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* The lease is best effort (ADR-0010: a raw Smartproxy endpoint is not a vendor-guaranteed
|
|
1216
|
+
* lease). When the attempt that ran on the bound endpoint fails at the transport (the
|
|
1217
|
+
* proxy refused it, the tunnel never opened, the socket died or timed out) the binding is
|
|
1218
|
+
* released so the ceremony can leave a dead endpoint before its lifetime elapses. Origin
|
|
1219
|
+
* responses, however bad, keep the binding: the endpoint is alive. Returns true when a
|
|
1220
|
+
* binding was released.
|
|
1221
|
+
*/
|
|
1222
|
+
function releaseFailedCeremonyEgress(attemptProxyUrl, error) {
|
|
1223
|
+
const bound = ceremonyEgressLease?.binding;
|
|
1224
|
+
if (!ceremonyEgressLease ||
|
|
1225
|
+
!bound ||
|
|
1226
|
+
!attemptProxyUrl ||
|
|
1227
|
+
attemptProxyUrl !== bound.proxyUrl ||
|
|
1228
|
+
!isBoundEgressTransportFailure(error)) {
|
|
1229
|
+
return false;
|
|
1230
|
+
}
|
|
1231
|
+
ceremonyEgressLease.dropBinding();
|
|
1232
|
+
syncCeremonyEgress();
|
|
1233
|
+
return true;
|
|
1234
|
+
}
|
|
1235
|
+
async function getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders) {
|
|
448
1236
|
if (closed) {
|
|
449
1237
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
450
1238
|
}
|
|
451
|
-
const
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
1239
|
+
const wreq = await getWreqModule();
|
|
1240
|
+
const { browser, os } = resolveWreqProfile(profile, wreq.getProfiles());
|
|
1241
|
+
const cacheKey = JSON.stringify({
|
|
1242
|
+
browser,
|
|
1243
|
+
os,
|
|
1244
|
+
proxyUrl,
|
|
1245
|
+
ignoreTlsErrors,
|
|
1246
|
+
headerOrder: defaultHeaders?.map(([name]) => name),
|
|
1247
|
+
});
|
|
1248
|
+
let entry = clients.get(cacheKey);
|
|
1249
|
+
if (!entry) {
|
|
1250
|
+
entry = {
|
|
1251
|
+
// The header shape only partitions sessions; it is never handed to wreq as
|
|
1252
|
+
// session `defaultHeaders`. wreq merges those into every request that lacks
|
|
1253
|
+
// one of their names, so a header the SDK left off on purpose — a credential
|
|
1254
|
+
// dropped on a cross-origin redirect hop, the body headers after POST -> GET,
|
|
1255
|
+
// a cookie for another host — would come back with the first request's value.
|
|
1256
|
+
// Every request already sends its complete ordered header list.
|
|
1257
|
+
session: wreq.createSession({
|
|
1258
|
+
browser,
|
|
1259
|
+
os,
|
|
1260
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
1261
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
1262
|
+
timeout: 30_000,
|
|
1263
|
+
}),
|
|
1264
|
+
tail: Promise.resolve(),
|
|
1265
|
+
};
|
|
1266
|
+
clients.set(cacheKey, entry);
|
|
463
1267
|
}
|
|
464
|
-
return
|
|
1268
|
+
return entry;
|
|
465
1269
|
}
|
|
466
|
-
async function
|
|
1270
|
+
async function withClient(profile, proxyUrl, ignoreTlsErrors, operation, signal, defaultHeaders) {
|
|
1271
|
+
throwIfAmbientAborted(signal);
|
|
1272
|
+
const entry = await getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders);
|
|
1273
|
+
throwIfAmbientAborted(signal);
|
|
1274
|
+
const previous = entry.tail;
|
|
1275
|
+
let release;
|
|
1276
|
+
entry.tail = new Promise((resolve) => {
|
|
1277
|
+
release = resolve;
|
|
1278
|
+
});
|
|
1279
|
+
let acquired = false;
|
|
1280
|
+
try {
|
|
1281
|
+
await waitForClientTurn(previous, signal);
|
|
1282
|
+
acquired = true;
|
|
1283
|
+
throwIfAmbientAborted(signal);
|
|
1284
|
+
const client = await entry.session;
|
|
1285
|
+
throwIfAmbientAborted(signal);
|
|
1286
|
+
const result = await operation(client);
|
|
1287
|
+
throwIfAmbientAborted(signal);
|
|
1288
|
+
return result;
|
|
1289
|
+
}
|
|
1290
|
+
finally {
|
|
1291
|
+
if (acquired) {
|
|
1292
|
+
release();
|
|
1293
|
+
}
|
|
1294
|
+
else {
|
|
1295
|
+
void previous.then(release, release);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
async function waitForClientTurn(previous, signal) {
|
|
1300
|
+
if (!signal) {
|
|
1301
|
+
await previous;
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
throwIfAmbientAborted(signal);
|
|
1305
|
+
await new Promise((resolve, reject) => {
|
|
1306
|
+
let settled = false;
|
|
1307
|
+
const settle = (operation) => {
|
|
1308
|
+
if (settled)
|
|
1309
|
+
return;
|
|
1310
|
+
settled = true;
|
|
1311
|
+
signal.removeEventListener("abort", onAbort);
|
|
1312
|
+
operation();
|
|
1313
|
+
};
|
|
1314
|
+
const onAbort = () => settle(() => reject(toAmbientCancellationError(signal)));
|
|
1315
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1316
|
+
void previous.then(() => settle(resolve), (error) => settle(() => reject(error)));
|
|
1317
|
+
if (signal.aborted)
|
|
1318
|
+
onAbort();
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
1322
|
+
const bound = ceremonyEgressLease?.binding;
|
|
1323
|
+
if (bound) {
|
|
1324
|
+
if (options?.proxy !== undefined || options?.proxyAttemptOffset !== undefined) {
|
|
1325
|
+
throw new SDKError("A ceremony-bound egress cannot be overridden by provider request options", {
|
|
1326
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1327
|
+
fix: "Remove `proxy` and `proxyAttemptOffset` from stealth requests made inside an auth ceremony.",
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
return {
|
|
1331
|
+
url: bound.proxyUrl,
|
|
1332
|
+
poolIndex: bound.poolIndex,
|
|
1333
|
+
proxyHash: proxyEndpointHash(bound.proxyUrl),
|
|
1334
|
+
vendor: bound.vendor,
|
|
1335
|
+
refreshEpoch: bound.refreshEpoch,
|
|
1336
|
+
lifetimeMinutes: bound.lifetimeMinutes,
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
467
1339
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
468
1340
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
469
1341
|
upstream: clientOptions.upstream,
|
|
470
|
-
apifuseConfig: clientOptions.apifuseConfig,
|
|
471
1342
|
affinityKey: clientOptions.affinityKey,
|
|
472
1343
|
proxyAttempt: computeProxyAttemptIndex({
|
|
473
1344
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
474
1345
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
475
1346
|
retryAttemptOffset: proxyAttempt,
|
|
476
1347
|
}),
|
|
1348
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
1349
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
1350
|
+
transportProtocols: ["http", "socks5"],
|
|
1351
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
477
1352
|
telemetry: clientOptions.telemetry,
|
|
1353
|
+
engineCredentials: clientOptions.engineCredentials,
|
|
478
1354
|
});
|
|
479
1355
|
if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
|
|
480
1356
|
hasWarnedMissingProxy = true;
|
|
@@ -484,50 +1360,152 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
484
1360
|
url: resolvedProxy.url,
|
|
485
1361
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
486
1362
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
1363
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
1364
|
+
refreshEpoch: refreshEpoch ?? 0,
|
|
1365
|
+
...(positiveDiagnosticNumber(resolvedProxy.diagnostics, "lifetimeMinutes") === undefined
|
|
1366
|
+
? {}
|
|
1367
|
+
: {
|
|
1368
|
+
lifetimeMinutes: positiveDiagnosticNumber(resolvedProxy.diagnostics, "lifetimeMinutes"),
|
|
1369
|
+
}),
|
|
487
1370
|
};
|
|
488
1371
|
}
|
|
1372
|
+
function bindCeremonyEgress(attemptProxy) {
|
|
1373
|
+
const lease = ceremonyEgressLease;
|
|
1374
|
+
if (!lease || !attemptProxy?.url || !attemptProxy.vendor)
|
|
1375
|
+
return;
|
|
1376
|
+
if (lease.binding) {
|
|
1377
|
+
// A concurrent request on this session may have bound the ceremony while this one
|
|
1378
|
+
// was in flight on an endpoint it selected before the binding existed (pool
|
|
1379
|
+
// rotation after a transport failure). Its response reached the origin from an
|
|
1380
|
+
// egress the handle does not name; accepting it would split the ceremony identity.
|
|
1381
|
+
if (attemptProxy.url !== lease.binding.proxyUrl) {
|
|
1382
|
+
throw new SDKError("The response arrived on an egress other than the ceremony's bound endpoint", {
|
|
1383
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
if (attemptProxy.poolIndex === undefined ||
|
|
1389
|
+
attemptProxy.lifetimeMinutes === undefined ||
|
|
1390
|
+
!clientOptions.affinityKey) {
|
|
1391
|
+
// Registry vendors always report pool index and session lifetime; missing either
|
|
1392
|
+
// is an SDK fault, not something the caller can correct.
|
|
1393
|
+
throw new SDKError("The selected egress cannot be represented by a ceremony lease", {
|
|
1394
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
const binding = {
|
|
1398
|
+
vendor: attemptProxy.vendor,
|
|
1399
|
+
proxyUrl: attemptProxy.url,
|
|
1400
|
+
poolIndex: attemptProxy.poolIndex,
|
|
1401
|
+
affinityKey: clientOptions.affinityKey,
|
|
1402
|
+
refreshEpoch: attemptProxy.refreshEpoch ?? 0,
|
|
1403
|
+
lifetimeMinutes: attemptProxy.lifetimeMinutes,
|
|
1404
|
+
};
|
|
1405
|
+
lease.bind(binding);
|
|
1406
|
+
}
|
|
489
1407
|
const session = {
|
|
490
|
-
async fetch(url,
|
|
491
|
-
|
|
492
|
-
const
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
1408
|
+
async fetch(url, callerOptions = {}) {
|
|
1409
|
+
expireCeremonyEgressLease();
|
|
1410
|
+
const requestBody = normalizeBody(callerOptions.body);
|
|
1411
|
+
// Snapshot the caller's headers with the body before the first send: the first
|
|
1412
|
+
// request and an explicit replay both use this copy, so a caller mutating its own
|
|
1413
|
+
// objects afterwards cannot make the two sends differ.
|
|
1414
|
+
const options = {
|
|
1415
|
+
...callerOptions,
|
|
1416
|
+
...(callerOptions.headers
|
|
1417
|
+
? {
|
|
1418
|
+
headers: Object.fromEntries(Object.entries(callerOptions.headers).map(([name, value]) => [
|
|
1419
|
+
name,
|
|
1420
|
+
Array.isArray(value) ? [...value] : value,
|
|
1421
|
+
])),
|
|
1422
|
+
}
|
|
1423
|
+
: {}),
|
|
1424
|
+
...(requestBody === undefined ? {} : { body: requestBody }),
|
|
1425
|
+
};
|
|
1426
|
+
const requestProfile = resolveStealthProfileSelection(options.stealth, defaultProfile);
|
|
1427
|
+
let challengeSolveAttempted = false;
|
|
1428
|
+
let challengeRefetchAttempted = false;
|
|
1429
|
+
let challengeSolveFailure;
|
|
1430
|
+
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
1431
|
+
try {
|
|
1432
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
1433
|
+
assertCallerHeadersSupported(options.headers ?? {});
|
|
1434
|
+
// Replace the caller's object with the validated copy: the walker, a
|
|
1435
|
+
// retry and an explicit replay all read this snapshot.
|
|
1436
|
+
const redirectPolicy = snapshotStealthRedirectPolicy(options);
|
|
1437
|
+
if (redirectPolicy)
|
|
1438
|
+
options.redirectPolicy = redirectPolicy;
|
|
1439
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
1440
|
+
const stealthRetryOptions = normalizeProxyTransportRetryOptions(options.retry, {
|
|
500
1441
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
501
1442
|
label: "Stealth",
|
|
502
|
-
})
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
1443
|
+
}) ??
|
|
1444
|
+
(hasExplicitRetryPolicy
|
|
1445
|
+
? undefined
|
|
1446
|
+
: createDefaultProxyTransportRetryOptions({
|
|
1447
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
1448
|
+
label: "Stealth",
|
|
1449
|
+
}));
|
|
1450
|
+
if (stealthRetryOptions) {
|
|
1451
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
1452
|
+
}
|
|
1453
|
+
return { hasExplicitRetryPolicy, method, stealthRetryOptions };
|
|
1454
|
+
}
|
|
1455
|
+
catch (error) {
|
|
1456
|
+
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
1457
|
+
}
|
|
1458
|
+
})();
|
|
1459
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
506
1460
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
507
1461
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
508
1462
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
509
|
-
|
|
1463
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
1464
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
1465
|
+
const policyProxy = clientOptions.proxyPolicy ??
|
|
510
1466
|
(typeof clientOptions.upstream?.proxy === "object"
|
|
511
|
-
? clientOptions.upstream.proxy
|
|
512
|
-
: undefined)
|
|
513
|
-
|
|
514
|
-
|
|
1467
|
+
? clientOptions.upstream.proxy
|
|
1468
|
+
: undefined);
|
|
1469
|
+
// The pool span is already bounded by each vendor's max pool size
|
|
1470
|
+
// (smartproxy ≤20, nodemaven ≤50), so the configured span never exceeds
|
|
1471
|
+
// the chain's true maximum — a large NodeMaven pool stays fully
|
|
1472
|
+
// reachable rather than being truncated at an arbitrary ceiling.
|
|
1473
|
+
const policyProxyAttemptCap = Math.max(1, policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE);
|
|
1474
|
+
// A registry vendor chain (smartproxy/nodemaven) is the only policy whose
|
|
1475
|
+
// successive attempts resolve a *different* endpoint, so it is the only one
|
|
1476
|
+
// that may widen the attempt cap to the pool span, de-duplicate endpoints,
|
|
1477
|
+
// and drive allocator stale-pool refresh. Deprecated custom/decodo policies
|
|
1478
|
+
// have no managed endpoint to rotate or refresh, so they follow the ordinary
|
|
1479
|
+
// transport-retry budget instead.
|
|
1480
|
+
const rotatesRegistryChain = usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
1481
|
+
const unboundMaxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
1482
|
+
// A bound ceremony has exactly one endpoint to try; releasing the binding after a
|
|
1483
|
+
// transport failure widens the budget back to the ordinary rotation span.
|
|
1484
|
+
let maxAttempts = ceremonyEgressLease?.binding ? 1 : unboundMaxAttempts;
|
|
1485
|
+
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
515
1486
|
let lastError;
|
|
516
1487
|
for (let refreshAttempt = 0; refreshAttempt <= MAX_POLICY_PROXY_POOL_REFRESHES; refreshAttempt += 1) {
|
|
517
1488
|
let stalePoolError;
|
|
518
1489
|
let stalePoolDiagnosticProxy;
|
|
519
1490
|
const attemptedProxies = new Set();
|
|
520
1491
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
1492
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
521
1493
|
let proxy;
|
|
522
1494
|
let attemptProxy;
|
|
523
|
-
|
|
1495
|
+
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
1496
|
+
let serializedUrl;
|
|
1497
|
+
let fallbackSensitiveValues = [];
|
|
1498
|
+
let fallbackRequestUrl;
|
|
1499
|
+
let fallbackRedactedUrl;
|
|
1500
|
+
let attemptStartedAt = Date.now();
|
|
524
1501
|
let attemptRecorded = false;
|
|
1502
|
+
const attemptEgressGeneration = currentEgressGeneration();
|
|
525
1503
|
const recordProxyAttempt = (outcome, errorCode, status) => {
|
|
526
1504
|
if (attemptRecorded || !proxy)
|
|
527
1505
|
return;
|
|
528
1506
|
attemptRecorded = true;
|
|
529
1507
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
530
|
-
provider: "smartproxy",
|
|
1508
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
531
1509
|
attempt: attempt + 1,
|
|
532
1510
|
...(attemptProxy?.poolIndex === undefined
|
|
533
1511
|
? {}
|
|
@@ -539,75 +1517,373 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
539
1517
|
durationMs: Date.now() - attemptStartedAt,
|
|
540
1518
|
});
|
|
541
1519
|
};
|
|
1520
|
+
// Shared tail of the first fetch, the automatic refetch, and an explicit replay: a
|
|
1521
|
+
// classified challenge is returned whatever `throwOnHttpError` says (documented on
|
|
1522
|
+
// StealthFetchOptions), any other non-2xx honours it, and the proxy attempt is
|
|
1523
|
+
// recorded once per delivered response.
|
|
1524
|
+
const finalizeAttempt = (attemptResponse, attemptNormalized) => {
|
|
1525
|
+
if (attemptNormalized.challenge)
|
|
1526
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSbsd(attemptNormalized.challenge.outcome));
|
|
1527
|
+
if (!attemptNormalized.challenge &&
|
|
1528
|
+
attemptResponse.status >= 400 &&
|
|
1529
|
+
options.throwOnHttpError !== false) {
|
|
1530
|
+
throw new TransportError(`Upstream request failed with status ${attemptResponse.status}`, { code: "upstream_http_error", status: attemptResponse.status });
|
|
1531
|
+
}
|
|
1532
|
+
recordProxyAttempt("ok", undefined, attemptResponse.status);
|
|
1533
|
+
return attemptNormalized;
|
|
1534
|
+
};
|
|
1535
|
+
const redactAttemptError = (error) => redactSensitiveError(error, serializedUrl?.sensitiveValues ?? fallbackSensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
1536
|
+
// Cancellation and resolver failures leave the transport path here, redacted; any
|
|
1537
|
+
// other failure becomes the TransportError the retry decision reasons about.
|
|
1538
|
+
const normalizeAttemptError = (error) => {
|
|
1539
|
+
let normalizedError;
|
|
1540
|
+
try {
|
|
1541
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1542
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
1543
|
+
}
|
|
1544
|
+
catch (normalizationError) {
|
|
1545
|
+
if (normalizationError instanceof TransportError &&
|
|
1546
|
+
normalizationError.code === "transport_cancelled") {
|
|
1547
|
+
recordProxyAttempt("error", proxyAttemptErrorCode(normalizationError), proxyAttemptStatus(normalizationError));
|
|
1548
|
+
}
|
|
1549
|
+
throw redactAttemptError(normalizationError);
|
|
1550
|
+
}
|
|
1551
|
+
if (challengeSolveFailure && error === challengeSolveFailure) {
|
|
1552
|
+
throw redactAttemptError(challengeSolveFailure.error);
|
|
1553
|
+
}
|
|
1554
|
+
return normalizedError;
|
|
1555
|
+
};
|
|
1556
|
+
const recordAttemptFailure = (normalizedError) => {
|
|
1557
|
+
const redactedError = redactAttemptError(normalizedError);
|
|
1558
|
+
recordProxyAttempt("error", proxyAttemptErrorCode(redactedError), proxyAttemptStatus(redactedError));
|
|
1559
|
+
return redactedError;
|
|
1560
|
+
};
|
|
542
1561
|
try {
|
|
1562
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1563
|
+
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
1564
|
+
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
1565
|
+
fallbackSensitiveValues = [
|
|
1566
|
+
...new Set([
|
|
1567
|
+
...Object.values(sensitiveParams ?? {}).map(String),
|
|
1568
|
+
...structural.sensitiveValues,
|
|
1569
|
+
]),
|
|
1570
|
+
].filter((value) => value !== "");
|
|
1571
|
+
fallbackRequestUrl = url;
|
|
1572
|
+
fallbackRedactedUrl = structural.redactedUrl;
|
|
543
1573
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
544
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
1574
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
1575
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
545
1576
|
proxy = attemptProxy.url;
|
|
546
|
-
if (proxy &&
|
|
1577
|
+
if (proxy && dedupeAllocatorEndpoints) {
|
|
1578
|
+
// An under-filled allocation repeats endpoints (via the modulo
|
|
1579
|
+
// pool mapping) before the flat offset crosses into the next
|
|
1580
|
+
// vendor. Skip an already-tried endpoint and advance the offset
|
|
1581
|
+
// rather than breaking — breaking here would strand the request on
|
|
1582
|
+
// the primary vendor and never reach the fallback leg.
|
|
547
1583
|
if (attemptedProxies.has(proxy)) {
|
|
548
|
-
|
|
1584
|
+
continue;
|
|
549
1585
|
}
|
|
550
1586
|
attemptedProxies.add(proxy);
|
|
551
1587
|
}
|
|
552
1588
|
const ignoreTlsErrors = Boolean(options.stealth?.insecureSkipVerify ??
|
|
553
1589
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify));
|
|
554
|
-
|
|
555
|
-
const requestUrl =
|
|
556
|
-
const
|
|
557
|
-
|
|
558
|
-
|
|
1590
|
+
serializedUrl = serializeRequestUrl(resolveRequestUrl(baseUrl, url), options.params, sensitiveParams);
|
|
1591
|
+
const { requestUrl } = serializedUrl;
|
|
1592
|
+
const wreq = await getWreqModule();
|
|
1593
|
+
const mapping = resolveWreqProfile(requestProfile, wreq.getProfiles());
|
|
1594
|
+
const emulationHeaders = Array.from(wreq.getEmulationHeaders(mapping.browser, mapping.os), ([name, value]) => [String(name), String(value)]);
|
|
1595
|
+
const chromeEmulationHeaders = mapping.browser.startsWith("chrome_")
|
|
1596
|
+
? emulationHeaders
|
|
1597
|
+
: undefined;
|
|
1598
|
+
const buildOrderedHeaders = chromeEmulationHeaders
|
|
1599
|
+
? (currentUrl, currentMethod, currentBody, currentHeaders) => buildChromeHeaderTuples({
|
|
1600
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1601
|
+
method: currentMethod,
|
|
1602
|
+
body: currentBody,
|
|
1603
|
+
headers: currentHeaders,
|
|
1604
|
+
requestUrl: currentUrl,
|
|
1605
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1606
|
+
requestClass: options.stealth?.requestClass,
|
|
1607
|
+
userActivation: options.stealth?.userActivation,
|
|
1608
|
+
})
|
|
1609
|
+
: undefined;
|
|
1610
|
+
const initialHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1611
|
+
if (!hasHeader(initialHeaders, "cookie")) {
|
|
1612
|
+
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
559
1613
|
if (cookieHeader)
|
|
560
|
-
|
|
1614
|
+
initialHeaders.cookie = cookieHeader;
|
|
561
1615
|
}
|
|
562
|
-
const
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
const
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
if (response.status >= 400) {
|
|
578
|
-
if (proxy &&
|
|
579
|
-
usesPolicyAllocator &&
|
|
580
|
-
isProxyEdgeTlsRejectedResponse(response.status, [JSON.stringify(responseHeadersToRecord(response.headers)), normalized.body].join("\n"))) {
|
|
581
|
-
throw createProxyEdgeTlsRejectedError(response.status);
|
|
1616
|
+
const defaultHeaders = buildOrderedHeaders?.(requestUrl, method, requestBody, initialHeaders);
|
|
1617
|
+
const fetchOnBoundSession = (fetchUrl, fetchMethod, fetchOptions, fetchSignal, orderedHeaders = buildOrderedHeaders, sessionDefaultHeaders = defaultHeaders, kind = "request") => withClient(requestProfile, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, fetchUrl, fetchMethod, fetchOptions, fetchSignal, orderedHeaders, clientOptions.stealthTelemetry
|
|
1618
|
+
? {
|
|
1619
|
+
sink: clientOptions.stealthTelemetry,
|
|
1620
|
+
profileId: requestProfile,
|
|
1621
|
+
proxyUsed: Boolean(proxy),
|
|
1622
|
+
kind,
|
|
1623
|
+
cookieJar,
|
|
1624
|
+
}
|
|
1625
|
+
: undefined), fetchSignal, sessionDefaultHeaders);
|
|
1626
|
+
const throwProxyTransportFault = (faultResponse, faultBody) => {
|
|
1627
|
+
if (!proxy)
|
|
1628
|
+
return;
|
|
1629
|
+
if (isProxyConnectFailureResponse(faultResponse, faultBody)) {
|
|
1630
|
+
throw createProxyConnectFailureError(faultBody);
|
|
582
1631
|
}
|
|
583
|
-
if (
|
|
1632
|
+
if (faultResponse.status < 400)
|
|
1633
|
+
return;
|
|
1634
|
+
if (usesPolicyAllocator &&
|
|
1635
|
+
isProxyEdgeTlsRejectedResponse(faultResponse.status, [JSON.stringify(responseHeadersToRecord(faultResponse.headers)), faultBody].join("\n"))) {
|
|
1636
|
+
throw createProxyEdgeTlsRejectedError(faultResponse.status);
|
|
1637
|
+
}
|
|
1638
|
+
if (isProxyAuthIpDeniedMessage(faultBody)) {
|
|
584
1639
|
throw createProxyAuthIpDeniedError();
|
|
585
1640
|
}
|
|
586
|
-
if (
|
|
1641
|
+
if (isProxyEdgeAuthRejectedMessage(faultBody)) {
|
|
587
1642
|
throw createProxyEdgeAuthRejectedError();
|
|
588
1643
|
}
|
|
589
|
-
if (
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
throw createProxyPoolStaleError(response.status);
|
|
1644
|
+
if (isProxyPoolStaleStatus(faultResponse.status) &&
|
|
1645
|
+
isProxyPoolStaleMessage(faultBody)) {
|
|
1646
|
+
throw createProxyPoolStaleError(faultResponse.status);
|
|
593
1647
|
}
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
1648
|
+
};
|
|
1649
|
+
let { normalized, response } = await fetchOnBoundSession(requestUrl, method, options, clientOptions.signal);
|
|
1650
|
+
throwProxyTransportFault(response, normalized.body);
|
|
1651
|
+
// Commit only after the selected endpoint reached the origin. A failed
|
|
1652
|
+
// proxy attempt may rotate pool index; the successful exact endpoint is
|
|
1653
|
+
// the one sealed into the ceremony handle.
|
|
1654
|
+
bindCeremonyEgress(attemptProxy);
|
|
1655
|
+
const akamaiSbsd = clientOptions.stealth?.challengeRuntime?.akamaiSbsd;
|
|
1656
|
+
const detected = akamaiSbsd
|
|
1657
|
+
? detectAkamaiSbsdChallenge(normalized, requestUrl, cookieJar, akamaiSbsd.allowedHosts, akamaiSbsdState)
|
|
1658
|
+
: undefined;
|
|
1659
|
+
if (akamaiSbsd && !detected)
|
|
1660
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSbsd({ detected: false }));
|
|
1661
|
+
if (detected && akamaiSbsd) {
|
|
1662
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSbsd("detected"));
|
|
1663
|
+
if (challengeSolveAttempted || challengeRefetchAttempted) {
|
|
1664
|
+
normalized.challenge = {
|
|
1665
|
+
challenge: detected,
|
|
1666
|
+
outcome: "challenge_persisted",
|
|
1667
|
+
};
|
|
1668
|
+
return finalizeAttempt(response, normalized);
|
|
1669
|
+
}
|
|
1670
|
+
const automaticReplayEligible = isProxyTransportRetryMethod(method, automaticChallengeRefetchPolicy, { body: requestBody, headers: options.headers });
|
|
1671
|
+
if (automaticReplayEligible && !akamaiSbsd.solve) {
|
|
1672
|
+
normalized.challenge = {
|
|
1673
|
+
challenge: detected,
|
|
1674
|
+
outcome: "resolver_unavailable",
|
|
1675
|
+
};
|
|
1676
|
+
return finalizeAttempt(response, normalized);
|
|
1677
|
+
}
|
|
1678
|
+
const emulationHeaderMap = new Map(emulationHeaders.map(([name, value]) => [name.toLowerCase(), value]));
|
|
1679
|
+
const initiatingHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1680
|
+
// Mirror what the protected request actually sent: `stealth.acceptLanguage` is
|
|
1681
|
+
// applied by the Chrome header builder only; Safari/Firefox sessions send the
|
|
1682
|
+
// emulation default.
|
|
1683
|
+
const sessionHeaders = {
|
|
1684
|
+
"User-Agent": requiredEmulationHeader(emulationHeaderMap, "user-agent"),
|
|
1685
|
+
"Accept-Language": initiatingHeaders["accept-language"] ??
|
|
1686
|
+
(chromeEmulationHeaders ? clientOptions.stealth?.acceptLanguage : undefined) ??
|
|
1687
|
+
requiredEmulationHeader(emulationHeaderMap, "accept-language"),
|
|
1688
|
+
};
|
|
1689
|
+
const resolverBuildHeaders = chromeEmulationHeaders
|
|
1690
|
+
? (currentUrl, currentMethod, currentBody, currentHeaders) => buildChromeHeaderTuples({
|
|
1691
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1692
|
+
method: currentMethod,
|
|
1693
|
+
body: currentBody,
|
|
1694
|
+
headers: currentHeaders,
|
|
1695
|
+
requestUrl: currentUrl,
|
|
1696
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1697
|
+
})
|
|
1698
|
+
: undefined;
|
|
1699
|
+
const resolverTransport = {
|
|
1700
|
+
sessionHeaders,
|
|
1701
|
+
getCookie(name, cookieUrl) {
|
|
1702
|
+
return cookieJar.get(name, cookieUrl);
|
|
1703
|
+
},
|
|
1704
|
+
async fetch(transportUrl, init) {
|
|
1705
|
+
const boundSignal = clientOptions.signal
|
|
1706
|
+
? AbortSignal.any([clientOptions.signal, init.signal])
|
|
1707
|
+
: init.signal;
|
|
1708
|
+
const transportHeaders = { ...(init.headers ?? {}) };
|
|
1709
|
+
if (resolverBuildHeaders) {
|
|
1710
|
+
for (const name of Object.keys(transportHeaders)) {
|
|
1711
|
+
if (name.toLowerCase() === "user-agent") {
|
|
1712
|
+
delete transportHeaders[name];
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
// Session defaults come from this request's own shape, never from the
|
|
1717
|
+
// initiating request: its defaults carry the upstream Cookie header,
|
|
1718
|
+
// which wreq would otherwise merge into the Hyper /ip call.
|
|
1719
|
+
const result = await fetchOnBoundSession(transportUrl, init.method, {
|
|
1720
|
+
method: init.method,
|
|
1721
|
+
headers: transportHeaders,
|
|
1722
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
1723
|
+
redirect: "manual",
|
|
1724
|
+
throwOnHttpError: false,
|
|
1725
|
+
...(init.maxBodyBytes === undefined
|
|
1726
|
+
? {}
|
|
1727
|
+
: { maxBodyBytes: init.maxBodyBytes }),
|
|
1728
|
+
}, boundSignal, resolverBuildHeaders, resolverBuildHeaders?.(transportUrl, init.method, init.body, normalizeHeaders(transportHeaders)), "resolver");
|
|
1729
|
+
return {
|
|
1730
|
+
status: result.normalized.status,
|
|
1731
|
+
headers: result.normalized.headers,
|
|
1732
|
+
body: result.normalized.body,
|
|
1733
|
+
cookies: resolverCookiesFromResponse(result.response, transportUrl),
|
|
1734
|
+
};
|
|
1735
|
+
},
|
|
1736
|
+
};
|
|
1737
|
+
// `options` is already the fetch-start snapshot; the replay gets its own copy
|
|
1738
|
+
// of the body bytes so the transport cannot alias the first send's buffer.
|
|
1739
|
+
const replayOptions = {
|
|
1740
|
+
...options,
|
|
1741
|
+
...(Buffer.isBuffer(requestBody) ? { body: Buffer.from(requestBody) } : {}),
|
|
1742
|
+
};
|
|
1743
|
+
const transactionKey = akamaiSbsdChallengeKey(detected, {
|
|
1744
|
+
wreqBrowser: mapping.browser,
|
|
1745
|
+
wreqOs: mapping.os,
|
|
1746
|
+
proxyUrl: proxy,
|
|
599
1747
|
});
|
|
1748
|
+
const challenged = { normalized, response };
|
|
1749
|
+
const solveAndReplay = async (explicitReplay) => {
|
|
1750
|
+
if (!akamaiSbsd.solve) {
|
|
1751
|
+
throw new SDKError("No resolver is available for the challenged request", {
|
|
1752
|
+
code: "RESOLVER_UNAVAILABLE",
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
// Only a solve that will actually run on this session needs the profile match.
|
|
1756
|
+
assertAkamaiSbsdClientProfile(akamaiSbsd.clientProfile, requestProfile);
|
|
1757
|
+
// A safe request that already solved this exact challenge on this identity
|
|
1758
|
+
// left the session cookies valid: the explicit replay reuses them once
|
|
1759
|
+
// instead of paying for a second solve.
|
|
1760
|
+
const reuseCompletedSuccess = explicitReplay && akamaiSbsdState.completedSuccessKey === transactionKey;
|
|
1761
|
+
if (reuseCompletedSuccess)
|
|
1762
|
+
akamaiSbsdState.completedSuccessKey = undefined;
|
|
1763
|
+
let ownsTransaction = false;
|
|
1764
|
+
if (!reuseCompletedSuccess) {
|
|
1765
|
+
let transaction = akamaiSbsdState.transactions.get(transactionKey);
|
|
1766
|
+
if (!transaction) {
|
|
1767
|
+
akamaiSbsdState.completedSuccessKey = undefined;
|
|
1768
|
+
challengeSolveAttempted = true;
|
|
1769
|
+
ownsTransaction = true;
|
|
1770
|
+
// The solve spans several round trips, so it runs under the client's ambient
|
|
1771
|
+
// signal and the resolver's own timeouts, not this fetch's per-request `timeout`.
|
|
1772
|
+
transaction = {
|
|
1773
|
+
result: akamaiSbsd
|
|
1774
|
+
.solve(detected, resolverTransport, clientOptions.signal ?? new AbortController().signal)
|
|
1775
|
+
.then(() => ({ solved: true }), (error) => ({ solved: false, error }))
|
|
1776
|
+
.finally(() => {
|
|
1777
|
+
akamaiSbsdState.transactions.delete(transactionKey);
|
|
1778
|
+
}),
|
|
1779
|
+
};
|
|
1780
|
+
akamaiSbsdState.transactions.set(transactionKey, transaction);
|
|
1781
|
+
}
|
|
1782
|
+
const transactionResult = await transaction.result;
|
|
1783
|
+
if (!transactionResult.solved) {
|
|
1784
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSbsd("solve_failed"));
|
|
1785
|
+
if (ownsTransaction) {
|
|
1786
|
+
// The proxy delivered the challenged response; the resolver failed.
|
|
1787
|
+
// Surface that failure as-is: it is not a transport fault to normalize
|
|
1788
|
+
// or retry.
|
|
1789
|
+
if (!explicitReplay) {
|
|
1790
|
+
recordProxyAttempt("ok", undefined, challenged.response.status);
|
|
1791
|
+
}
|
|
1792
|
+
challengeSolveFailure = { error: transactionResult.error };
|
|
1793
|
+
throw challengeSolveFailure;
|
|
1794
|
+
}
|
|
1795
|
+
return {
|
|
1796
|
+
normalized: {
|
|
1797
|
+
...challenged.normalized,
|
|
1798
|
+
challenge: { challenge: detected, outcome: "solve_failed" },
|
|
1799
|
+
},
|
|
1800
|
+
response: challenged.response,
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
challengeRefetchAttempted = true;
|
|
1805
|
+
// An explicit replay is a new transport exchange on the bound endpoint and
|
|
1806
|
+
// gets its own proxy-attempt record timed from here, not from the initiating
|
|
1807
|
+
// fetch (the caller's think time in between is not transport duration); the
|
|
1808
|
+
// automatic refetch stays within the initiating attempt's record.
|
|
1809
|
+
if (explicitReplay) {
|
|
1810
|
+
// The solve above may have taken seconds; if another session of this client
|
|
1811
|
+
// moved the ceremony meanwhile, the mutation must not go out on the retired
|
|
1812
|
+
// endpoint.
|
|
1813
|
+
if (currentEgressGeneration() !== attemptEgressGeneration) {
|
|
1814
|
+
throw new SDKError("The ceremony's egress binding changed while the replay was being solved", { code: "REPLAY_SESSION_MISMATCH" });
|
|
1815
|
+
}
|
|
1816
|
+
attemptRecorded = false;
|
|
1817
|
+
attemptStartedAt = Date.now();
|
|
1818
|
+
}
|
|
1819
|
+
if (!explicitReplay)
|
|
1820
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSafeRefetch());
|
|
1821
|
+
const replayed = await fetchOnBoundSession(requestUrl, method, replayOptions, clientOptions.signal);
|
|
1822
|
+
throwProxyTransportFault(replayed.response, replayed.normalized.body);
|
|
1823
|
+
const persisted = detectAkamaiSbsdChallenge(replayed.normalized, requestUrl, cookieJar, akamaiSbsd.allowedHosts, akamaiSbsdState);
|
|
1824
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordSbsd(persisted ? "challenge_persisted" : "refetch_clear"));
|
|
1825
|
+
if (persisted) {
|
|
1826
|
+
replayed.normalized.challenge = {
|
|
1827
|
+
challenge: persisted,
|
|
1828
|
+
outcome: "challenge_persisted",
|
|
1829
|
+
};
|
|
1830
|
+
}
|
|
1831
|
+
else if (ownsTransaction && !explicitReplay) {
|
|
1832
|
+
akamaiSbsdState.completedSuccessKey = transactionKey;
|
|
1833
|
+
}
|
|
1834
|
+
return replayed;
|
|
1835
|
+
};
|
|
1836
|
+
if (!automaticReplayEligible) {
|
|
1837
|
+
normalized.challenge = {
|
|
1838
|
+
challenge: detected,
|
|
1839
|
+
outcome: "replay_required",
|
|
1840
|
+
};
|
|
1841
|
+
challengedReplays.set(normalized, {
|
|
1842
|
+
consumed: false,
|
|
1843
|
+
generation: attemptEgressGeneration,
|
|
1844
|
+
async replay() {
|
|
1845
|
+
try {
|
|
1846
|
+
const replayed = await solveAndReplay(true);
|
|
1847
|
+
return finalizeAttempt(replayed.response, replayed.normalized);
|
|
1848
|
+
}
|
|
1849
|
+
catch (error) {
|
|
1850
|
+
const normalizedError = normalizeAttemptError(error);
|
|
1851
|
+
releaseFailedCeremonyEgress(proxy, normalizedError);
|
|
1852
|
+
throw recordAttemptFailure(normalizedError);
|
|
1853
|
+
}
|
|
1854
|
+
},
|
|
1855
|
+
});
|
|
1856
|
+
return finalizeAttempt(response, normalized);
|
|
1857
|
+
}
|
|
1858
|
+
({ normalized, response } = await solveAndReplay(false));
|
|
600
1859
|
}
|
|
601
|
-
|
|
602
|
-
return normalized;
|
|
1860
|
+
return finalizeAttempt(response, normalized);
|
|
603
1861
|
}
|
|
604
1862
|
catch (error) {
|
|
605
|
-
const normalizedError =
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
1863
|
+
const normalizedError = normalizeAttemptError(error);
|
|
1864
|
+
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
1865
|
+
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
1866
|
+
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
1867
|
+
const redactedError = recordAttemptFailure(normalizedError);
|
|
1868
|
+
lastError = redactedError;
|
|
1869
|
+
const releasedBoundEgress = releaseFailedCeremonyEgress(proxy, normalizedError);
|
|
1870
|
+
if (releasedBoundEgress) {
|
|
1871
|
+
// The request now proceeds as an unbound one with the full rotation budget;
|
|
1872
|
+
// the attempt spent on the dead endpoint does not count against it.
|
|
1873
|
+
maxAttempts = attempt + 1 + unboundMaxAttempts;
|
|
1874
|
+
}
|
|
1875
|
+
if (challengeSolveAttempted || challengeRefetchAttempted) {
|
|
1876
|
+
// The single refetch is spent (owner or waiter of a shared solve): another
|
|
1877
|
+
// transport attempt would replay the original request and may rotate the
|
|
1878
|
+
// proxy the solved jar is bound to.
|
|
1879
|
+
throw redactedError;
|
|
1880
|
+
}
|
|
1881
|
+
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
1882
|
+
// Also the path a released bound endpoint takes after a proxy-level refusal:
|
|
1883
|
+
// the stale-pool bookkeeping here is what drives allocator invalidation and the
|
|
1884
|
+
// refresh pass when the fresh selection lands on the same endpoint again.
|
|
1885
|
+
stalePoolError = redactedError;
|
|
1886
|
+
if (runProxyAuthDiagnostic) {
|
|
611
1887
|
stalePoolDiagnosticProxy = proxy;
|
|
612
1888
|
}
|
|
613
1889
|
if (attempt + 1 < maxAttempts) {
|
|
@@ -615,37 +1891,67 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
615
1891
|
}
|
|
616
1892
|
break;
|
|
617
1893
|
}
|
|
618
|
-
if (
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
1894
|
+
if (releasedBoundEgress &&
|
|
1895
|
+
attempt + 1 < maxAttempts &&
|
|
1896
|
+
retryErrorCode === PROXY_CONNECT_FAILURE_CODE) {
|
|
1897
|
+
// The tunnel to the released endpoint never opened, so the origin never saw the
|
|
1898
|
+
// request: selecting and binding a fresh endpoint in this request is safe for
|
|
1899
|
+
// any method. Ambiguous failures (reset, timeout) fall through to the ordinary
|
|
1900
|
+
// method-aware retry decision below with the binding already released.
|
|
1901
|
+
continue;
|
|
1902
|
+
}
|
|
1903
|
+
// Cap the number of transport retries. For a policy-allocator chain,
|
|
1904
|
+
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
1905
|
+
// rotates across the concatenated vendor pool spans), so a transport
|
|
1906
|
+
// failure is a signal to advance to the next endpoint — potentially
|
|
1907
|
+
// crossing into the fallback vendor — not to retry the same endpoint.
|
|
1908
|
+
// Truncating that rotation at the per-endpoint retry budget would
|
|
1909
|
+
// strand the request on the primary vendor and never reach the
|
|
1910
|
+
// fallback, since the crossover only happens once the flat attempt
|
|
1911
|
+
// index exceeds the primary vendor's pool size (~10-20).
|
|
1912
|
+
// resolvePolicyTransportAttemptCap widens to the full chain span only
|
|
1913
|
+
// for implicit, safe-method allocator requests; explicit retry
|
|
1914
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
1915
|
+
// static/non-registry vendors keep the per-endpoint retry budget.
|
|
1916
|
+
const transportRetryCap = resolvePolicyTransportAttemptCap({
|
|
1917
|
+
policy: policyProxy,
|
|
1918
|
+
usesPolicyAllocator,
|
|
1919
|
+
retryAttempts: stealthRetryOptions?.attempts ?? 1,
|
|
1920
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
1921
|
+
method,
|
|
1922
|
+
});
|
|
1923
|
+
if (attempt + 1 < transportRetryCap &&
|
|
622
1924
|
shouldRetryProxyTransportAttempt({
|
|
623
|
-
error:
|
|
1925
|
+
error: { code: retryErrorCode },
|
|
624
1926
|
explicitRetry: hasExplicitRetryPolicy,
|
|
625
1927
|
method,
|
|
626
1928
|
options: stealthRetryOptions,
|
|
627
1929
|
proxyUsed: Boolean(proxy),
|
|
628
1930
|
})) {
|
|
629
1931
|
if (stealthRetryOptions) {
|
|
630
|
-
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1));
|
|
1932
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1), clientOptions.signal);
|
|
631
1933
|
}
|
|
1934
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
632
1935
|
continue;
|
|
633
1936
|
}
|
|
634
|
-
throw
|
|
1937
|
+
throw redactedError;
|
|
635
1938
|
}
|
|
636
1939
|
}
|
|
637
|
-
if (
|
|
1940
|
+
if (rotatesRegistryChain &&
|
|
638
1941
|
stalePoolError &&
|
|
639
1942
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES) {
|
|
1943
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1944
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordPoolRefresh());
|
|
640
1945
|
await invalidateProxyResolutionCacheAsync({
|
|
641
1946
|
proxyPolicy: clientOptions.proxyPolicy,
|
|
642
1947
|
upstream: clientOptions.upstream,
|
|
643
1948
|
affinityKey: clientOptions.affinityKey,
|
|
644
1949
|
});
|
|
1950
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
645
1951
|
continue;
|
|
646
1952
|
}
|
|
647
1953
|
const proxyAuthDiagnostic = stalePoolError && stalePoolDiagnosticProxy
|
|
648
|
-
? await classifyProxyAuthDiagnostic(
|
|
1954
|
+
? await classifyProxyAuthDiagnostic(requestProfile, stalePoolDiagnosticProxy)
|
|
649
1955
|
: undefined;
|
|
650
1956
|
if (proxyAuthDiagnostic === "source_ip_denied") {
|
|
651
1957
|
throw createProxyAuthIpDeniedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
@@ -662,89 +1968,198 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
662
1968
|
}
|
|
663
1969
|
break;
|
|
664
1970
|
}
|
|
1971
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
665
1972
|
throw normalizeStealthTransportError(lastError);
|
|
666
1973
|
},
|
|
1974
|
+
async replayChallenged(challengedResponse) {
|
|
1975
|
+
// Lease expiry is evaluated when a fetch starts, not here: the replay belongs to the
|
|
1976
|
+
// exchange that was challenged on the bound endpoint (Akamai state is IP-bound), so a
|
|
1977
|
+
// lifetime boundary crossed during the solve does not move it elsewhere. The next
|
|
1978
|
+
// fetch drops the expired binding and rebinds.
|
|
1979
|
+
const replay = challengedReplays.get(challengedResponse);
|
|
1980
|
+
if (!replay || challengedResponse.challenge?.outcome !== "replay_required") {
|
|
1981
|
+
throw new SDKError("The challenged response does not belong to this stealth session", {
|
|
1982
|
+
code: "REPLAY_SESSION_MISMATCH",
|
|
1983
|
+
});
|
|
1984
|
+
}
|
|
1985
|
+
syncCeremonyEgress();
|
|
1986
|
+
if (replay.generation !== currentEgressGeneration()) {
|
|
1987
|
+
// The ceremony rebound to another egress after this request ran (in this session or
|
|
1988
|
+
// another of the same client); solving and replaying on the retired endpoint would
|
|
1989
|
+
// split the identity the handle names.
|
|
1990
|
+
throw new SDKError("The challenged response predates the ceremony's current egress binding", { code: "REPLAY_SESSION_MISMATCH" });
|
|
1991
|
+
}
|
|
1992
|
+
if (replay.consumed) {
|
|
1993
|
+
throw new SDKError("The challenged request has already used its one replay budget", {
|
|
1994
|
+
code: "REPLAY_ALREADY_ATTEMPTED",
|
|
1995
|
+
});
|
|
1996
|
+
}
|
|
1997
|
+
replay.consumed = true;
|
|
1998
|
+
return replay.replay();
|
|
1999
|
+
},
|
|
667
2000
|
cookies: cookieJar,
|
|
668
2001
|
redirects: {
|
|
669
2002
|
async run(options) {
|
|
670
2003
|
const maxHops = options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
671
2004
|
? 10
|
|
672
2005
|
: Math.max(0, Math.floor(options.maxHops));
|
|
2006
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, sensitiveParams, headers: initialHeaders, ...fetchOptions } = options;
|
|
673
2007
|
const hops = [];
|
|
674
|
-
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
675
2008
|
let method = normalizeMethod(options.method ?? "GET");
|
|
676
2009
|
let body = options.body;
|
|
2010
|
+
let headers = initialHeaders;
|
|
677
2011
|
let response;
|
|
678
2012
|
const visitedRequests = new Set();
|
|
679
|
-
const
|
|
2013
|
+
const initialParams = params
|
|
2014
|
+
? Object.fromEntries(Object.entries(params).map(([key, value]) => [
|
|
2015
|
+
key,
|
|
2016
|
+
Array.isArray(value) ? [...value] : value,
|
|
2017
|
+
]))
|
|
2018
|
+
: undefined;
|
|
2019
|
+
const normalizedSensitiveParams = normalizeSensitiveParams(sensitiveParams);
|
|
2020
|
+
const initialSensitiveParams = normalizedSensitiveParams
|
|
2021
|
+
? { ...normalizedSensitiveParams }
|
|
2022
|
+
: undefined;
|
|
2023
|
+
const sensitiveParamNames = initialSensitiveParams
|
|
2024
|
+
? Object.keys(initialSensitiveParams)
|
|
2025
|
+
: [];
|
|
2026
|
+
const callerStructural = redactUrlQueryParams(options.url, sensitiveParamNames);
|
|
2027
|
+
const sensitiveValues = new Set([
|
|
2028
|
+
...Object.values(initialSensitiveParams ?? {}),
|
|
2029
|
+
...callerStructural.sensitiveValues,
|
|
2030
|
+
].filter((value) => value !== ""));
|
|
2031
|
+
const redactRedirectUrl = (value) => {
|
|
2032
|
+
const structural = redactUrlQueryParams(value, [
|
|
2033
|
+
...new Set([...sensitiveParamNames, ...sensitiveQueryParamNames(value)]),
|
|
2034
|
+
]);
|
|
2035
|
+
for (const sensitiveValue of structural.sensitiveValues) {
|
|
2036
|
+
sensitiveValues.add(sensitiveValue);
|
|
2037
|
+
}
|
|
2038
|
+
return redactSensitiveText(structural.redactedUrl, [...sensitiveValues]);
|
|
2039
|
+
};
|
|
2040
|
+
let currentUrl;
|
|
2041
|
+
let initialUrl;
|
|
2042
|
+
try {
|
|
2043
|
+
currentUrl = resolveUrl(baseUrl, options.url);
|
|
2044
|
+
redactRedirectUrl(currentUrl);
|
|
2045
|
+
initialUrl = serializeRequestUrl(currentUrl, initialParams, initialSensitiveParams);
|
|
2046
|
+
for (const value of initialUrl.sensitiveValues) {
|
|
2047
|
+
if (value !== "")
|
|
2048
|
+
sensitiveValues.add(value);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
catch (error) {
|
|
2052
|
+
throw redactSensitiveError(error, [...sensitiveValues], options.url, redactRedirectUrl(options.url));
|
|
2053
|
+
}
|
|
680
2054
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
2055
|
+
const outboundUrl = hopIndex === 0 ? initialUrl.requestUrl : serializeRequestUrl(currentUrl).requestUrl;
|
|
2056
|
+
// Preserve params-only loop bookkeeping from before sensitiveParams:
|
|
2057
|
+
// the first visited key is the caller's resolved URL, not its expanded query.
|
|
2058
|
+
const visitedUrl = hopIndex === 0 && !initialSensitiveParams ? currentUrl : outboundUrl;
|
|
2059
|
+
visitedRequests.add(`${method} ${visitedUrl}`);
|
|
2060
|
+
try {
|
|
2061
|
+
response = await session.fetch(currentUrl, {
|
|
2062
|
+
...fetchOptions,
|
|
2063
|
+
...(headers ? { headers } : {}),
|
|
2064
|
+
body,
|
|
2065
|
+
method,
|
|
2066
|
+
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
2067
|
+
...(hopIndex === 0 && initialSensitiveParams
|
|
2068
|
+
? { sensitiveParams: initialSensitiveParams }
|
|
2069
|
+
: {}),
|
|
2070
|
+
redirect: "manual",
|
|
2071
|
+
throwOnHttpError: false,
|
|
2072
|
+
});
|
|
2073
|
+
}
|
|
2074
|
+
catch (error) {
|
|
2075
|
+
throw redactSensitiveError(error, [...sensitiveValues], outboundUrl, redactRedirectUrl(outboundUrl));
|
|
2076
|
+
}
|
|
2077
|
+
// StealthResponse.url is programmatic metadata and remains raw. Only the
|
|
2078
|
+
// redirect hop emitted below is a diagnostic surface.
|
|
2079
|
+
const responseUrl = response.url ?? (hopIndex === 0 && initialSensitiveParams ? outboundUrl : currentUrl);
|
|
690
2080
|
if (!isRedirectStatus(response.status)) {
|
|
691
2081
|
return {
|
|
692
2082
|
final: response,
|
|
693
2083
|
hops,
|
|
694
2084
|
reason: "completed",
|
|
695
2085
|
cookies: cookieJar.snapshot(),
|
|
2086
|
+
cookieStore: cookieJar.serialize(),
|
|
696
2087
|
};
|
|
697
2088
|
}
|
|
698
2089
|
const location = locationHeader(response.headers);
|
|
699
|
-
const
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
2090
|
+
const redactedResponseUrl = redactRedirectUrl(responseUrl);
|
|
2091
|
+
const redactedLocation = location ? redactRedirectUrl(location) : undefined;
|
|
2092
|
+
let nextUrl;
|
|
2093
|
+
try {
|
|
2094
|
+
nextUrl = resolveRedirectUrl(location, responseUrl);
|
|
2095
|
+
}
|
|
2096
|
+
catch (error) {
|
|
2097
|
+
throw redactSensitiveError(error, [...sensitiveValues], location, redactedLocation);
|
|
2098
|
+
}
|
|
2099
|
+
const realHop = {
|
|
2100
|
+
url: responseUrl,
|
|
704
2101
|
status: response.status,
|
|
705
2102
|
method,
|
|
706
2103
|
...(location ? { location } : {}),
|
|
707
2104
|
...(nextUrl ? { nextUrl } : {}),
|
|
708
2105
|
};
|
|
2106
|
+
const hop = {
|
|
2107
|
+
...realHop,
|
|
2108
|
+
url: redactedResponseUrl,
|
|
2109
|
+
...(redactedLocation ? { location: redactedLocation } : {}),
|
|
2110
|
+
...(nextUrl ? { nextUrl: redactRedirectUrl(nextUrl) } : {}),
|
|
2111
|
+
};
|
|
709
2112
|
hops.push(hop);
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
2113
|
+
let shouldStop = false;
|
|
2114
|
+
if (stopWhen) {
|
|
2115
|
+
try {
|
|
2116
|
+
shouldStop = await stopWhen(realHop);
|
|
2117
|
+
}
|
|
2118
|
+
catch (error) {
|
|
2119
|
+
let sanitizedError = error;
|
|
2120
|
+
for (const [rawUrl, safeUrl] of [
|
|
2121
|
+
[responseUrl, redactedResponseUrl],
|
|
2122
|
+
[location, redactedLocation],
|
|
2123
|
+
[nextUrl, nextUrl ? redactRedirectUrl(nextUrl) : undefined],
|
|
2124
|
+
]) {
|
|
2125
|
+
if (!rawUrl || !safeUrl)
|
|
2126
|
+
continue;
|
|
2127
|
+
sanitizedError = redactSensitiveError(sanitizedError, [...sensitiveValues], rawUrl, safeUrl);
|
|
2128
|
+
}
|
|
2129
|
+
throw sanitizedError;
|
|
2130
|
+
}
|
|
717
2131
|
}
|
|
718
|
-
|
|
2132
|
+
const decision = evaluateRedirectHop({
|
|
2133
|
+
status: response.status,
|
|
2134
|
+
method,
|
|
2135
|
+
nextUrl,
|
|
2136
|
+
shouldStop,
|
|
2137
|
+
redirectCount: hops.length,
|
|
2138
|
+
maxHops,
|
|
2139
|
+
visitedRequests,
|
|
2140
|
+
});
|
|
2141
|
+
if (decision.kind === "stop") {
|
|
719
2142
|
return {
|
|
720
2143
|
final: response,
|
|
721
2144
|
hops,
|
|
722
|
-
reason:
|
|
2145
|
+
reason: decision.reason,
|
|
723
2146
|
cookies: cookieJar.snapshot(),
|
|
2147
|
+
cookieStore: cookieJar.serialize(),
|
|
724
2148
|
};
|
|
725
2149
|
}
|
|
726
|
-
if (
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
};
|
|
2150
|
+
if (headers) {
|
|
2151
|
+
headers = redirectHopHeaders({
|
|
2152
|
+
from: currentUrl,
|
|
2153
|
+
to: decision.nextUrl,
|
|
2154
|
+
methodChanged: decision.nextMethod !== method,
|
|
2155
|
+
}, headers);
|
|
733
2156
|
}
|
|
734
|
-
|
|
735
|
-
if (nextMethod !== method) {
|
|
2157
|
+
if (decision.nextMethod !== method) {
|
|
736
2158
|
body = undefined;
|
|
737
2159
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
hops,
|
|
742
|
-
reason: "loop",
|
|
743
|
-
cookies: cookieJar.snapshot(),
|
|
744
|
-
};
|
|
745
|
-
}
|
|
746
|
-
method = nextMethod;
|
|
747
|
-
currentUrl = nextUrl;
|
|
2160
|
+
emitStealthTelemetry(clientOptions.stealthTelemetry, (sink) => sink.recordRedirectHop());
|
|
2161
|
+
method = decision.nextMethod;
|
|
2162
|
+
currentUrl = decision.nextUrl;
|
|
748
2163
|
}
|
|
749
2164
|
if (!response) {
|
|
750
2165
|
response = await session.fetch(currentUrl, {
|
|
@@ -761,25 +2176,63 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
761
2176
|
hops,
|
|
762
2177
|
reason: "max_hops",
|
|
763
2178
|
cookies: cookieJar.snapshot(),
|
|
2179
|
+
cookieStore: cookieJar.serialize(),
|
|
764
2180
|
};
|
|
765
2181
|
},
|
|
766
2182
|
},
|
|
767
2183
|
close() {
|
|
768
2184
|
closed = true;
|
|
2185
|
+
clearAkamaiSbsdState();
|
|
2186
|
+
for (const client of clients.values()) {
|
|
2187
|
+
void client.session
|
|
2188
|
+
.then((session) => session.close())
|
|
2189
|
+
.catch((error) => {
|
|
2190
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2191
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
769
2194
|
clients.clear();
|
|
770
2195
|
},
|
|
771
2196
|
};
|
|
772
2197
|
return session;
|
|
773
|
-
async function classifyProxyAuthDiagnostic(
|
|
2198
|
+
async function classifyProxyAuthDiagnostic(profile, proxy) {
|
|
774
2199
|
try {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
2200
|
+
return await withClient(profile, proxy, false, async (client) => {
|
|
2201
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
2202
|
+
await client.clearCookies();
|
|
2203
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
2204
|
+
const finishExchange = startStealthExchange(clientOptions.stealthTelemetry
|
|
2205
|
+
? {
|
|
2206
|
+
sink: clientOptions.stealthTelemetry,
|
|
2207
|
+
profileId: profile,
|
|
2208
|
+
proxyUsed: true,
|
|
2209
|
+
kind: "proxy_diagnostic",
|
|
2210
|
+
cookieJar,
|
|
2211
|
+
}
|
|
2212
|
+
: undefined, "GET", {}, clientOptions.signal);
|
|
2213
|
+
let status;
|
|
2214
|
+
let exchangeError;
|
|
2215
|
+
try {
|
|
2216
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
2217
|
+
method: "GET",
|
|
2218
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
2219
|
+
...(clientOptions.signal ? { signal: clientOptions.signal } : {}),
|
|
2220
|
+
});
|
|
2221
|
+
status = response.status;
|
|
2222
|
+
const normalized = await normalizeResponseWithSignal(response, undefined, undefined, clientOptions.signal);
|
|
2223
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
2224
|
+
}
|
|
2225
|
+
catch (error) {
|
|
2226
|
+
exchangeError = error;
|
|
2227
|
+
throw error;
|
|
2228
|
+
}
|
|
2229
|
+
finally {
|
|
2230
|
+
finishExchange(status, exchangeError);
|
|
2231
|
+
}
|
|
2232
|
+
}, clientOptions.signal);
|
|
781
2233
|
}
|
|
782
2234
|
catch (error) {
|
|
2235
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
783
2236
|
const message = error instanceof Error
|
|
784
2237
|
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
785
2238
|
.filter(Boolean)
|
|
@@ -808,13 +2261,15 @@ function hasHeader(headers, name) {
|
|
|
808
2261
|
const needle = name.toLowerCase();
|
|
809
2262
|
return Object.keys(headers).some((key) => key.toLowerCase() === needle);
|
|
810
2263
|
}
|
|
811
|
-
export function createStealthClient(baseUrl,
|
|
812
|
-
|
|
813
|
-
|
|
2264
|
+
export function createStealthClient(baseUrl, clientOptions = {}) {
|
|
2265
|
+
if (typeof clientOptions === "string") {
|
|
2266
|
+
resolveStealthProfileSelection(clientOptions);
|
|
2267
|
+
}
|
|
2268
|
+
const defaultProfile = resolveStealthProfileSelection(clientOptions.stealth);
|
|
814
2269
|
let sharedSession = null;
|
|
815
2270
|
function getSharedSession() {
|
|
816
2271
|
if (!sharedSession) {
|
|
817
|
-
sharedSession = createSessionFetcher(baseUrl, defaultProfile,
|
|
2272
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, clientOptions);
|
|
818
2273
|
}
|
|
819
2274
|
return sharedSession;
|
|
820
2275
|
}
|
|
@@ -823,8 +2278,8 @@ export function createStealthClient(baseUrl, defaultProfileOrOptions = DEFAULT_P
|
|
|
823
2278
|
return getSharedSession().fetch(url, options);
|
|
824
2279
|
},
|
|
825
2280
|
createSession(opts) {
|
|
826
|
-
const sessionProfile = opts?.
|
|
827
|
-
return createSessionFetcher(baseUrl, sessionProfile,
|
|
2281
|
+
const sessionProfile = resolveStealthProfileSelection(opts?.stealth, defaultProfile);
|
|
2282
|
+
return createSessionFetcher(baseUrl, sessionProfile, clientOptions);
|
|
828
2283
|
},
|
|
829
2284
|
close() {
|
|
830
2285
|
sharedSession?.close();
|