@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +537 -33
- package/CHANGELOG.md +205 -1
- package/README.md +83 -22
- package/SUBMISSION.md +2 -2
- package/bin/apifuse-check.ts +165 -11
- package/bin/apifuse-dev.ts +57 -13
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-migrate-shape.ts +202 -0
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +357 -38
- package/bin/apifuse-perf.ts +14 -13
- package/bin/apifuse-record.ts +709 -72
- package/bin/apifuse-submit-check.ts +2344 -324
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +27 -0
- package/dist/cli/create.d.ts +4 -1
- package/dist/cli/create.js +42 -37
- package/dist/cli/migrate-operation-declaration.d.ts +59 -0
- package/dist/cli/migrate-operation-declaration.js +1249 -0
- package/dist/cli/migrate-operation-shape.d.ts +44 -0
- package/dist/cli/migrate-operation-shape.js +113 -0
- package/dist/cli/migrate-provider-shape.d.ts +52 -0
- package/dist/cli/migrate-provider-shape.js +578 -0
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +5 -5
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/dist/cli/templates/provider/provider.json.tpl +6 -0
- package/dist/config/loader.d.ts +179 -16
- package/dist/config/loader.js +435 -127
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +12 -5
- package/dist/contract.js +24 -10
- package/dist/declaration-validation.d.ts +33 -0
- package/dist/declaration-validation.js +266 -0
- package/dist/define.d.ts +66 -29
- package/dist/define.js +778 -69
- package/dist/dev.d.ts +3 -0
- package/dist/dev.js +1 -1
- package/dist/engine.d.ts +78 -0
- package/dist/engine.js +133 -0
- package/dist/error-observability.d.ts +7 -0
- package/dist/error-observability.js +61 -0
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +122 -0
- package/dist/errors.d.ts +33 -0
- package/dist/errors.js +40 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +227 -0
- package/dist/health-scenario.d.ts +1842 -0
- package/dist/health-scenario.js +624 -0
- package/dist/index.d.ts +21 -10
- package/dist/index.js +12 -7
- package/dist/lint.d.ts +23 -11
- package/dist/lint.js +572 -69
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +10 -2
- package/dist/provider.js +4 -1
- package/dist/runtime/auth-flow.d.ts +5 -1
- package/dist/runtime/auth-flow.js +6 -0
- package/dist/runtime/browser.d.ts +1 -0
- package/dist/runtime/browser.js +492 -49
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +13 -1
- package/dist/runtime/choice.js +490 -102
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/env.js +12 -0
- package/dist/runtime/executor.d.ts +3 -2
- package/dist/runtime/executor.js +26 -25
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +515 -53
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +366 -8
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +397 -16
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-telemetry.d.ts +80 -1
- package/dist/runtime/proxy-telemetry.js +154 -47
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +14 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +22 -0
- package/dist/runtime/resolver-vendors/browser.js +377 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +531 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +408 -0
- package/dist/runtime/resolver-vendors/types.d.ts +94 -0
- package/dist/runtime/resolver-vendors/types.js +96 -0
- package/dist/runtime/resolver.d.ts +72 -0
- package/dist/runtime/resolver.js +771 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +277 -71
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +41 -5
- package/dist/runtime/stealth.js +997 -291
- package/dist/runtime/stt.js +1 -12
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +62 -0
- package/dist/runtime/trace.d.ts +5 -0
- package/dist/runtime/trace.js +43 -10
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/error-observability.d.ts +1 -0
- package/dist/server/error-observability.js +1 -0
- package/dist/server/index.d.ts +5 -2
- package/dist/server/index.js +2 -2
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test.d.ts +14 -3
- package/dist/server/self-test.js +129 -58
- package/dist/server/serve-implementation.d.ts +230 -0
- package/dist/server/serve-implementation.js +2265 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1131
- package/dist/server/trace-output.d.ts +6 -0
- package/dist/server/trace-output.js +79 -0
- package/dist/server/types.d.ts +30 -5
- package/dist/server/types.js +13 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +9 -4
- package/dist/stealth/profiles.js +160 -211
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +489 -21
- package/dist/trace-sanitization.d.ts +5 -0
- package/dist/trace-sanitization.js +45 -0
- package/dist/types.d.ts +673 -131
- package/dist/types.js +1 -0
- package/package.json +45 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/approval-override.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/codemod-syntax.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/connection-precedence.ts.txt +10 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/docs-conflict.ts.txt +8 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-hoisted-array-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-imported-array-spread.ts.txt +9 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-map.ts.txt +5 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-mixed-array-spread.ts.txt +24 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-operation.ts.txt +16 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/factory-map.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoist-all.ts.txt +31 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoisted-const.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/imported-spread.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-map.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-cast-tail.ts.txt +21 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-ekitan.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-override.ts.txt +14 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/missing-english-locale.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/no-safety.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/non-literal.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/redundant-approval.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/safety-conflict.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/stream.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/tool-router-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/unparseable.ts.txt +4 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/verbatim-template.ts.txt +12 -0
- package/src/cli/commands.ts +33 -0
- package/src/cli/create.ts +50 -37
- package/src/cli/migrate-operation-declaration.ts +1751 -0
- package/src/cli/migrate-operation-shape.ts +184 -0
- package/src/cli/migrate-provider-shape.ts +772 -0
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +5 -5
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/src/cli/templates/provider/provider.json.tpl +6 -0
- package/src/config/loader.ts +690 -163
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +12 -5
- package/src/contract.ts +24 -10
- package/src/declaration-validation.ts +330 -0
- package/src/define.ts +1048 -131
- package/src/dev.ts +4 -1
- package/src/engine.ts +279 -0
- package/src/error-observability.ts +64 -0
- package/src/error-resolution.ts +127 -0
- package/src/errors.ts +68 -0
- package/src/fixture-sanitization.ts +264 -0
- package/src/health-scenario.ts +875 -0
- package/src/index.ts +220 -13
- package/src/lint.ts +698 -90
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +161 -3
- package/src/runtime/auth-flow.ts +12 -0
- package/src/runtime/browser.ts +661 -63
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +631 -120
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/env.ts +13 -0
- package/src/runtime/executor.ts +43 -31
- package/src/runtime/http.ts +641 -61
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/instrumentation.ts +520 -15
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/otlp.ts +467 -21
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-telemetry.ts +244 -75
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +23 -0
- package/src/runtime/resolver-shared.ts +33 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +533 -0
- package/src/runtime/resolver-vendors/capsolver.ts +707 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +540 -0
- package/src/runtime/resolver-vendors/types.ts +212 -0
- package/src/runtime/resolver.ts +1157 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +394 -77
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +1280 -336
- package/src/runtime/stt.ts +1 -19
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +78 -0
- package/src/runtime/trace.ts +57 -17
- package/src/serve.ts +6 -1
- package/src/server/error-observability.ts +1 -0
- package/src/server/index.ts +34 -2
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test.ts +191 -68
- package/src/server/serve-implementation.ts +3378 -0
- package/src/server/serve.ts +1 -1632
- package/src/server/trace-output.ts +129 -0
- package/src/server/types.ts +13 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +202 -231
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +658 -15
- package/src/trace-sanitization.ts +63 -0
- package/src/types.ts +788 -174
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,191 +1,407 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, resolveProxyConfigAsync, SMARTPROXY_MAX_POOL_SIZE, } from "../config/loader.js";
|
|
2
|
+
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, policyResolvesRegistryVendorChain, resolvePolicyProxyPoolSpan, resolvePolicyTransportAttemptCap, resolveProxyConfigAsync, vendorFromResolvedSource, } from "../config/loader.js";
|
|
4
3
|
import { SDKError, TransportError } from "../errors.js";
|
|
5
|
-
import { getStealthProfile } from "../stealth/profiles.js";
|
|
4
|
+
import { DEFAULT_STEALTH_BROWSER, DEFAULT_STEALTH_OS, getStealthProfile, resolveStealthProfileSelection, } from "../stealth/profiles.js";
|
|
5
|
+
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
6
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
7
7
|
import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefaultProxyTransportRetryOptions, normalizeProxyTransportRetryOptions, shouldRetryProxyTransportAttempt, validateUnsafeProxyTransportRetryMethods, } from "./proxy-retry-policy.js";
|
|
8
|
-
import {
|
|
9
|
-
|
|
8
|
+
import { evaluateRedirectHop, isRedirectStatus, nextRedirectMethod, resolveRedirectUrl, } from "./redirects.js";
|
|
9
|
+
import { isSensitiveKey, normalizeSensitiveParams, redactSensitiveError, redactSensitiveRequestError, redactSensitiveText, redactUrlQueryParams, serializeRequestUrl, } from "./request-options.js";
|
|
10
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
11
|
+
export const DEFAULT_STEALTH_PROFILE = Object.freeze({
|
|
12
|
+
browser: DEFAULT_STEALTH_BROWSER,
|
|
13
|
+
os: DEFAULT_STEALTH_OS,
|
|
14
|
+
});
|
|
10
15
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
11
|
-
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
12
16
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
17
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
14
18
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
15
19
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
16
20
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
17
21
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE];
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"chrome-133",
|
|
25
|
-
"chrome-144",
|
|
26
|
-
"chrome-146-psk",
|
|
27
|
-
"chrome-131-psk",
|
|
28
|
-
"chrome-130-psk",
|
|
29
|
-
"edge-131",
|
|
22
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
23
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
24
|
+
"content-encoding",
|
|
25
|
+
"content-language",
|
|
26
|
+
"content-location",
|
|
27
|
+
"content-type",
|
|
30
28
|
]);
|
|
31
|
-
const CHROME_IMPIT_BY_MAJOR = {
|
|
32
|
-
100: "chrome100",
|
|
33
|
-
101: "chrome101",
|
|
34
|
-
104: "chrome104",
|
|
35
|
-
107: "chrome107",
|
|
36
|
-
110: "chrome110",
|
|
37
|
-
116: "chrome116",
|
|
38
|
-
124: "chrome124",
|
|
39
|
-
125: "chrome125",
|
|
40
|
-
131: "chrome131",
|
|
41
|
-
136: "chrome136",
|
|
42
|
-
142: "chrome142",
|
|
43
|
-
};
|
|
44
|
-
const FIREFOX_IMPIT_BY_MAJOR = {
|
|
45
|
-
128: "firefox128",
|
|
46
|
-
133: "firefox133",
|
|
47
|
-
135: "firefox135",
|
|
48
|
-
144: "firefox144",
|
|
49
|
-
};
|
|
50
29
|
function isRecord(value) {
|
|
51
30
|
return typeof value === "object" && value !== null;
|
|
52
31
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const value = nameValue.slice(separatorIndex + 1).trim();
|
|
71
|
-
if (name)
|
|
72
|
-
this.cookies[name] = value;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
get(name) {
|
|
76
|
-
return this.cookies[name];
|
|
77
|
-
}
|
|
78
|
-
getAll() {
|
|
79
|
-
return { ...this.cookies };
|
|
80
|
-
}
|
|
81
|
-
has(name) {
|
|
82
|
-
return Object.hasOwn(this.cookies, name);
|
|
83
|
-
}
|
|
84
|
-
toString() {
|
|
85
|
-
return Object.entries(this.cookies)
|
|
86
|
-
.map(([name, value]) => `${name}=${value}`)
|
|
87
|
-
.join("; ");
|
|
88
|
-
}
|
|
89
|
-
toHeader() {
|
|
90
|
-
return this.toString();
|
|
32
|
+
function sensitiveQueryParamNames(url) {
|
|
33
|
+
const queryStart = url.indexOf("?");
|
|
34
|
+
if (queryStart === -1)
|
|
35
|
+
return [];
|
|
36
|
+
const fragmentStart = url.indexOf("#", queryStart);
|
|
37
|
+
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
38
|
+
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
39
|
+
}
|
|
40
|
+
let wreqModulePromise;
|
|
41
|
+
function getWreqModule() {
|
|
42
|
+
if (!wreqModulePromise) {
|
|
43
|
+
wreqModulePromise = import("wreq-js").catch((error) => {
|
|
44
|
+
throw new SDKError(`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`, {
|
|
45
|
+
code: "stealth_transport_unavailable",
|
|
46
|
+
cause: error instanceof Error ? error : undefined,
|
|
47
|
+
});
|
|
48
|
+
});
|
|
91
49
|
}
|
|
92
|
-
|
|
93
|
-
|
|
50
|
+
return wreqModulePromise;
|
|
51
|
+
}
|
|
52
|
+
function parseProfileIdentifier(identifier) {
|
|
53
|
+
const match = /^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(identifier.toLowerCase());
|
|
54
|
+
if (!match?.[1] || !match[2])
|
|
55
|
+
return null;
|
|
56
|
+
return {
|
|
57
|
+
family: match[1],
|
|
58
|
+
version: match[2].split(/[._]/).map(Number),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
function compareVersionDistance(target, left, right) {
|
|
62
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
63
|
+
for (let index = 0; index < width; index += 1) {
|
|
64
|
+
const targetPart = target[index] ?? 0;
|
|
65
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
66
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
67
|
+
if (leftDistance !== rightDistance)
|
|
68
|
+
return leftDistance - rightDistance;
|
|
94
69
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
72
|
+
function closestWreqProfile(identifier, wreqProfiles) {
|
|
73
|
+
const requested = parseProfileIdentifier(identifier);
|
|
74
|
+
if (!requested)
|
|
75
|
+
return undefined;
|
|
76
|
+
let closest;
|
|
77
|
+
for (const candidateName of wreqProfiles) {
|
|
78
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
79
|
+
if (!candidate || candidate.family !== requested.family)
|
|
80
|
+
continue;
|
|
81
|
+
if (!closest ||
|
|
82
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0) {
|
|
83
|
+
closest = { name: candidateName, version: candidate.version };
|
|
100
84
|
}
|
|
101
85
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
86
|
+
return closest?.name;
|
|
87
|
+
}
|
|
88
|
+
export function resolveWreqProfile(selection, wreqProfiles) {
|
|
89
|
+
const profile = getStealthProfile(selection);
|
|
90
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
91
|
+
const os = profile.os;
|
|
92
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
93
|
+
if (!browser) {
|
|
94
|
+
throw new SDKError(`Stealth profile ${profile.browser}/${profile.os} cannot be mapped to a wreq-js browser profile.`);
|
|
106
95
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
96
|
+
return { browser, os };
|
|
97
|
+
}
|
|
98
|
+
function resolveUrl(baseUrl, url) {
|
|
99
|
+
return new URL(url, baseUrl).toString();
|
|
100
|
+
}
|
|
101
|
+
function headerEntriesFromHeaders(headers) {
|
|
102
|
+
return Array.from(headers.entries());
|
|
103
|
+
}
|
|
104
|
+
function normalizeHeaders(headers) {
|
|
105
|
+
return Object.fromEntries(normalizedHeaderEntries(headers));
|
|
106
|
+
}
|
|
107
|
+
function trimOuterHttpWhitespace(value) {
|
|
108
|
+
return value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, "");
|
|
109
|
+
}
|
|
110
|
+
function normalizedHeaderEntries(headers) {
|
|
111
|
+
const entries = [];
|
|
112
|
+
const indices = new Map();
|
|
113
|
+
for (const [originalName, originalValue] of Object.entries(headers)) {
|
|
114
|
+
if (originalValue === undefined)
|
|
115
|
+
continue;
|
|
116
|
+
const name = originalName.toLowerCase();
|
|
117
|
+
const value = (Array.isArray(originalValue) ? originalValue : [originalValue])
|
|
118
|
+
.map(trimOuterHttpWhitespace)
|
|
119
|
+
.join(", ");
|
|
120
|
+
const existingIndex = indices.get(name);
|
|
121
|
+
if (existingIndex === undefined) {
|
|
122
|
+
indices.set(name, entries.length);
|
|
123
|
+
entries.push([name, value]);
|
|
124
|
+
continue;
|
|
113
125
|
}
|
|
114
|
-
|
|
126
|
+
const existing = entries[existingIndex];
|
|
127
|
+
if (existing)
|
|
128
|
+
existing[1] = `${existing[1]}, ${value}`;
|
|
115
129
|
}
|
|
130
|
+
return entries;
|
|
116
131
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
132
|
+
const SDK_OWNED_EXACT_CHROME_HEADERS = new Set([
|
|
133
|
+
"host",
|
|
134
|
+
"connection",
|
|
135
|
+
"user-agent",
|
|
136
|
+
"sec-ch-ua",
|
|
137
|
+
"sec-ch-ua-mobile",
|
|
138
|
+
"sec-ch-ua-platform",
|
|
139
|
+
"accept-encoding",
|
|
140
|
+
]);
|
|
141
|
+
// Non-pseudo-header order of real Chrome 149. The fixture captures
|
|
142
|
+
// (chrome-ground-truth-capture.json, chrome-extended-capture.json,
|
|
143
|
+
// h1-casing-capture.json) were taken through Playwright's `locale` option, which
|
|
144
|
+
// installs Accept-Language via DevTools next to User-Agent; real Chrome only
|
|
145
|
+
// receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
|
|
146
|
+
// Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
|
|
147
|
+
const CHROME_HEADER_ORDERS = {
|
|
148
|
+
navigation: [
|
|
149
|
+
"sec-ch-ua",
|
|
150
|
+
"sec-ch-ua-mobile",
|
|
151
|
+
"sec-ch-ua-platform",
|
|
152
|
+
"upgrade-insecure-requests",
|
|
153
|
+
"user-agent",
|
|
154
|
+
"accept",
|
|
155
|
+
"sec-fetch-site",
|
|
156
|
+
"sec-fetch-mode",
|
|
157
|
+
"sec-fetch-user",
|
|
158
|
+
"sec-fetch-dest",
|
|
159
|
+
"accept-encoding",
|
|
160
|
+
"accept-language",
|
|
161
|
+
"cookie",
|
|
162
|
+
"priority",
|
|
163
|
+
],
|
|
164
|
+
// xhr has no fixed table: it is emulated by chrome149HeaderOrder.
|
|
165
|
+
post: [
|
|
166
|
+
"content-length",
|
|
167
|
+
"sec-ch-ua-platform",
|
|
168
|
+
"user-agent",
|
|
169
|
+
"sec-ch-ua",
|
|
170
|
+
"content-type",
|
|
171
|
+
"sec-ch-ua-mobile",
|
|
172
|
+
"accept",
|
|
173
|
+
"origin",
|
|
174
|
+
"sec-fetch-site",
|
|
175
|
+
"sec-fetch-mode",
|
|
176
|
+
"sec-fetch-dest",
|
|
177
|
+
"referer",
|
|
178
|
+
"accept-encoding",
|
|
179
|
+
"accept-language",
|
|
180
|
+
"cookie",
|
|
181
|
+
"priority",
|
|
182
|
+
],
|
|
183
|
+
};
|
|
184
|
+
const CHROME_H1_HEADER_ORDERS = {
|
|
185
|
+
navigation: [
|
|
186
|
+
"host",
|
|
187
|
+
"connection",
|
|
188
|
+
"sec-ch-ua",
|
|
189
|
+
"sec-ch-ua-mobile",
|
|
190
|
+
"sec-ch-ua-platform",
|
|
191
|
+
"upgrade-insecure-requests",
|
|
192
|
+
"user-agent",
|
|
193
|
+
"accept",
|
|
194
|
+
"sec-fetch-site",
|
|
195
|
+
"sec-fetch-mode",
|
|
196
|
+
"sec-fetch-user",
|
|
197
|
+
"sec-fetch-dest",
|
|
198
|
+
"accept-encoding",
|
|
199
|
+
"accept-language",
|
|
200
|
+
"cookie",
|
|
201
|
+
],
|
|
202
|
+
xhr: [
|
|
203
|
+
"host",
|
|
204
|
+
"connection",
|
|
205
|
+
"sec-ch-ua-platform",
|
|
206
|
+
"cache-control",
|
|
207
|
+
"x-requested-with",
|
|
208
|
+
"user-agent",
|
|
209
|
+
"sec-ch-ua",
|
|
210
|
+
"sec-ch-ua-mobile",
|
|
211
|
+
"accept",
|
|
212
|
+
"sec-fetch-site",
|
|
213
|
+
"sec-fetch-mode",
|
|
214
|
+
"sec-fetch-dest",
|
|
215
|
+
"referer",
|
|
216
|
+
"accept-encoding",
|
|
217
|
+
"accept-language",
|
|
218
|
+
"cookie",
|
|
219
|
+
],
|
|
220
|
+
post: [
|
|
221
|
+
"host",
|
|
222
|
+
"connection",
|
|
223
|
+
"content-length",
|
|
224
|
+
"sec-ch-ua-platform",
|
|
225
|
+
"user-agent",
|
|
226
|
+
"sec-ch-ua",
|
|
227
|
+
"content-type",
|
|
228
|
+
"sec-ch-ua-mobile",
|
|
229
|
+
"accept",
|
|
230
|
+
"origin",
|
|
231
|
+
"sec-fetch-site",
|
|
232
|
+
"sec-fetch-mode",
|
|
233
|
+
"sec-fetch-dest",
|
|
234
|
+
"referer",
|
|
235
|
+
"accept-encoding",
|
|
236
|
+
"accept-language",
|
|
237
|
+
"cookie",
|
|
238
|
+
],
|
|
239
|
+
};
|
|
240
|
+
const CHROME_H1_HEADER_NAMES = {
|
|
241
|
+
host: "Host",
|
|
242
|
+
connection: "Connection",
|
|
243
|
+
"upgrade-insecure-requests": "Upgrade-Insecure-Requests",
|
|
244
|
+
"user-agent": "User-Agent",
|
|
245
|
+
"accept-language": "Accept-Language",
|
|
246
|
+
accept: "Accept",
|
|
247
|
+
"sec-fetch-site": "Sec-Fetch-Site",
|
|
248
|
+
"sec-fetch-mode": "Sec-Fetch-Mode",
|
|
249
|
+
"sec-fetch-user": "Sec-Fetch-User",
|
|
250
|
+
"sec-fetch-dest": "Sec-Fetch-Dest",
|
|
251
|
+
"accept-encoding": "Accept-Encoding",
|
|
252
|
+
cookie: "Cookie",
|
|
253
|
+
"cache-control": "Cache-Control",
|
|
254
|
+
"x-requested-with": "X-Requested-With",
|
|
255
|
+
referer: "Referer",
|
|
256
|
+
"content-length": "Content-Length",
|
|
257
|
+
"content-type": "Content-Type",
|
|
258
|
+
origin: "Origin",
|
|
259
|
+
range: "Range",
|
|
260
|
+
};
|
|
261
|
+
function chromeHeaderOrder(requestClass, isHttp1, caller) {
|
|
262
|
+
if (isHttp1)
|
|
263
|
+
return CHROME_H1_HEADER_ORDERS[requestClass];
|
|
264
|
+
if (requestClass !== "xhr")
|
|
265
|
+
return CHROME_HEADER_ORDERS[requestClass];
|
|
266
|
+
// Cookie and Referer are forbidden Fetch headers, so they never enter the page
|
|
267
|
+
// Fetch Headers map the emulator models. Range does occupy a map bucket
|
|
268
|
+
// (m1-capture.json range_*), but HttpCache::Transaction removes it from the
|
|
269
|
+
// request headers and PartialData re-adds it at the tail.
|
|
270
|
+
const order = chrome149HeaderOrder([...caller.keys()].filter((name) => name !== "cookie" && name !== "referer"));
|
|
271
|
+
if (caller.has("range"))
|
|
272
|
+
order.splice(order.indexOf("range"), 1);
|
|
273
|
+
// //net appends Accept-Encoding and, unless the caller supplied one,
|
|
274
|
+
// Accept-Language (URLRequestHttpJob::AddExtraHeaders), then writes the Cookie
|
|
275
|
+
// header (SetCookieHeaderAndStart); the HTTP cache re-adds a caller Range after
|
|
276
|
+
// that, and the HTTP/2 Priority header stays last.
|
|
277
|
+
let insertAt = order.indexOf("accept-encoding") + 1;
|
|
278
|
+
if (order[insertAt] === "accept-language")
|
|
279
|
+
insertAt += 1;
|
|
280
|
+
for (const name of ["cookie", "range"]) {
|
|
281
|
+
if (!caller.has(name))
|
|
282
|
+
continue;
|
|
283
|
+
order.splice(insertAt, 0, name);
|
|
284
|
+
insertAt += 1;
|
|
127
285
|
}
|
|
128
|
-
return
|
|
286
|
+
return order;
|
|
287
|
+
}
|
|
288
|
+
function normalizedCallerHeaderEntries(headers) {
|
|
289
|
+
const entries = normalizedHeaderEntries(headers);
|
|
290
|
+
assertCallerHeadersSupported(entries);
|
|
291
|
+
return entries;
|
|
292
|
+
}
|
|
293
|
+
function normalizedCallerHeaderEntriesFromRecord(headers) {
|
|
294
|
+
// The record has been through normalizeHeaders (lowercase, merged names); the
|
|
295
|
+
// lowercase here only guards later insertions such as the cookie jar's header.
|
|
296
|
+
const entries = Object.entries(headers).map(([name, value]) => [
|
|
297
|
+
name.toLowerCase(),
|
|
298
|
+
value,
|
|
299
|
+
]);
|
|
300
|
+
assertCallerHeadersSupported(entries);
|
|
301
|
+
return entries;
|
|
129
302
|
}
|
|
130
|
-
function
|
|
131
|
-
|
|
132
|
-
|
|
303
|
+
function assertCallerHeadersSupported(entries) {
|
|
304
|
+
for (const [name] of entries) {
|
|
305
|
+
if (SDK_OWNED_EXACT_CHROME_HEADERS.has(name) || name.startsWith("sec-fetch-")) {
|
|
306
|
+
throw new SDKError(`Stealth transport owns the "${name}" header; remove it from headers.`, {
|
|
307
|
+
code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED",
|
|
308
|
+
});
|
|
309
|
+
}
|
|
133
310
|
}
|
|
134
|
-
|
|
311
|
+
}
|
|
312
|
+
function chromeRequestClass(method, requestedClass) {
|
|
313
|
+
if (requestedClass)
|
|
314
|
+
return requestedClass;
|
|
315
|
+
if (method === "POST")
|
|
316
|
+
return "post";
|
|
317
|
+
return "navigation";
|
|
318
|
+
}
|
|
319
|
+
function secFetchSite(requestUrl, referer) {
|
|
320
|
+
if (!referer)
|
|
321
|
+
return "none";
|
|
135
322
|
try {
|
|
136
|
-
|
|
323
|
+
return new URL(referer).origin === new URL(requestUrl).origin ? "same-origin" : "cross-site";
|
|
137
324
|
}
|
|
138
325
|
catch {
|
|
139
|
-
|
|
140
|
-
// profile strings still run with the transport default instead of failing
|
|
141
|
-
// before the request starts. Removed built-in profile aliases above remain
|
|
142
|
-
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
143
|
-
return "chrome142";
|
|
326
|
+
return "cross-site";
|
|
144
327
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
328
|
+
}
|
|
329
|
+
function requiredEmulationHeader(headers, name) {
|
|
330
|
+
const value = headers.get(name);
|
|
331
|
+
if (value !== undefined)
|
|
332
|
+
return value;
|
|
333
|
+
throw new SDKError(`wreq-js Chrome emulation exposes no ${name} header.`, {
|
|
334
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
function buildChromeHeaderTuples(options) {
|
|
338
|
+
const callerEntries = normalizedCallerHeaderEntriesFromRecord(options.headers);
|
|
339
|
+
const caller = new Map(callerEntries);
|
|
340
|
+
const emulation = new Map(Array.from(options.emulationHeaders, ([name, value]) => [name.toLowerCase(), value]));
|
|
341
|
+
const requestClass = chromeRequestClass(options.method, options.requestClass);
|
|
342
|
+
const referer = caller.get("referer");
|
|
343
|
+
const fetchSite = secFetchSite(options.requestUrl, referer);
|
|
344
|
+
const isNavigation = requestClass === "navigation";
|
|
345
|
+
const values = new Map([
|
|
346
|
+
["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
|
|
347
|
+
["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
|
|
348
|
+
["sec-ch-ua-platform", requiredEmulationHeader(emulation, "sec-ch-ua-platform")],
|
|
349
|
+
["user-agent", requiredEmulationHeader(emulation, "user-agent")],
|
|
350
|
+
[
|
|
351
|
+
"accept-language",
|
|
352
|
+
caller.get("accept-language") ??
|
|
353
|
+
options.acceptLanguage ??
|
|
354
|
+
requiredEmulationHeader(emulation, "accept-language"),
|
|
355
|
+
],
|
|
356
|
+
["accept", isNavigation ? requiredEmulationHeader(emulation, "accept") : "*/*"],
|
|
357
|
+
[
|
|
358
|
+
"accept-encoding",
|
|
359
|
+
caller.has("range") ? "identity" : requiredEmulationHeader(emulation, "accept-encoding"),
|
|
360
|
+
],
|
|
361
|
+
["priority", isNavigation ? requiredEmulationHeader(emulation, "priority") : "u=1, i"],
|
|
362
|
+
["sec-fetch-site", fetchSite],
|
|
363
|
+
["sec-fetch-mode", isNavigation ? "navigate" : "cors"],
|
|
364
|
+
["sec-fetch-dest", isNavigation ? "document" : "empty"],
|
|
365
|
+
]);
|
|
366
|
+
if (isNavigation) {
|
|
367
|
+
values.set("upgrade-insecure-requests", "1");
|
|
368
|
+
values.set("sec-fetch-user", "?1");
|
|
149
369
|
}
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
370
|
+
for (const [name, value] of callerEntries)
|
|
371
|
+
values.set(name, value);
|
|
372
|
+
for (const name of ["content-type", "origin", "referer"]) {
|
|
373
|
+
const value = caller.get(name);
|
|
374
|
+
if (value !== undefined)
|
|
375
|
+
values.set(name, value);
|
|
153
376
|
}
|
|
154
|
-
if (
|
|
155
|
-
|
|
377
|
+
if (requestClass === "post") {
|
|
378
|
+
values.set("content-length", caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")));
|
|
156
379
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
for (const [name, value] of Object.entries(headers)) {
|
|
380
|
+
const isHttp1 = new URL(options.requestUrl).protocol === "http:";
|
|
381
|
+
if (isHttp1) {
|
|
382
|
+
values.set("host", new URL(options.requestUrl).host);
|
|
383
|
+
values.set("connection", "keep-alive");
|
|
384
|
+
}
|
|
385
|
+
const order = chromeHeaderOrder(requestClass, isHttp1, caller);
|
|
386
|
+
const tuples = [];
|
|
387
|
+
const placed = new Set();
|
|
388
|
+
for (const name of order) {
|
|
389
|
+
const value = values.get(name);
|
|
168
390
|
if (value === undefined)
|
|
169
391
|
continue;
|
|
170
|
-
|
|
392
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
393
|
+
placed.add(name);
|
|
394
|
+
}
|
|
395
|
+
for (const [name, value] of callerEntries) {
|
|
396
|
+
if (placed.has(name))
|
|
397
|
+
continue;
|
|
398
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
171
399
|
}
|
|
172
|
-
return
|
|
400
|
+
return tuples;
|
|
173
401
|
}
|
|
174
402
|
function hasOwn(object, key) {
|
|
175
403
|
return Object.hasOwn(object, key);
|
|
176
404
|
}
|
|
177
|
-
function toImpitCookieJar(cookieJar) {
|
|
178
|
-
return {
|
|
179
|
-
setCookie(cookie, _url, cb) {
|
|
180
|
-
cookieJar.setFromCookieStrings([cookie]);
|
|
181
|
-
if (typeof cb === "function")
|
|
182
|
-
cb();
|
|
183
|
-
},
|
|
184
|
-
getCookieString(_url) {
|
|
185
|
-
return cookieJar.toString();
|
|
186
|
-
},
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
405
|
function assertNoUnsupportedFingerprintOverrides(options) {
|
|
190
406
|
if (!isRecord(options))
|
|
191
407
|
return;
|
|
@@ -199,7 +415,7 @@ function assertNoUnsupportedFingerprintOverrides(options) {
|
|
|
199
415
|
unsupported.push("stealth.h2");
|
|
200
416
|
if (unsupported.length === 0)
|
|
201
417
|
return;
|
|
202
|
-
throw new SDKError(`ctx.stealth.fetch uses
|
|
418
|
+
throw new SDKError(`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use stealth.browser and stealth.os instead.`);
|
|
203
419
|
}
|
|
204
420
|
function responseHeadersToRecord(headers) {
|
|
205
421
|
const record = {};
|
|
@@ -234,10 +450,15 @@ function splitCombinedSetCookieHeader(headerValue) {
|
|
|
234
450
|
cookieStrings.push(finalCookie);
|
|
235
451
|
return cookieStrings;
|
|
236
452
|
}
|
|
237
|
-
export async function normalizeResponse(response, requestUrl) {
|
|
453
|
+
export async function normalizeResponse(response, requestUrl, maxBodyBytes) {
|
|
454
|
+
return normalizeResponseWithSignal(response, requestUrl, maxBodyBytes);
|
|
455
|
+
}
|
|
456
|
+
async function normalizeResponseWithSignal(response, requestUrl, maxBodyBytes, signal) {
|
|
238
457
|
const headers = Object.fromEntries(response.headers.entries());
|
|
239
|
-
const cookies = new
|
|
240
|
-
const bodyBytes =
|
|
458
|
+
const cookies = new StealthCookieJar(setCookieHeadersFromResponse(response.headers), response.url ?? requestUrl);
|
|
459
|
+
const bodyBytes = maxBodyBytes === undefined
|
|
460
|
+
? await readResponseArrayBuffer(response, signal)
|
|
461
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes, signal);
|
|
241
462
|
const body = new TextDecoder().decode(bodyBytes);
|
|
242
463
|
return {
|
|
243
464
|
status: response.status,
|
|
@@ -263,6 +484,125 @@ export async function normalizeResponse(response, requestUrl) {
|
|
|
263
484
|
},
|
|
264
485
|
};
|
|
265
486
|
}
|
|
487
|
+
async function readResponseArrayBuffer(response, signal) {
|
|
488
|
+
if (!signal)
|
|
489
|
+
return response.arrayBuffer();
|
|
490
|
+
throwIfAmbientAborted(signal);
|
|
491
|
+
return new Promise((resolve, reject) => {
|
|
492
|
+
let settled = false;
|
|
493
|
+
const settle = (operation) => {
|
|
494
|
+
if (settled)
|
|
495
|
+
return;
|
|
496
|
+
settled = true;
|
|
497
|
+
signal.removeEventListener("abort", onAbort);
|
|
498
|
+
operation();
|
|
499
|
+
};
|
|
500
|
+
const onAbort = () => {
|
|
501
|
+
const error = toAmbientCancellationError(signal);
|
|
502
|
+
try {
|
|
503
|
+
void response.body?.cancel().catch(() => undefined);
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
// Preserve the cancellation error if accessing or cancelling the body fails.
|
|
507
|
+
}
|
|
508
|
+
settle(() => reject(error));
|
|
509
|
+
};
|
|
510
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
511
|
+
try {
|
|
512
|
+
void response.arrayBuffer().then((body) => settle(() => resolve(body)), (error) => settle(() => reject(error)));
|
|
513
|
+
}
|
|
514
|
+
catch (error) {
|
|
515
|
+
settle(() => reject(error));
|
|
516
|
+
}
|
|
517
|
+
if (signal.aborted)
|
|
518
|
+
onAbort();
|
|
519
|
+
});
|
|
520
|
+
}
|
|
521
|
+
function responseTooLargeError(maxBodyBytes, observedBytes) {
|
|
522
|
+
return new TransportError(`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`, {
|
|
523
|
+
code: "response_too_large",
|
|
524
|
+
category: "upstream_http",
|
|
525
|
+
retryable: false,
|
|
526
|
+
status: 0,
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
function declaredContentLength(headers) {
|
|
530
|
+
const contentLength = headers.get("content-length")?.trim();
|
|
531
|
+
if (!contentLength || !/^\d+$/.test(contentLength))
|
|
532
|
+
return undefined;
|
|
533
|
+
const parsed = Number(contentLength);
|
|
534
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
535
|
+
}
|
|
536
|
+
async function readResponseBodyWithLimit(response, maxBodyBytes, signal) {
|
|
537
|
+
throwIfAmbientAborted(signal);
|
|
538
|
+
const contentLength = declaredContentLength(response.headers);
|
|
539
|
+
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
540
|
+
await response.body?.cancel().catch(() => undefined);
|
|
541
|
+
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
542
|
+
}
|
|
543
|
+
if (!response.body) {
|
|
544
|
+
throw new TransportError("Response body stream is unavailable", {
|
|
545
|
+
code: "transport_stream_unavailable",
|
|
546
|
+
category: "upstream_http",
|
|
547
|
+
status: 0,
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
const reader = response.body.getReader();
|
|
551
|
+
const chunks = [];
|
|
552
|
+
let receivedBytes = 0;
|
|
553
|
+
try {
|
|
554
|
+
while (true) {
|
|
555
|
+
const { done, value } = await readResponseBodyChunk(reader, signal);
|
|
556
|
+
if (done)
|
|
557
|
+
break;
|
|
558
|
+
if (!value)
|
|
559
|
+
continue;
|
|
560
|
+
receivedBytes += value.byteLength;
|
|
561
|
+
if (receivedBytes > maxBodyBytes) {
|
|
562
|
+
await reader.cancel().catch(() => undefined);
|
|
563
|
+
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
564
|
+
}
|
|
565
|
+
chunks.push(value);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
finally {
|
|
569
|
+
reader.releaseLock();
|
|
570
|
+
}
|
|
571
|
+
return concatenateResponseBodyChunks(chunks, receivedBytes);
|
|
572
|
+
}
|
|
573
|
+
function concatenateResponseBodyChunks(chunks, receivedBytes) {
|
|
574
|
+
const bodyBytes = new Uint8Array(receivedBytes);
|
|
575
|
+
let offset = 0;
|
|
576
|
+
for (const chunk of chunks) {
|
|
577
|
+
bodyBytes.set(chunk, offset);
|
|
578
|
+
offset += chunk.byteLength;
|
|
579
|
+
}
|
|
580
|
+
return bodyBytes.buffer;
|
|
581
|
+
}
|
|
582
|
+
function readResponseBodyChunk(reader, signal) {
|
|
583
|
+
if (!signal)
|
|
584
|
+
return reader.read();
|
|
585
|
+
throwIfAmbientAborted(signal);
|
|
586
|
+
return new Promise((resolve, reject) => {
|
|
587
|
+
let settled = false;
|
|
588
|
+
const settle = (operation) => {
|
|
589
|
+
if (settled)
|
|
590
|
+
return;
|
|
591
|
+
settled = true;
|
|
592
|
+
signal.removeEventListener("abort", onAbort);
|
|
593
|
+
operation();
|
|
594
|
+
};
|
|
595
|
+
const onAbort = () => {
|
|
596
|
+
const error = toAmbientCancellationError(signal);
|
|
597
|
+
void reader.cancel().catch(() => undefined);
|
|
598
|
+
settle(() => reject(error));
|
|
599
|
+
};
|
|
600
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
601
|
+
void reader.read().then((chunk) => settle(() => resolve(chunk)), (error) => settle(() => reject(error)));
|
|
602
|
+
if (signal.aborted)
|
|
603
|
+
onAbort();
|
|
604
|
+
});
|
|
605
|
+
}
|
|
266
606
|
function normalizeBody(body) {
|
|
267
607
|
if (body === undefined) {
|
|
268
608
|
return "";
|
|
@@ -395,8 +735,38 @@ function normalizeStealthTransportError(error) {
|
|
|
395
735
|
cause: error instanceof Error ? error : undefined,
|
|
396
736
|
});
|
|
397
737
|
}
|
|
398
|
-
function
|
|
399
|
-
|
|
738
|
+
function toAmbientCancellationError(signal, error = signal.reason) {
|
|
739
|
+
if (error instanceof TransportError && error.code === "transport_cancelled") {
|
|
740
|
+
return error;
|
|
741
|
+
}
|
|
742
|
+
return new TransportError("Request cancelled", {
|
|
743
|
+
code: "transport_cancelled",
|
|
744
|
+
status: 0,
|
|
745
|
+
retryable: false,
|
|
746
|
+
...(error !== undefined
|
|
747
|
+
? { cause: error instanceof Error ? error : new Error(String(error)) }
|
|
748
|
+
: {}),
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
function throwIfAmbientAborted(signal) {
|
|
752
|
+
if (signal?.aborted)
|
|
753
|
+
throw toAmbientCancellationError(signal);
|
|
754
|
+
}
|
|
755
|
+
function sleep(ms, signal) {
|
|
756
|
+
if (!signal)
|
|
757
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
758
|
+
throwIfAmbientAborted(signal);
|
|
759
|
+
return new Promise((resolve, reject) => {
|
|
760
|
+
const onAbort = () => {
|
|
761
|
+
clearTimeout(timer);
|
|
762
|
+
reject(toAmbientCancellationError(signal));
|
|
763
|
+
};
|
|
764
|
+
const timer = setTimeout(() => {
|
|
765
|
+
signal.removeEventListener("abort", onAbort);
|
|
766
|
+
resolve();
|
|
767
|
+
}, ms);
|
|
768
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
769
|
+
});
|
|
400
770
|
}
|
|
401
771
|
function normalizeMethod(method) {
|
|
402
772
|
switch (method.toUpperCase()) {
|
|
@@ -420,16 +790,6 @@ function normalizeMethod(method) {
|
|
|
420
790
|
throw new SDKError(`Unsupported stealth method: ${method}`);
|
|
421
791
|
}
|
|
422
792
|
}
|
|
423
|
-
function isRedirectStatus(status) {
|
|
424
|
-
return [301, 302, 303, 307, 308].includes(status);
|
|
425
|
-
}
|
|
426
|
-
function nextRedirectMethod(status, method) {
|
|
427
|
-
if (status === 303 && method !== "HEAD")
|
|
428
|
-
return "GET";
|
|
429
|
-
if ((status === 301 || status === 302) && method === "POST")
|
|
430
|
-
return "GET";
|
|
431
|
-
return method;
|
|
432
|
-
}
|
|
433
793
|
function locationHeader(headers) {
|
|
434
794
|
for (const [name, value] of Object.entries(headers)) {
|
|
435
795
|
if (name.toLowerCase() === "location")
|
|
@@ -437,43 +797,206 @@ function locationHeader(headers) {
|
|
|
437
797
|
}
|
|
438
798
|
return undefined;
|
|
439
799
|
}
|
|
800
|
+
function withoutRedirectBodyHeaders(headers) {
|
|
801
|
+
return Object.fromEntries(Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())));
|
|
802
|
+
}
|
|
803
|
+
function assertStealthRedirectUrl(url) {
|
|
804
|
+
const protocol = new URL(url).protocol;
|
|
805
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
806
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
807
|
+
code: "transport_invalid_url",
|
|
808
|
+
status: 0,
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
function discardStealthRedirectBody(response) {
|
|
813
|
+
try {
|
|
814
|
+
const cancellation = response.body?.cancel();
|
|
815
|
+
if (cancellation)
|
|
816
|
+
void cancellation.catch(() => undefined);
|
|
817
|
+
}
|
|
818
|
+
catch {
|
|
819
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
820
|
+
// failure must not replace or delay that decision.
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
async function fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, signal, buildHeaders) {
|
|
824
|
+
let currentUrl = requestUrl;
|
|
825
|
+
let currentMethod = method;
|
|
826
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
827
|
+
let currentHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
828
|
+
let followedHops = 0;
|
|
829
|
+
let response;
|
|
830
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
831
|
+
while (true) {
|
|
832
|
+
throwIfAmbientAborted(signal);
|
|
833
|
+
const headers = { ...currentHeaders };
|
|
834
|
+
if (!hasHeader(headers, "cookie")) {
|
|
835
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
836
|
+
if (cookieHeader)
|
|
837
|
+
headers.cookie = cookieHeader;
|
|
838
|
+
}
|
|
839
|
+
const requestInit = {
|
|
840
|
+
headers: buildHeaders
|
|
841
|
+
? buildHeaders(currentUrl, currentMethod, currentBody, headers)
|
|
842
|
+
: headers,
|
|
843
|
+
method: currentMethod,
|
|
844
|
+
redirect: "manual",
|
|
845
|
+
...(new URL(currentUrl).protocol === "http:" ? { disableDefaultHeaders: true } : {}),
|
|
846
|
+
...(signal ? { signal } : {}),
|
|
847
|
+
};
|
|
848
|
+
if (currentBody !== undefined)
|
|
849
|
+
requestInit.body = currentBody;
|
|
850
|
+
await transport.clearCookies();
|
|
851
|
+
throwIfAmbientAborted(signal);
|
|
852
|
+
const remainingTimeout = deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
853
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
854
|
+
throw new TransportError("Request timed out", {
|
|
855
|
+
code: "transport_timeout",
|
|
856
|
+
status: 0,
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
if (remainingTimeout !== undefined)
|
|
860
|
+
requestInit.timeout = remainingTimeout;
|
|
861
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
862
|
+
if (signal?.aborted) {
|
|
863
|
+
discardStealthRedirectBody(response);
|
|
864
|
+
throw toAmbientCancellationError(signal);
|
|
865
|
+
}
|
|
866
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers), response.url ?? currentUrl);
|
|
867
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual")
|
|
868
|
+
break;
|
|
869
|
+
if (options.redirect === "error") {
|
|
870
|
+
discardStealthRedirectBody(response);
|
|
871
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
872
|
+
code: "transport_network_error",
|
|
873
|
+
status: 0,
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
const nextUrl = resolveRedirectUrl(response.headers.get("location") ?? undefined, response.url ?? currentUrl);
|
|
877
|
+
if (!nextUrl)
|
|
878
|
+
break;
|
|
879
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
880
|
+
discardStealthRedirectBody(response);
|
|
881
|
+
throw new TransportError(`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`, { code: "transport_network_error", status: 0 });
|
|
882
|
+
}
|
|
883
|
+
assertStealthRedirectUrl(nextUrl);
|
|
884
|
+
discardStealthRedirectBody(response);
|
|
885
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
886
|
+
if (nextMethod !== currentMethod) {
|
|
887
|
+
currentBody = undefined;
|
|
888
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
889
|
+
}
|
|
890
|
+
currentMethod = nextMethod;
|
|
891
|
+
currentUrl = nextUrl;
|
|
892
|
+
followedHops += 1;
|
|
893
|
+
}
|
|
894
|
+
const normalized = await normalizeResponseWithSignal(response, currentUrl, options.maxBodyBytes, signal);
|
|
895
|
+
if (followedHops > 0)
|
|
896
|
+
normalized.redirected = true;
|
|
897
|
+
return { normalized, response };
|
|
898
|
+
}
|
|
440
899
|
function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
441
900
|
const clients = new Map();
|
|
442
901
|
let closed = false;
|
|
443
902
|
let hasWarnedMissingProxy = false;
|
|
444
903
|
const warn = clientOptions.warn ?? console.warn;
|
|
445
|
-
const cookieJar = new
|
|
446
|
-
|
|
447
|
-
function getClient(profileName, proxyUrl, ignoreTlsErrors) {
|
|
904
|
+
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
905
|
+
async function getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders) {
|
|
448
906
|
if (closed) {
|
|
449
907
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
450
908
|
}
|
|
451
|
-
const
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
909
|
+
const wreq = await getWreqModule();
|
|
910
|
+
const { browser, os } = resolveWreqProfile(profile, wreq.getProfiles());
|
|
911
|
+
const cacheKey = JSON.stringify({
|
|
912
|
+
browser,
|
|
913
|
+
os,
|
|
914
|
+
proxyUrl,
|
|
915
|
+
ignoreTlsErrors,
|
|
916
|
+
headerOrder: defaultHeaders?.map(([name]) => name),
|
|
917
|
+
});
|
|
918
|
+
let entry = clients.get(cacheKey);
|
|
919
|
+
if (!entry) {
|
|
920
|
+
entry = {
|
|
921
|
+
session: wreq.createSession({
|
|
922
|
+
browser,
|
|
923
|
+
os,
|
|
924
|
+
...(defaultHeaders ? { defaultHeaders } : {}),
|
|
925
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
926
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
927
|
+
timeout: 30_000,
|
|
928
|
+
}),
|
|
929
|
+
tail: Promise.resolve(),
|
|
930
|
+
};
|
|
931
|
+
clients.set(cacheKey, entry);
|
|
932
|
+
}
|
|
933
|
+
return entry;
|
|
934
|
+
}
|
|
935
|
+
async function withClient(profile, proxyUrl, ignoreTlsErrors, operation, signal, defaultHeaders) {
|
|
936
|
+
throwIfAmbientAborted(signal);
|
|
937
|
+
const entry = await getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders);
|
|
938
|
+
throwIfAmbientAborted(signal);
|
|
939
|
+
const previous = entry.tail;
|
|
940
|
+
let release;
|
|
941
|
+
entry.tail = new Promise((resolve) => {
|
|
942
|
+
release = resolve;
|
|
943
|
+
});
|
|
944
|
+
let acquired = false;
|
|
945
|
+
try {
|
|
946
|
+
await waitForClientTurn(previous, signal);
|
|
947
|
+
acquired = true;
|
|
948
|
+
throwIfAmbientAborted(signal);
|
|
949
|
+
const client = await entry.session;
|
|
950
|
+
throwIfAmbientAborted(signal);
|
|
951
|
+
const result = await operation(client);
|
|
952
|
+
throwIfAmbientAborted(signal);
|
|
953
|
+
return result;
|
|
954
|
+
}
|
|
955
|
+
finally {
|
|
956
|
+
if (acquired) {
|
|
957
|
+
release();
|
|
958
|
+
}
|
|
959
|
+
else {
|
|
960
|
+
void previous.then(release, release);
|
|
961
|
+
}
|
|
463
962
|
}
|
|
464
|
-
return client;
|
|
465
963
|
}
|
|
466
|
-
async function
|
|
964
|
+
async function waitForClientTurn(previous, signal) {
|
|
965
|
+
if (!signal) {
|
|
966
|
+
await previous;
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
throwIfAmbientAborted(signal);
|
|
970
|
+
await new Promise((resolve, reject) => {
|
|
971
|
+
let settled = false;
|
|
972
|
+
const settle = (operation) => {
|
|
973
|
+
if (settled)
|
|
974
|
+
return;
|
|
975
|
+
settled = true;
|
|
976
|
+
signal.removeEventListener("abort", onAbort);
|
|
977
|
+
operation();
|
|
978
|
+
};
|
|
979
|
+
const onAbort = () => settle(() => reject(toAmbientCancellationError(signal)));
|
|
980
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
981
|
+
void previous.then(() => settle(resolve), (error) => settle(() => reject(error)));
|
|
982
|
+
if (signal.aborted)
|
|
983
|
+
onAbort();
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
467
987
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
468
988
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
469
989
|
upstream: clientOptions.upstream,
|
|
470
|
-
apifuseConfig: clientOptions.apifuseConfig,
|
|
471
990
|
affinityKey: clientOptions.affinityKey,
|
|
472
991
|
proxyAttempt: computeProxyAttemptIndex({
|
|
473
992
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
474
993
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
475
994
|
retryAttemptOffset: proxyAttempt,
|
|
476
995
|
}),
|
|
996
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
997
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
998
|
+
transportProtocols: ["http", "socks5"],
|
|
999
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
477
1000
|
telemetry: clientOptions.telemetry,
|
|
478
1001
|
});
|
|
479
1002
|
if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
|
|
@@ -484,42 +1007,74 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
484
1007
|
url: resolvedProxy.url,
|
|
485
1008
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
486
1009
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
1010
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
487
1011
|
};
|
|
488
1012
|
}
|
|
489
1013
|
const session = {
|
|
490
1014
|
async fetch(url, options = {}) {
|
|
491
|
-
const
|
|
492
|
-
const hasExplicitRetryPolicy =
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
? undefined
|
|
499
|
-
: createDefaultProxyTransportRetryOptions({
|
|
1015
|
+
const requestProfile = resolveStealthProfileSelection(options.stealth, defaultProfile);
|
|
1016
|
+
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
1017
|
+
try {
|
|
1018
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
1019
|
+
normalizedCallerHeaderEntries(options.headers ?? {});
|
|
1020
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
1021
|
+
const stealthRetryOptions = normalizeProxyTransportRetryOptions(options.retry, {
|
|
500
1022
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
501
1023
|
label: "Stealth",
|
|
502
|
-
})
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
1024
|
+
}) ??
|
|
1025
|
+
(hasExplicitRetryPolicy
|
|
1026
|
+
? undefined
|
|
1027
|
+
: createDefaultProxyTransportRetryOptions({
|
|
1028
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
1029
|
+
label: "Stealth",
|
|
1030
|
+
}));
|
|
1031
|
+
if (stealthRetryOptions) {
|
|
1032
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
1033
|
+
}
|
|
1034
|
+
return { hasExplicitRetryPolicy, method, stealthRetryOptions };
|
|
1035
|
+
}
|
|
1036
|
+
catch (error) {
|
|
1037
|
+
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
1038
|
+
}
|
|
1039
|
+
})();
|
|
1040
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
506
1041
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
507
1042
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
508
1043
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
509
|
-
|
|
1044
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
1045
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
1046
|
+
const policyProxy = clientOptions.proxyPolicy ??
|
|
510
1047
|
(typeof clientOptions.upstream?.proxy === "object"
|
|
511
|
-
? clientOptions.upstream.proxy
|
|
512
|
-
: undefined)
|
|
513
|
-
|
|
514
|
-
|
|
1048
|
+
? clientOptions.upstream.proxy
|
|
1049
|
+
: undefined);
|
|
1050
|
+
// The pool span is already bounded by each vendor's max pool size
|
|
1051
|
+
// (smartproxy ≤20, nodemaven ≤50), so the configured span never exceeds
|
|
1052
|
+
// the chain's true maximum — a large NodeMaven pool stays fully
|
|
1053
|
+
// reachable rather than being truncated at an arbitrary ceiling.
|
|
1054
|
+
const policyProxyAttemptCap = Math.max(1, policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE);
|
|
1055
|
+
// A registry vendor chain (smartproxy/nodemaven) is the only policy whose
|
|
1056
|
+
// successive attempts resolve a *different* endpoint, so it is the only one
|
|
1057
|
+
// that may widen the attempt cap to the pool span, de-duplicate endpoints,
|
|
1058
|
+
// and drive allocator stale-pool refresh. Deprecated custom/decodo policies
|
|
1059
|
+
// have no managed endpoint to rotate or refresh, so they follow the ordinary
|
|
1060
|
+
// transport-retry budget instead.
|
|
1061
|
+
const rotatesRegistryChain = usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
1062
|
+
const maxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
1063
|
+
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
515
1064
|
let lastError;
|
|
516
1065
|
for (let refreshAttempt = 0; refreshAttempt <= MAX_POLICY_PROXY_POOL_REFRESHES; refreshAttempt += 1) {
|
|
517
1066
|
let stalePoolError;
|
|
518
1067
|
let stalePoolDiagnosticProxy;
|
|
519
1068
|
const attemptedProxies = new Set();
|
|
520
1069
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
1070
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
521
1071
|
let proxy;
|
|
522
1072
|
let attemptProxy;
|
|
1073
|
+
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
1074
|
+
let serializedUrl;
|
|
1075
|
+
let fallbackSensitiveValues = [];
|
|
1076
|
+
let fallbackRequestUrl;
|
|
1077
|
+
let fallbackRedactedUrl;
|
|
523
1078
|
const attemptStartedAt = Date.now();
|
|
524
1079
|
let attemptRecorded = false;
|
|
525
1080
|
const recordProxyAttempt = (outcome, errorCode, status) => {
|
|
@@ -527,7 +1082,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
527
1082
|
return;
|
|
528
1083
|
attemptRecorded = true;
|
|
529
1084
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
530
|
-
provider: "smartproxy",
|
|
1085
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
531
1086
|
attempt: attempt + 1,
|
|
532
1087
|
...(attemptProxy?.poolIndex === undefined
|
|
533
1088
|
? {}
|
|
@@ -540,37 +1095,60 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
540
1095
|
});
|
|
541
1096
|
};
|
|
542
1097
|
try {
|
|
1098
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1099
|
+
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
1100
|
+
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
1101
|
+
fallbackSensitiveValues = [
|
|
1102
|
+
...new Set([
|
|
1103
|
+
...Object.values(sensitiveParams ?? {}).map(String),
|
|
1104
|
+
...structural.sensitiveValues,
|
|
1105
|
+
]),
|
|
1106
|
+
].filter((value) => value !== "");
|
|
1107
|
+
fallbackRequestUrl = url;
|
|
1108
|
+
fallbackRedactedUrl = structural.redactedUrl;
|
|
543
1109
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
544
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
1110
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
1111
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
545
1112
|
proxy = attemptProxy.url;
|
|
546
|
-
if (proxy &&
|
|
1113
|
+
if (proxy && dedupeAllocatorEndpoints) {
|
|
1114
|
+
// An under-filled allocation repeats endpoints (via the modulo
|
|
1115
|
+
// pool mapping) before the flat offset crosses into the next
|
|
1116
|
+
// vendor. Skip an already-tried endpoint and advance the offset
|
|
1117
|
+
// rather than breaking — breaking here would strand the request on
|
|
1118
|
+
// the primary vendor and never reach the fallback leg.
|
|
547
1119
|
if (attemptedProxies.has(proxy)) {
|
|
548
|
-
|
|
1120
|
+
continue;
|
|
549
1121
|
}
|
|
550
1122
|
attemptedProxies.add(proxy);
|
|
551
1123
|
}
|
|
552
1124
|
const ignoreTlsErrors = Boolean(options.stealth?.insecureSkipVerify ??
|
|
553
1125
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify));
|
|
554
|
-
|
|
555
|
-
const requestUrl =
|
|
556
|
-
const
|
|
557
|
-
|
|
558
|
-
|
|
1126
|
+
serializedUrl = serializeRequestUrl(resolveUrl(baseUrl, url), options.params, sensitiveParams);
|
|
1127
|
+
const { requestUrl } = serializedUrl;
|
|
1128
|
+
const wreq = await getWreqModule();
|
|
1129
|
+
const mapping = resolveWreqProfile(requestProfile, wreq.getProfiles());
|
|
1130
|
+
const chromeEmulationHeaders = mapping.browser.startsWith("chrome_")
|
|
1131
|
+
? Array.from(wreq.getEmulationHeaders(mapping.browser, mapping.os), ([name, value]) => [String(name), String(value)])
|
|
1132
|
+
: undefined;
|
|
1133
|
+
const buildOrderedHeaders = chromeEmulationHeaders
|
|
1134
|
+
? (currentUrl, currentMethod, currentBody, currentHeaders) => buildChromeHeaderTuples({
|
|
1135
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1136
|
+
method: currentMethod,
|
|
1137
|
+
body: currentBody,
|
|
1138
|
+
headers: currentHeaders,
|
|
1139
|
+
requestUrl: currentUrl,
|
|
1140
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1141
|
+
requestClass: options.stealth?.requestClass,
|
|
1142
|
+
})
|
|
1143
|
+
: undefined;
|
|
1144
|
+
const initialHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1145
|
+
if (!hasHeader(initialHeaders, "cookie")) {
|
|
1146
|
+
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
559
1147
|
if (cookieHeader)
|
|
560
|
-
|
|
1148
|
+
initialHeaders.cookie = cookieHeader;
|
|
561
1149
|
}
|
|
562
|
-
const
|
|
563
|
-
|
|
564
|
-
method,
|
|
565
|
-
...(options.redirect ? { redirect: options.redirect } : {}),
|
|
566
|
-
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
567
|
-
};
|
|
568
|
-
if (options.body !== undefined) {
|
|
569
|
-
requestInit.body = normalizeBody(options.body);
|
|
570
|
-
}
|
|
571
|
-
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(requestUrl, requestInit);
|
|
572
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
573
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
1150
|
+
const defaultHeaders = buildOrderedHeaders?.(requestUrl, method, options.body === undefined ? undefined : normalizeBody(options.body), initialHeaders);
|
|
1151
|
+
const { normalized, response } = await withClient(requestProfile, proxy, ignoreTlsErrors, (transport) => fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options, clientOptions.signal, buildOrderedHeaders), clientOptions.signal, defaultHeaders);
|
|
574
1152
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
575
1153
|
throw createProxyConnectFailureError(normalized.body);
|
|
576
1154
|
}
|
|
@@ -602,12 +1180,29 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
602
1180
|
return normalized;
|
|
603
1181
|
}
|
|
604
1182
|
catch (error) {
|
|
605
|
-
const
|
|
1183
|
+
const sensitiveValues = serializedUrl?.sensitiveValues ?? fallbackSensitiveValues;
|
|
1184
|
+
let normalizedError;
|
|
1185
|
+
try {
|
|
1186
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1187
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
1188
|
+
}
|
|
1189
|
+
catch (normalizationError) {
|
|
1190
|
+
const redactedNormalizationError = redactSensitiveError(normalizationError, sensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
1191
|
+
if (normalizationError instanceof TransportError &&
|
|
1192
|
+
normalizationError.code === "transport_cancelled") {
|
|
1193
|
+
recordProxyAttempt("error", proxyAttemptErrorCode(normalizationError), proxyAttemptStatus(normalizationError));
|
|
1194
|
+
}
|
|
1195
|
+
throw redactedNormalizationError;
|
|
1196
|
+
}
|
|
1197
|
+
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
1198
|
+
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
1199
|
+
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
1200
|
+
normalizedError = redactSensitiveError(normalizedError, sensitiveValues, serializedUrl?.requestUrl ?? fallbackRequestUrl, serializedUrl?.redactedUrl ?? fallbackRedactedUrl);
|
|
606
1201
|
recordProxyAttempt("error", proxyAttemptErrorCode(normalizedError), proxyAttemptStatus(normalizedError));
|
|
607
1202
|
lastError = normalizedError;
|
|
608
|
-
if (proxy &&
|
|
1203
|
+
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
609
1204
|
stalePoolError = normalizedError;
|
|
610
|
-
if (
|
|
1205
|
+
if (runProxyAuthDiagnostic) {
|
|
611
1206
|
stalePoolDiagnosticProxy = proxy;
|
|
612
1207
|
}
|
|
613
1208
|
if (attempt + 1 < maxAttempts) {
|
|
@@ -615,37 +1210,57 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
615
1210
|
}
|
|
616
1211
|
break;
|
|
617
1212
|
}
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
1213
|
+
// Cap the number of transport retries. For a policy-allocator chain,
|
|
1214
|
+
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
1215
|
+
// rotates across the concatenated vendor pool spans), so a transport
|
|
1216
|
+
// failure is a signal to advance to the next endpoint — potentially
|
|
1217
|
+
// crossing into the fallback vendor — not to retry the same endpoint.
|
|
1218
|
+
// Truncating that rotation at the per-endpoint retry budget would
|
|
1219
|
+
// strand the request on the primary vendor and never reach the
|
|
1220
|
+
// fallback, since the crossover only happens once the flat attempt
|
|
1221
|
+
// index exceeds the primary vendor's pool size (~10-20).
|
|
1222
|
+
// resolvePolicyTransportAttemptCap widens to the full chain span only
|
|
1223
|
+
// for implicit, safe-method allocator requests; explicit retry
|
|
1224
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
1225
|
+
// static/non-registry vendors keep the per-endpoint retry budget.
|
|
1226
|
+
const transportRetryCap = resolvePolicyTransportAttemptCap({
|
|
1227
|
+
policy: policyProxy,
|
|
1228
|
+
usesPolicyAllocator,
|
|
1229
|
+
retryAttempts: stealthRetryOptions?.attempts ?? 1,
|
|
1230
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
1231
|
+
method,
|
|
1232
|
+
});
|
|
1233
|
+
if (attempt + 1 < transportRetryCap &&
|
|
622
1234
|
shouldRetryProxyTransportAttempt({
|
|
623
|
-
error:
|
|
1235
|
+
error: { code: retryErrorCode },
|
|
624
1236
|
explicitRetry: hasExplicitRetryPolicy,
|
|
625
1237
|
method,
|
|
626
1238
|
options: stealthRetryOptions,
|
|
627
1239
|
proxyUsed: Boolean(proxy),
|
|
628
1240
|
})) {
|
|
629
1241
|
if (stealthRetryOptions) {
|
|
630
|
-
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1));
|
|
1242
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1), clientOptions.signal);
|
|
631
1243
|
}
|
|
1244
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
632
1245
|
continue;
|
|
633
1246
|
}
|
|
634
1247
|
throw normalizedError;
|
|
635
1248
|
}
|
|
636
1249
|
}
|
|
637
|
-
if (
|
|
1250
|
+
if (rotatesRegistryChain &&
|
|
638
1251
|
stalePoolError &&
|
|
639
1252
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES) {
|
|
1253
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
640
1254
|
await invalidateProxyResolutionCacheAsync({
|
|
641
1255
|
proxyPolicy: clientOptions.proxyPolicy,
|
|
642
1256
|
upstream: clientOptions.upstream,
|
|
643
1257
|
affinityKey: clientOptions.affinityKey,
|
|
644
1258
|
});
|
|
1259
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
645
1260
|
continue;
|
|
646
1261
|
}
|
|
647
1262
|
const proxyAuthDiagnostic = stalePoolError && stalePoolDiagnosticProxy
|
|
648
|
-
? await classifyProxyAuthDiagnostic(
|
|
1263
|
+
? await classifyProxyAuthDiagnostic(requestProfile, stalePoolDiagnosticProxy)
|
|
649
1264
|
: undefined;
|
|
650
1265
|
if (proxyAuthDiagnostic === "source_ip_denied") {
|
|
651
1266
|
throw createProxyAuthIpDeniedError(stalePoolError instanceof Error ? stalePoolError : undefined);
|
|
@@ -662,6 +1277,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
662
1277
|
}
|
|
663
1278
|
break;
|
|
664
1279
|
}
|
|
1280
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
665
1281
|
throw normalizeStealthTransportError(lastError);
|
|
666
1282
|
},
|
|
667
1283
|
cookies: cookieJar,
|
|
@@ -670,81 +1286,153 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
670
1286
|
const maxHops = options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
671
1287
|
? 10
|
|
672
1288
|
: Math.max(0, Math.floor(options.maxHops));
|
|
1289
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, sensitiveParams, ...fetchOptions } = options;
|
|
673
1290
|
const hops = [];
|
|
674
|
-
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
675
1291
|
let method = normalizeMethod(options.method ?? "GET");
|
|
676
1292
|
let body = options.body;
|
|
677
1293
|
let response;
|
|
678
1294
|
const visitedRequests = new Set();
|
|
679
|
-
const
|
|
1295
|
+
const initialParams = params
|
|
1296
|
+
? Object.fromEntries(Object.entries(params).map(([key, value]) => [
|
|
1297
|
+
key,
|
|
1298
|
+
Array.isArray(value) ? [...value] : value,
|
|
1299
|
+
]))
|
|
1300
|
+
: undefined;
|
|
1301
|
+
const normalizedSensitiveParams = normalizeSensitiveParams(sensitiveParams);
|
|
1302
|
+
const initialSensitiveParams = normalizedSensitiveParams
|
|
1303
|
+
? { ...normalizedSensitiveParams }
|
|
1304
|
+
: undefined;
|
|
1305
|
+
const sensitiveParamNames = initialSensitiveParams
|
|
1306
|
+
? Object.keys(initialSensitiveParams)
|
|
1307
|
+
: [];
|
|
1308
|
+
const callerStructural = redactUrlQueryParams(options.url, sensitiveParamNames);
|
|
1309
|
+
const sensitiveValues = new Set([
|
|
1310
|
+
...Object.values(initialSensitiveParams ?? {}),
|
|
1311
|
+
...callerStructural.sensitiveValues,
|
|
1312
|
+
].filter((value) => value !== ""));
|
|
1313
|
+
const redactRedirectUrl = (value) => {
|
|
1314
|
+
const structural = redactUrlQueryParams(value, [
|
|
1315
|
+
...new Set([...sensitiveParamNames, ...sensitiveQueryParamNames(value)]),
|
|
1316
|
+
]);
|
|
1317
|
+
for (const sensitiveValue of structural.sensitiveValues) {
|
|
1318
|
+
sensitiveValues.add(sensitiveValue);
|
|
1319
|
+
}
|
|
1320
|
+
return redactSensitiveText(structural.redactedUrl, [...sensitiveValues]);
|
|
1321
|
+
};
|
|
1322
|
+
let currentUrl;
|
|
1323
|
+
let initialUrl;
|
|
1324
|
+
try {
|
|
1325
|
+
currentUrl = resolveUrl(baseUrl, options.url);
|
|
1326
|
+
redactRedirectUrl(currentUrl);
|
|
1327
|
+
initialUrl = serializeRequestUrl(currentUrl, initialParams, initialSensitiveParams);
|
|
1328
|
+
for (const value of initialUrl.sensitiveValues) {
|
|
1329
|
+
if (value !== "")
|
|
1330
|
+
sensitiveValues.add(value);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
catch (error) {
|
|
1334
|
+
throw redactSensitiveError(error, [...sensitiveValues], options.url, redactRedirectUrl(options.url));
|
|
1335
|
+
}
|
|
680
1336
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
1337
|
+
const outboundUrl = hopIndex === 0 ? initialUrl.requestUrl : serializeRequestUrl(currentUrl).requestUrl;
|
|
1338
|
+
// Preserve params-only loop bookkeeping from before sensitiveParams:
|
|
1339
|
+
// the first visited key is the caller's resolved URL, not its expanded query.
|
|
1340
|
+
const visitedUrl = hopIndex === 0 && !initialSensitiveParams ? currentUrl : outboundUrl;
|
|
1341
|
+
visitedRequests.add(`${method} ${visitedUrl}`);
|
|
1342
|
+
try {
|
|
1343
|
+
response = await session.fetch(currentUrl, {
|
|
1344
|
+
...fetchOptions,
|
|
1345
|
+
body,
|
|
1346
|
+
method,
|
|
1347
|
+
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
1348
|
+
...(hopIndex === 0 && initialSensitiveParams
|
|
1349
|
+
? { sensitiveParams: initialSensitiveParams }
|
|
1350
|
+
: {}),
|
|
1351
|
+
redirect: "manual",
|
|
1352
|
+
throwOnHttpError: false,
|
|
1353
|
+
});
|
|
1354
|
+
}
|
|
1355
|
+
catch (error) {
|
|
1356
|
+
throw redactSensitiveError(error, [...sensitiveValues], outboundUrl, redactRedirectUrl(outboundUrl));
|
|
1357
|
+
}
|
|
1358
|
+
// StealthResponse.url is programmatic metadata and remains raw. Only the
|
|
1359
|
+
// redirect hop emitted below is a diagnostic surface.
|
|
1360
|
+
const responseUrl = response.url ?? (hopIndex === 0 && initialSensitiveParams ? outboundUrl : currentUrl);
|
|
690
1361
|
if (!isRedirectStatus(response.status)) {
|
|
691
1362
|
return {
|
|
692
1363
|
final: response,
|
|
693
1364
|
hops,
|
|
694
1365
|
reason: "completed",
|
|
695
1366
|
cookies: cookieJar.snapshot(),
|
|
1367
|
+
cookieStore: cookieJar.serialize(),
|
|
696
1368
|
};
|
|
697
1369
|
}
|
|
698
1370
|
const location = locationHeader(response.headers);
|
|
699
|
-
const
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
1371
|
+
const redactedResponseUrl = redactRedirectUrl(responseUrl);
|
|
1372
|
+
const redactedLocation = location ? redactRedirectUrl(location) : undefined;
|
|
1373
|
+
let nextUrl;
|
|
1374
|
+
try {
|
|
1375
|
+
nextUrl = resolveRedirectUrl(location, responseUrl);
|
|
1376
|
+
}
|
|
1377
|
+
catch (error) {
|
|
1378
|
+
throw redactSensitiveError(error, [...sensitiveValues], location, redactedLocation);
|
|
1379
|
+
}
|
|
1380
|
+
const realHop = {
|
|
1381
|
+
url: responseUrl,
|
|
704
1382
|
status: response.status,
|
|
705
1383
|
method,
|
|
706
1384
|
...(location ? { location } : {}),
|
|
707
1385
|
...(nextUrl ? { nextUrl } : {}),
|
|
708
1386
|
};
|
|
1387
|
+
const hop = {
|
|
1388
|
+
...realHop,
|
|
1389
|
+
url: redactedResponseUrl,
|
|
1390
|
+
...(redactedLocation ? { location: redactedLocation } : {}),
|
|
1391
|
+
...(nextUrl ? { nextUrl: redactRedirectUrl(nextUrl) } : {}),
|
|
1392
|
+
};
|
|
709
1393
|
hops.push(hop);
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
1394
|
+
let shouldStop = false;
|
|
1395
|
+
if (stopWhen) {
|
|
1396
|
+
try {
|
|
1397
|
+
shouldStop = await stopWhen(realHop);
|
|
1398
|
+
}
|
|
1399
|
+
catch (error) {
|
|
1400
|
+
let sanitizedError = error;
|
|
1401
|
+
for (const [rawUrl, safeUrl] of [
|
|
1402
|
+
[responseUrl, redactedResponseUrl],
|
|
1403
|
+
[location, redactedLocation],
|
|
1404
|
+
[nextUrl, nextUrl ? redactRedirectUrl(nextUrl) : undefined],
|
|
1405
|
+
]) {
|
|
1406
|
+
if (!rawUrl || !safeUrl)
|
|
1407
|
+
continue;
|
|
1408
|
+
sanitizedError = redactSensitiveError(sanitizedError, [...sensitiveValues], rawUrl, safeUrl);
|
|
1409
|
+
}
|
|
1410
|
+
throw sanitizedError;
|
|
1411
|
+
}
|
|
725
1412
|
}
|
|
726
|
-
|
|
1413
|
+
const decision = evaluateRedirectHop({
|
|
1414
|
+
status: response.status,
|
|
1415
|
+
method,
|
|
1416
|
+
nextUrl,
|
|
1417
|
+
shouldStop,
|
|
1418
|
+
redirectCount: hops.length,
|
|
1419
|
+
maxHops,
|
|
1420
|
+
visitedRequests,
|
|
1421
|
+
});
|
|
1422
|
+
if (decision.kind === "stop") {
|
|
727
1423
|
return {
|
|
728
1424
|
final: response,
|
|
729
1425
|
hops,
|
|
730
|
-
reason:
|
|
1426
|
+
reason: decision.reason,
|
|
731
1427
|
cookies: cookieJar.snapshot(),
|
|
1428
|
+
cookieStore: cookieJar.serialize(),
|
|
732
1429
|
};
|
|
733
1430
|
}
|
|
734
|
-
|
|
735
|
-
if (nextMethod !== method) {
|
|
1431
|
+
if (decision.nextMethod !== method) {
|
|
736
1432
|
body = undefined;
|
|
737
1433
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
final: response,
|
|
741
|
-
hops,
|
|
742
|
-
reason: "loop",
|
|
743
|
-
cookies: cookieJar.snapshot(),
|
|
744
|
-
};
|
|
745
|
-
}
|
|
746
|
-
method = nextMethod;
|
|
747
|
-
currentUrl = nextUrl;
|
|
1434
|
+
method = decision.nextMethod;
|
|
1435
|
+
currentUrl = decision.nextUrl;
|
|
748
1436
|
}
|
|
749
1437
|
if (!response) {
|
|
750
1438
|
response = await session.fetch(currentUrl, {
|
|
@@ -761,25 +1449,41 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
761
1449
|
hops,
|
|
762
1450
|
reason: "max_hops",
|
|
763
1451
|
cookies: cookieJar.snapshot(),
|
|
1452
|
+
cookieStore: cookieJar.serialize(),
|
|
764
1453
|
};
|
|
765
1454
|
},
|
|
766
1455
|
},
|
|
767
1456
|
close() {
|
|
768
1457
|
closed = true;
|
|
1458
|
+
for (const client of clients.values()) {
|
|
1459
|
+
void client.session
|
|
1460
|
+
.then((session) => session.close())
|
|
1461
|
+
.catch((error) => {
|
|
1462
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1463
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1464
|
+
});
|
|
1465
|
+
}
|
|
769
1466
|
clients.clear();
|
|
770
1467
|
},
|
|
771
1468
|
};
|
|
772
1469
|
return session;
|
|
773
|
-
async function classifyProxyAuthDiagnostic(
|
|
1470
|
+
async function classifyProxyAuthDiagnostic(profile, proxy) {
|
|
774
1471
|
try {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
1472
|
+
return await withClient(profile, proxy, false, async (client) => {
|
|
1473
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1474
|
+
await client.clearCookies();
|
|
1475
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1476
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1477
|
+
method: "GET",
|
|
1478
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1479
|
+
...(clientOptions.signal ? { signal: clientOptions.signal } : {}),
|
|
1480
|
+
});
|
|
1481
|
+
const normalized = await normalizeResponseWithSignal(response, undefined, undefined, clientOptions.signal);
|
|
1482
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1483
|
+
}, clientOptions.signal);
|
|
781
1484
|
}
|
|
782
1485
|
catch (error) {
|
|
1486
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
783
1487
|
const message = error instanceof Error
|
|
784
1488
|
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
785
1489
|
.filter(Boolean)
|
|
@@ -808,13 +1512,15 @@ function hasHeader(headers, name) {
|
|
|
808
1512
|
const needle = name.toLowerCase();
|
|
809
1513
|
return Object.keys(headers).some((key) => key.toLowerCase() === needle);
|
|
810
1514
|
}
|
|
811
|
-
export function createStealthClient(baseUrl,
|
|
812
|
-
|
|
813
|
-
|
|
1515
|
+
export function createStealthClient(baseUrl, clientOptions = {}) {
|
|
1516
|
+
if (typeof clientOptions === "string") {
|
|
1517
|
+
resolveStealthProfileSelection(clientOptions);
|
|
1518
|
+
}
|
|
1519
|
+
const defaultProfile = resolveStealthProfileSelection(clientOptions.stealth);
|
|
814
1520
|
let sharedSession = null;
|
|
815
1521
|
function getSharedSession() {
|
|
816
1522
|
if (!sharedSession) {
|
|
817
|
-
sharedSession = createSessionFetcher(baseUrl, defaultProfile,
|
|
1523
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, clientOptions);
|
|
818
1524
|
}
|
|
819
1525
|
return sharedSession;
|
|
820
1526
|
}
|
|
@@ -823,8 +1529,8 @@ export function createStealthClient(baseUrl, defaultProfileOrOptions = DEFAULT_P
|
|
|
823
1529
|
return getSharedSession().fetch(url, options);
|
|
824
1530
|
},
|
|
825
1531
|
createSession(opts) {
|
|
826
|
-
const sessionProfile = opts?.
|
|
827
|
-
return createSessionFetcher(baseUrl, sessionProfile,
|
|
1532
|
+
const sessionProfile = resolveStealthProfileSelection(opts?.stealth, defaultProfile);
|
|
1533
|
+
return createSessionFetcher(baseUrl, sessionProfile, clientOptions);
|
|
828
1534
|
},
|
|
829
1535
|
close() {
|
|
830
1536
|
sharedSession?.close();
|