@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +537 -33
- package/CHANGELOG.md +201 -1
- package/README.md +83 -22
- package/SUBMISSION.md +2 -2
- package/bin/apifuse-check.ts +165 -11
- package/bin/apifuse-dev.ts +57 -13
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-migrate-shape.ts +202 -0
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +357 -38
- package/bin/apifuse-perf.ts +14 -13
- package/bin/apifuse-record.ts +709 -72
- package/bin/apifuse-submit-check.ts +2344 -324
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +27 -0
- package/dist/cli/create.d.ts +4 -1
- package/dist/cli/create.js +42 -37
- package/dist/cli/migrate-operation-declaration.d.ts +59 -0
- package/dist/cli/migrate-operation-declaration.js +1178 -0
- package/dist/cli/migrate-operation-shape.d.ts +44 -0
- package/dist/cli/migrate-operation-shape.js +113 -0
- package/dist/cli/migrate-provider-shape.d.ts +52 -0
- package/dist/cli/migrate-provider-shape.js +578 -0
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +5 -5
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/dist/cli/templates/provider/provider.json.tpl +6 -0
- package/dist/config/loader.d.ts +179 -16
- package/dist/config/loader.js +435 -127
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +12 -5
- package/dist/contract.js +24 -10
- package/dist/declaration-validation.d.ts +33 -0
- package/dist/declaration-validation.js +266 -0
- package/dist/define.d.ts +66 -29
- package/dist/define.js +778 -69
- package/dist/dev.d.ts +3 -0
- package/dist/dev.js +1 -1
- package/dist/engine.d.ts +78 -0
- package/dist/engine.js +133 -0
- package/dist/error-observability.d.ts +7 -0
- package/dist/error-observability.js +61 -0
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +122 -0
- package/dist/errors.d.ts +33 -0
- package/dist/errors.js +40 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +227 -0
- package/dist/health-scenario.d.ts +1842 -0
- package/dist/health-scenario.js +624 -0
- package/dist/index.d.ts +21 -10
- package/dist/index.js +12 -7
- package/dist/lint.d.ts +23 -11
- package/dist/lint.js +572 -69
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +10 -2
- package/dist/provider.js +4 -1
- package/dist/runtime/auth-flow.d.ts +5 -1
- package/dist/runtime/auth-flow.js +6 -0
- package/dist/runtime/browser.d.ts +1 -0
- package/dist/runtime/browser.js +492 -49
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +13 -1
- package/dist/runtime/choice.js +490 -102
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/env.js +12 -0
- package/dist/runtime/executor.d.ts +3 -2
- package/dist/runtime/executor.js +26 -25
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +515 -53
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +366 -8
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +397 -16
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-telemetry.d.ts +80 -1
- package/dist/runtime/proxy-telemetry.js +154 -47
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +14 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +22 -0
- package/dist/runtime/resolver-vendors/browser.js +377 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +531 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +408 -0
- package/dist/runtime/resolver-vendors/types.d.ts +94 -0
- package/dist/runtime/resolver-vendors/types.js +96 -0
- package/dist/runtime/resolver.d.ts +72 -0
- package/dist/runtime/resolver.js +771 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +277 -71
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +41 -5
- package/dist/runtime/stealth.js +997 -291
- package/dist/runtime/stt.js +1 -12
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +62 -0
- package/dist/runtime/trace.d.ts +5 -0
- package/dist/runtime/trace.js +43 -10
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/error-observability.d.ts +1 -0
- package/dist/server/error-observability.js +1 -0
- package/dist/server/index.d.ts +5 -2
- package/dist/server/index.js +2 -2
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test.d.ts +14 -3
- package/dist/server/self-test.js +129 -58
- package/dist/server/serve-implementation.d.ts +230 -0
- package/dist/server/serve-implementation.js +2265 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1131
- package/dist/server/trace-output.d.ts +6 -0
- package/dist/server/trace-output.js +79 -0
- package/dist/server/types.d.ts +30 -5
- package/dist/server/types.js +13 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +9 -4
- package/dist/stealth/profiles.js +160 -211
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +489 -21
- package/dist/trace-sanitization.d.ts +5 -0
- package/dist/trace-sanitization.js +45 -0
- package/dist/types.d.ts +673 -131
- package/dist/types.js +1 -0
- package/package.json +45 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/approval-override.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/codemod-syntax.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/connection-precedence.ts.txt +10 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/docs-conflict.ts.txt +8 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-map.ts.txt +5 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-operation.ts.txt +16 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/factory-map.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoist-all.ts.txt +31 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoisted-const.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/imported-spread.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-map.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-cast-tail.ts.txt +21 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-ekitan.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-override.ts.txt +14 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/missing-english-locale.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/no-safety.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/non-literal.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/redundant-approval.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/safety-conflict.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/stream.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/tool-router-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/unparseable.ts.txt +4 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/verbatim-template.ts.txt +12 -0
- package/src/cli/commands.ts +33 -0
- package/src/cli/create.ts +50 -37
- package/src/cli/migrate-operation-declaration.ts +1654 -0
- package/src/cli/migrate-operation-shape.ts +184 -0
- package/src/cli/migrate-provider-shape.ts +772 -0
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +5 -5
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/src/cli/templates/provider/provider.json.tpl +6 -0
- package/src/config/loader.ts +690 -163
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +12 -5
- package/src/contract.ts +24 -10
- package/src/declaration-validation.ts +330 -0
- package/src/define.ts +1048 -131
- package/src/dev.ts +4 -1
- package/src/engine.ts +279 -0
- package/src/error-observability.ts +64 -0
- package/src/error-resolution.ts +127 -0
- package/src/errors.ts +68 -0
- package/src/fixture-sanitization.ts +264 -0
- package/src/health-scenario.ts +875 -0
- package/src/index.ts +220 -13
- package/src/lint.ts +698 -90
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +161 -3
- package/src/runtime/auth-flow.ts +12 -0
- package/src/runtime/browser.ts +661 -63
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +631 -120
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/env.ts +13 -0
- package/src/runtime/executor.ts +43 -31
- package/src/runtime/http.ts +641 -61
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/instrumentation.ts +520 -15
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/otlp.ts +467 -21
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-telemetry.ts +244 -75
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +23 -0
- package/src/runtime/resolver-shared.ts +33 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +533 -0
- package/src/runtime/resolver-vendors/capsolver.ts +707 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +540 -0
- package/src/runtime/resolver-vendors/types.ts +212 -0
- package/src/runtime/resolver.ts +1157 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +394 -77
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +1280 -336
- package/src/runtime/stt.ts +1 -19
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +78 -0
- package/src/runtime/trace.ts +57 -17
- package/src/serve.ts +6 -1
- package/src/server/error-observability.ts +1 -0
- package/src/server/index.ts +34 -2
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test.ts +191 -68
- package/src/server/serve-implementation.ts +3378 -0
- package/src/server/serve.ts +1 -1632
- package/src/server/trace-output.ts +129 -0
- package/src/server/types.ts +13 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +202 -231
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +658 -15
- package/src/trace-sanitization.ts +63 -0
- package/src/types.ts +788 -174
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/runtime/stealth.ts
CHANGED
|
@@ -1,26 +1,40 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import type {
|
|
3
|
-
|
|
2
|
+
import type {
|
|
3
|
+
BrowserProfile,
|
|
4
|
+
EmulationOS,
|
|
5
|
+
RequestInit as WreqRequestInit,
|
|
6
|
+
Session as WreqSession,
|
|
7
|
+
} from "wreq-js";
|
|
4
8
|
|
|
5
|
-
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
9
|
+
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
6
10
|
import {
|
|
7
11
|
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
8
12
|
invalidateProxyResolutionCacheAsync,
|
|
9
13
|
ProxyResolutionError,
|
|
14
|
+
policyResolvesRegistryVendorChain,
|
|
15
|
+
resolvePolicyProxyPoolSpan,
|
|
16
|
+
resolvePolicyTransportAttemptCap,
|
|
10
17
|
resolveProxyConfigAsync,
|
|
11
|
-
|
|
18
|
+
vendorFromResolvedSource,
|
|
12
19
|
} from "../config/loader.js";
|
|
13
20
|
import { SDKError, TransportError } from "../errors.js";
|
|
14
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
DEFAULT_STEALTH_BROWSER,
|
|
23
|
+
DEFAULT_STEALTH_OS,
|
|
24
|
+
getStealthProfile,
|
|
25
|
+
resolveStealthProfileSelection,
|
|
26
|
+
} from "../stealth/profiles.js";
|
|
15
27
|
import type {
|
|
16
|
-
CookieJar,
|
|
17
28
|
HttpMethod,
|
|
18
29
|
StealthClient,
|
|
19
30
|
StealthFetchOptions,
|
|
31
|
+
StealthProfileDescriptor,
|
|
32
|
+
StealthProfileSelection,
|
|
20
33
|
StealthRedirectHop,
|
|
21
34
|
StealthResponse,
|
|
22
35
|
StealthSession,
|
|
23
36
|
} from "../types.js";
|
|
37
|
+
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
24
38
|
import {
|
|
25
39
|
createProxyAuthIpDeniedError,
|
|
26
40
|
createProxyEdgeAuthRejectedError,
|
|
@@ -44,14 +58,31 @@ import {
|
|
|
44
58
|
shouldRetryProxyTransportAttempt,
|
|
45
59
|
validateUnsafeProxyTransportRetryMethods,
|
|
46
60
|
} from "./proxy-retry-policy.js";
|
|
47
|
-
import {
|
|
48
|
-
|
|
49
|
-
|
|
61
|
+
import {
|
|
62
|
+
evaluateRedirectHop,
|
|
63
|
+
isRedirectStatus,
|
|
64
|
+
nextRedirectMethod,
|
|
65
|
+
resolveRedirectUrl,
|
|
66
|
+
} from "./redirects.js";
|
|
67
|
+
import {
|
|
68
|
+
isSensitiveKey,
|
|
69
|
+
normalizeSensitiveParams,
|
|
70
|
+
redactSensitiveError,
|
|
71
|
+
redactSensitiveRequestError,
|
|
72
|
+
redactSensitiveText,
|
|
73
|
+
redactUrlQueryParams,
|
|
74
|
+
serializeRequestUrl,
|
|
75
|
+
} from "./request-options.js";
|
|
76
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
77
|
+
|
|
78
|
+
export const DEFAULT_STEALTH_PROFILE: StealthProfileDescriptor = Object.freeze({
|
|
79
|
+
browser: DEFAULT_STEALTH_BROWSER,
|
|
80
|
+
os: DEFAULT_STEALTH_OS,
|
|
81
|
+
});
|
|
50
82
|
|
|
51
83
|
const MISSING_PROXY_WARNING =
|
|
52
84
|
"[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
53
85
|
|
|
54
|
-
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
55
86
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
56
87
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
57
88
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
@@ -59,9 +90,39 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
59
90
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
60
91
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
61
92
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
93
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
94
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
95
|
+
"content-encoding",
|
|
96
|
+
"content-language",
|
|
97
|
+
"content-location",
|
|
98
|
+
"content-type",
|
|
99
|
+
]);
|
|
100
|
+
|
|
101
|
+
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
102
|
+
return typeof value === "object" && value !== null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function sensitiveQueryParamNames(url: string): string[] {
|
|
106
|
+
const queryStart = url.indexOf("?");
|
|
107
|
+
if (queryStart === -1) return [];
|
|
108
|
+
const fragmentStart = url.indexOf("#", queryStart);
|
|
109
|
+
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
110
|
+
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
111
|
+
}
|
|
62
112
|
|
|
63
113
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
64
114
|
warn?: (message: string) => void;
|
|
115
|
+
/** Abort all requests issued by this client. */
|
|
116
|
+
signal?: AbortSignal;
|
|
117
|
+
/** Browser identity and declaration-wide HTTP language defaults. */
|
|
118
|
+
stealth?: StealthProfileSelection & {
|
|
119
|
+
/**
|
|
120
|
+
* Default Accept-Language value, emitted where Chrome's network layer appends
|
|
121
|
+
* it (after Accept-Encoding). Requests may override it through `headers`, in
|
|
122
|
+
* which case it is placed like any other caller header.
|
|
123
|
+
*/
|
|
124
|
+
acceptLanguage?: string;
|
|
125
|
+
};
|
|
65
126
|
/**
|
|
66
127
|
* Proxy-only stealth transport overrides. Use only for upstream proxy products
|
|
67
128
|
* that terminate CONNECT with a private CA instead of tunneling the origin
|
|
@@ -70,225 +131,471 @@ export type StealthClientOptions = ProxyResolutionOptions & {
|
|
|
70
131
|
proxyStealth?: { insecureSkipVerify?: boolean };
|
|
71
132
|
};
|
|
72
133
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
"chrome-130",
|
|
78
|
-
"chrome-131",
|
|
79
|
-
"chrome-133",
|
|
80
|
-
"chrome-144",
|
|
81
|
-
"chrome-146-psk",
|
|
82
|
-
"chrome-131-psk",
|
|
83
|
-
"chrome-130-psk",
|
|
84
|
-
"edge-131",
|
|
85
|
-
]);
|
|
86
|
-
|
|
87
|
-
type ImpitBrowser = Browser;
|
|
88
|
-
type ImpitRequestInit = RequestInit;
|
|
89
|
-
|
|
90
|
-
const CHROME_IMPIT_BY_MAJOR: Record<number, ImpitBrowser> = {
|
|
91
|
-
100: "chrome100",
|
|
92
|
-
101: "chrome101",
|
|
93
|
-
104: "chrome104",
|
|
94
|
-
107: "chrome107",
|
|
95
|
-
110: "chrome110",
|
|
96
|
-
116: "chrome116",
|
|
97
|
-
124: "chrome124",
|
|
98
|
-
125: "chrome125",
|
|
99
|
-
131: "chrome131",
|
|
100
|
-
136: "chrome136",
|
|
101
|
-
142: "chrome142",
|
|
134
|
+
type StealthTransportHeaders = {
|
|
135
|
+
entries(): IterableIterator<[string, string]>;
|
|
136
|
+
get(name: string): string | null;
|
|
137
|
+
getSetCookie?: () => string[];
|
|
102
138
|
};
|
|
103
139
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
140
|
+
type StealthTransportBody = {
|
|
141
|
+
cancel(): Promise<void>;
|
|
142
|
+
getReader(): {
|
|
143
|
+
read(): Promise<{ done: boolean; value?: Uint8Array }>;
|
|
144
|
+
cancel(): Promise<void>;
|
|
145
|
+
releaseLock(): void;
|
|
146
|
+
};
|
|
109
147
|
};
|
|
110
148
|
|
|
111
|
-
type StealthTransportResponse =
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
149
|
+
type StealthTransportResponse = {
|
|
150
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
151
|
+
headers: StealthTransportHeaders;
|
|
152
|
+
status: number;
|
|
153
|
+
body?: StealthTransportBody | null;
|
|
115
154
|
url?: string;
|
|
116
155
|
redirected?: boolean;
|
|
117
156
|
};
|
|
118
157
|
|
|
119
|
-
type StealthMethod =
|
|
120
|
-
type StealthRequestInit =
|
|
158
|
+
type StealthMethod = HttpMethod | "TRACE";
|
|
159
|
+
type StealthRequestInit = WreqRequestInit & {
|
|
121
160
|
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
122
161
|
};
|
|
123
162
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
163
|
+
type WreqSessionCacheEntry = {
|
|
164
|
+
session: Promise<WreqSession>;
|
|
165
|
+
tail: Promise<void>;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
type WreqModule = typeof import("wreq-js");
|
|
127
169
|
|
|
128
|
-
|
|
129
|
-
private readonly cookies: Record<string, string>;
|
|
170
|
+
let wreqModulePromise: Promise<WreqModule> | undefined;
|
|
130
171
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
172
|
+
function getWreqModule(): Promise<WreqModule> {
|
|
173
|
+
if (!wreqModulePromise) {
|
|
174
|
+
wreqModulePromise = import("wreq-js").catch((error: unknown) => {
|
|
175
|
+
throw new SDKError(
|
|
176
|
+
`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`,
|
|
177
|
+
{
|
|
178
|
+
code: "stealth_transport_unavailable",
|
|
179
|
+
cause: error instanceof Error ? error : undefined,
|
|
180
|
+
},
|
|
181
|
+
);
|
|
182
|
+
});
|
|
134
183
|
}
|
|
135
184
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
const [nameValue] = cookieString.split(";");
|
|
139
|
-
if (!nameValue) {
|
|
140
|
-
continue;
|
|
141
|
-
}
|
|
185
|
+
return wreqModulePromise;
|
|
186
|
+
}
|
|
142
187
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
188
|
+
function parseProfileIdentifier(identifier: string): {
|
|
189
|
+
family: string;
|
|
190
|
+
version: number[];
|
|
191
|
+
} | null {
|
|
192
|
+
const match =
|
|
193
|
+
/^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(
|
|
194
|
+
identifier.toLowerCase(),
|
|
195
|
+
);
|
|
196
|
+
if (!match?.[1] || !match[2]) return null;
|
|
197
|
+
return {
|
|
198
|
+
family: match[1],
|
|
199
|
+
version: match[2].split(/[._]/).map(Number),
|
|
200
|
+
};
|
|
201
|
+
}
|
|
147
202
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
203
|
+
function compareVersionDistance(target: number[], left: number[], right: number[]): number {
|
|
204
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
205
|
+
for (let index = 0; index < width; index += 1) {
|
|
206
|
+
const targetPart = target[index] ?? 0;
|
|
207
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
208
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
209
|
+
if (leftDistance !== rightDistance) return leftDistance - rightDistance;
|
|
152
210
|
}
|
|
211
|
+
return 0;
|
|
212
|
+
}
|
|
153
213
|
|
|
154
|
-
|
|
155
|
-
|
|
214
|
+
function closestWreqProfile(
|
|
215
|
+
identifier: string,
|
|
216
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
217
|
+
): BrowserProfile | undefined {
|
|
218
|
+
const requested = parseProfileIdentifier(identifier);
|
|
219
|
+
if (!requested) return undefined;
|
|
220
|
+
|
|
221
|
+
let closest: { name: BrowserProfile; version: number[] } | undefined;
|
|
222
|
+
for (const candidateName of wreqProfiles) {
|
|
223
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
224
|
+
if (!candidate || candidate.family !== requested.family) continue;
|
|
225
|
+
if (
|
|
226
|
+
!closest ||
|
|
227
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0
|
|
228
|
+
) {
|
|
229
|
+
closest = { name: candidateName, version: candidate.version };
|
|
230
|
+
}
|
|
156
231
|
}
|
|
232
|
+
return closest?.name;
|
|
233
|
+
}
|
|
157
234
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
235
|
+
export function resolveWreqProfile(
|
|
236
|
+
selection: StealthProfileSelection,
|
|
237
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
238
|
+
): {
|
|
239
|
+
browser: BrowserProfile;
|
|
240
|
+
os: EmulationOS;
|
|
241
|
+
} {
|
|
242
|
+
const profile = getStealthProfile(selection);
|
|
243
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
244
|
+
const os = profile.os;
|
|
161
245
|
|
|
162
|
-
|
|
163
|
-
|
|
246
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
247
|
+
if (!browser) {
|
|
248
|
+
throw new SDKError(
|
|
249
|
+
`Stealth profile ${profile.browser}/${profile.os} cannot be mapped to a wreq-js browser profile.`,
|
|
250
|
+
);
|
|
164
251
|
}
|
|
252
|
+
return { browser, os };
|
|
253
|
+
}
|
|
165
254
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
.join("; ");
|
|
170
|
-
}
|
|
255
|
+
function resolveUrl(baseUrl: string, url: string): string {
|
|
256
|
+
return new URL(url, baseUrl).toString();
|
|
257
|
+
}
|
|
171
258
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
259
|
+
function headerEntriesFromHeaders(headers: StealthTransportHeaders): [string, string][] {
|
|
260
|
+
return Array.from(headers.entries());
|
|
261
|
+
}
|
|
175
262
|
|
|
176
|
-
|
|
177
|
-
return this.getAll();
|
|
178
|
-
}
|
|
263
|
+
type HeaderTuple = [string, string];
|
|
179
264
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
265
|
+
function normalizeHeaders(
|
|
266
|
+
headers: Record<string, string | string[] | undefined>,
|
|
267
|
+
): Record<string, string> {
|
|
268
|
+
return Object.fromEntries(normalizedHeaderEntries(headers));
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function trimOuterHttpWhitespace(value: string): string {
|
|
272
|
+
return value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, "");
|
|
273
|
+
}
|
|
186
274
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
275
|
+
function normalizedHeaderEntries(
|
|
276
|
+
headers: Record<string, string | string[] | undefined>,
|
|
277
|
+
): HeaderTuple[] {
|
|
278
|
+
const entries: HeaderTuple[] = [];
|
|
279
|
+
const indices = new Map<string, number>();
|
|
280
|
+
for (const [originalName, originalValue] of Object.entries(headers)) {
|
|
281
|
+
if (originalValue === undefined) continue;
|
|
282
|
+
const name = originalName.toLowerCase();
|
|
283
|
+
const value = (Array.isArray(originalValue) ? originalValue : [originalValue])
|
|
284
|
+
.map(trimOuterHttpWhitespace)
|
|
285
|
+
.join(", ");
|
|
286
|
+
const existingIndex = indices.get(name);
|
|
287
|
+
if (existingIndex === undefined) {
|
|
288
|
+
indices.set(name, entries.length);
|
|
289
|
+
entries.push([name, value]);
|
|
290
|
+
continue;
|
|
190
291
|
}
|
|
292
|
+
const existing = entries[existingIndex];
|
|
293
|
+
if (existing) existing[1] = `${existing[1]}, ${value}`;
|
|
191
294
|
}
|
|
295
|
+
return entries;
|
|
296
|
+
}
|
|
192
297
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
298
|
+
const SDK_OWNED_EXACT_CHROME_HEADERS = new Set([
|
|
299
|
+
"host",
|
|
300
|
+
"connection",
|
|
301
|
+
"user-agent",
|
|
302
|
+
"sec-ch-ua",
|
|
303
|
+
"sec-ch-ua-mobile",
|
|
304
|
+
"sec-ch-ua-platform",
|
|
305
|
+
"accept-encoding",
|
|
306
|
+
]);
|
|
200
307
|
|
|
201
|
-
|
|
308
|
+
// Non-pseudo-header order of real Chrome 149. The fixture captures
|
|
309
|
+
// (chrome-ground-truth-capture.json, chrome-extended-capture.json,
|
|
310
|
+
// h1-casing-capture.json) were taken through Playwright's `locale` option, which
|
|
311
|
+
// installs Accept-Language via DevTools next to User-Agent; real Chrome only
|
|
312
|
+
// receives it from //net (URLRequestHttpJob::AddExtraHeaders), after
|
|
313
|
+
// Accept-Encoding and before Cookie, as al-placement-capture.json B/C show.
|
|
314
|
+
const CHROME_HEADER_ORDERS = {
|
|
315
|
+
navigation: [
|
|
316
|
+
"sec-ch-ua",
|
|
317
|
+
"sec-ch-ua-mobile",
|
|
318
|
+
"sec-ch-ua-platform",
|
|
319
|
+
"upgrade-insecure-requests",
|
|
320
|
+
"user-agent",
|
|
321
|
+
"accept",
|
|
322
|
+
"sec-fetch-site",
|
|
323
|
+
"sec-fetch-mode",
|
|
324
|
+
"sec-fetch-user",
|
|
325
|
+
"sec-fetch-dest",
|
|
326
|
+
"accept-encoding",
|
|
327
|
+
"accept-language",
|
|
328
|
+
"cookie",
|
|
329
|
+
"priority",
|
|
330
|
+
],
|
|
331
|
+
// xhr has no fixed table: it is emulated by chrome149HeaderOrder.
|
|
332
|
+
post: [
|
|
333
|
+
"content-length",
|
|
334
|
+
"sec-ch-ua-platform",
|
|
335
|
+
"user-agent",
|
|
336
|
+
"sec-ch-ua",
|
|
337
|
+
"content-type",
|
|
338
|
+
"sec-ch-ua-mobile",
|
|
339
|
+
"accept",
|
|
340
|
+
"origin",
|
|
341
|
+
"sec-fetch-site",
|
|
342
|
+
"sec-fetch-mode",
|
|
343
|
+
"sec-fetch-dest",
|
|
344
|
+
"referer",
|
|
345
|
+
"accept-encoding",
|
|
346
|
+
"accept-language",
|
|
347
|
+
"cookie",
|
|
348
|
+
"priority",
|
|
349
|
+
],
|
|
350
|
+
} as const;
|
|
351
|
+
|
|
352
|
+
const CHROME_H1_HEADER_ORDERS = {
|
|
353
|
+
navigation: [
|
|
354
|
+
"host",
|
|
355
|
+
"connection",
|
|
356
|
+
"sec-ch-ua",
|
|
357
|
+
"sec-ch-ua-mobile",
|
|
358
|
+
"sec-ch-ua-platform",
|
|
359
|
+
"upgrade-insecure-requests",
|
|
360
|
+
"user-agent",
|
|
361
|
+
"accept",
|
|
362
|
+
"sec-fetch-site",
|
|
363
|
+
"sec-fetch-mode",
|
|
364
|
+
"sec-fetch-user",
|
|
365
|
+
"sec-fetch-dest",
|
|
366
|
+
"accept-encoding",
|
|
367
|
+
"accept-language",
|
|
368
|
+
"cookie",
|
|
369
|
+
],
|
|
370
|
+
xhr: [
|
|
371
|
+
"host",
|
|
372
|
+
"connection",
|
|
373
|
+
"sec-ch-ua-platform",
|
|
374
|
+
"cache-control",
|
|
375
|
+
"x-requested-with",
|
|
376
|
+
"user-agent",
|
|
377
|
+
"sec-ch-ua",
|
|
378
|
+
"sec-ch-ua-mobile",
|
|
379
|
+
"accept",
|
|
380
|
+
"sec-fetch-site",
|
|
381
|
+
"sec-fetch-mode",
|
|
382
|
+
"sec-fetch-dest",
|
|
383
|
+
"referer",
|
|
384
|
+
"accept-encoding",
|
|
385
|
+
"accept-language",
|
|
386
|
+
"cookie",
|
|
387
|
+
],
|
|
388
|
+
post: [
|
|
389
|
+
"host",
|
|
390
|
+
"connection",
|
|
391
|
+
"content-length",
|
|
392
|
+
"sec-ch-ua-platform",
|
|
393
|
+
"user-agent",
|
|
394
|
+
"sec-ch-ua",
|
|
395
|
+
"content-type",
|
|
396
|
+
"sec-ch-ua-mobile",
|
|
397
|
+
"accept",
|
|
398
|
+
"origin",
|
|
399
|
+
"sec-fetch-site",
|
|
400
|
+
"sec-fetch-mode",
|
|
401
|
+
"sec-fetch-dest",
|
|
402
|
+
"referer",
|
|
403
|
+
"accept-encoding",
|
|
404
|
+
"accept-language",
|
|
405
|
+
"cookie",
|
|
406
|
+
],
|
|
407
|
+
} as const;
|
|
408
|
+
|
|
409
|
+
const CHROME_H1_HEADER_NAMES: Record<string, string> = {
|
|
410
|
+
host: "Host",
|
|
411
|
+
connection: "Connection",
|
|
412
|
+
"upgrade-insecure-requests": "Upgrade-Insecure-Requests",
|
|
413
|
+
"user-agent": "User-Agent",
|
|
414
|
+
"accept-language": "Accept-Language",
|
|
415
|
+
accept: "Accept",
|
|
416
|
+
"sec-fetch-site": "Sec-Fetch-Site",
|
|
417
|
+
"sec-fetch-mode": "Sec-Fetch-Mode",
|
|
418
|
+
"sec-fetch-user": "Sec-Fetch-User",
|
|
419
|
+
"sec-fetch-dest": "Sec-Fetch-Dest",
|
|
420
|
+
"accept-encoding": "Accept-Encoding",
|
|
421
|
+
cookie: "Cookie",
|
|
422
|
+
"cache-control": "Cache-Control",
|
|
423
|
+
"x-requested-with": "X-Requested-With",
|
|
424
|
+
referer: "Referer",
|
|
425
|
+
"content-length": "Content-Length",
|
|
426
|
+
"content-type": "Content-Type",
|
|
427
|
+
origin: "Origin",
|
|
428
|
+
range: "Range",
|
|
429
|
+
};
|
|
430
|
+
|
|
431
|
+
type ChromeRequestClass = keyof typeof CHROME_HEADER_ORDERS | "xhr";
|
|
432
|
+
|
|
433
|
+
function chromeHeaderOrder(
|
|
434
|
+
requestClass: ChromeRequestClass,
|
|
435
|
+
isHttp1: boolean,
|
|
436
|
+
caller: ReadonlyMap<string, string>,
|
|
437
|
+
): readonly string[] {
|
|
438
|
+
if (isHttp1) return CHROME_H1_HEADER_ORDERS[requestClass];
|
|
439
|
+
if (requestClass !== "xhr") return CHROME_HEADER_ORDERS[requestClass];
|
|
440
|
+
// Cookie and Referer are forbidden Fetch headers, so they never enter the page
|
|
441
|
+
// Fetch Headers map the emulator models. Range does occupy a map bucket
|
|
442
|
+
// (m1-capture.json range_*), but HttpCache::Transaction removes it from the
|
|
443
|
+
// request headers and PartialData re-adds it at the tail.
|
|
444
|
+
const order = chrome149HeaderOrder(
|
|
445
|
+
[...caller.keys()].filter((name) => name !== "cookie" && name !== "referer"),
|
|
446
|
+
);
|
|
447
|
+
if (caller.has("range")) order.splice(order.indexOf("range"), 1);
|
|
448
|
+
// //net appends Accept-Encoding and, unless the caller supplied one,
|
|
449
|
+
// Accept-Language (URLRequestHttpJob::AddExtraHeaders), then writes the Cookie
|
|
450
|
+
// header (SetCookieHeaderAndStart); the HTTP cache re-adds a caller Range after
|
|
451
|
+
// that, and the HTTP/2 Priority header stays last.
|
|
452
|
+
let insertAt = order.indexOf("accept-encoding") + 1;
|
|
453
|
+
if (order[insertAt] === "accept-language") insertAt += 1;
|
|
454
|
+
for (const name of ["cookie", "range"] as const) {
|
|
455
|
+
if (!caller.has(name)) continue;
|
|
456
|
+
order.splice(insertAt, 0, name);
|
|
457
|
+
insertAt += 1;
|
|
202
458
|
}
|
|
459
|
+
return order;
|
|
203
460
|
}
|
|
204
461
|
|
|
205
|
-
function
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
462
|
+
function normalizedCallerHeaderEntries(
|
|
463
|
+
headers: Record<string, string | string[] | undefined>,
|
|
464
|
+
): HeaderTuple[] {
|
|
465
|
+
const entries = normalizedHeaderEntries(headers);
|
|
466
|
+
assertCallerHeadersSupported(entries);
|
|
467
|
+
return entries;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function normalizedCallerHeaderEntriesFromRecord(headers: Record<string, string>): HeaderTuple[] {
|
|
471
|
+
// The record has been through normalizeHeaders (lowercase, merged names); the
|
|
472
|
+
// lowercase here only guards later insertions such as the cookie jar's header.
|
|
473
|
+
const entries: HeaderTuple[] = Object.entries(headers).map(([name, value]) => [
|
|
474
|
+
name.toLowerCase(),
|
|
475
|
+
value,
|
|
476
|
+
]);
|
|
477
|
+
assertCallerHeadersSupported(entries);
|
|
478
|
+
return entries;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function assertCallerHeadersSupported(entries: readonly HeaderTuple[]): void {
|
|
482
|
+
for (const [name] of entries) {
|
|
483
|
+
if (SDK_OWNED_EXACT_CHROME_HEADERS.has(name) || name.startsWith("sec-fetch-")) {
|
|
484
|
+
throw new SDKError(`Stealth transport owns the "${name}" header; remove it from headers.`, {
|
|
485
|
+
code: "STEALTH_HEADER_OVERRIDE_UNSUPPORTED",
|
|
486
|
+
});
|
|
219
487
|
}
|
|
220
488
|
}
|
|
221
|
-
return closestBrowser ?? "chrome142";
|
|
222
489
|
}
|
|
223
490
|
|
|
224
|
-
function
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
491
|
+
function chromeRequestClass(
|
|
492
|
+
method: StealthMethod,
|
|
493
|
+
requestedClass?: ChromeRequestClass,
|
|
494
|
+
): ChromeRequestClass {
|
|
495
|
+
if (requestedClass) return requestedClass;
|
|
496
|
+
if (method === "POST") return "post";
|
|
497
|
+
return "navigation";
|
|
498
|
+
}
|
|
228
499
|
|
|
229
|
-
|
|
500
|
+
function secFetchSite(requestUrl: string, referer: string | undefined): string {
|
|
501
|
+
if (!referer) return "none";
|
|
230
502
|
try {
|
|
231
|
-
|
|
503
|
+
return new URL(referer).origin === new URL(requestUrl).origin ? "same-origin" : "cross-site";
|
|
232
504
|
} catch {
|
|
233
|
-
|
|
234
|
-
// profile strings still run with the transport default instead of failing
|
|
235
|
-
// before the request starts. Removed built-in profile aliases above remain
|
|
236
|
-
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
237
|
-
return "chrome142";
|
|
505
|
+
return "cross-site";
|
|
238
506
|
}
|
|
507
|
+
}
|
|
239
508
|
|
|
240
|
-
|
|
241
|
-
const
|
|
242
|
-
if (
|
|
243
|
-
|
|
509
|
+
function requiredEmulationHeader(headers: ReadonlyMap<string, string>, name: string): string {
|
|
510
|
+
const value = headers.get(name);
|
|
511
|
+
if (value !== undefined) return value;
|
|
512
|
+
throw new SDKError(`wreq-js Chrome emulation exposes no ${name} header.`, {
|
|
513
|
+
code: "STEALTH_PROFILE_UNAVAILABLE",
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function buildChromeHeaderTuples(options: {
|
|
518
|
+
emulationHeaders: Iterable<[string, string]>;
|
|
519
|
+
method: StealthMethod;
|
|
520
|
+
body?: string;
|
|
521
|
+
headers: Record<string, string>;
|
|
522
|
+
requestUrl: string;
|
|
523
|
+
acceptLanguage?: string;
|
|
524
|
+
requestClass?: ChromeRequestClass;
|
|
525
|
+
}): HeaderTuple[] {
|
|
526
|
+
const callerEntries = normalizedCallerHeaderEntriesFromRecord(options.headers);
|
|
527
|
+
const caller = new Map(callerEntries);
|
|
528
|
+
const emulation = new Map(
|
|
529
|
+
Array.from(
|
|
530
|
+
options.emulationHeaders,
|
|
531
|
+
([name, value]) => [name.toLowerCase(), value] as HeaderTuple,
|
|
532
|
+
),
|
|
533
|
+
);
|
|
534
|
+
const requestClass = chromeRequestClass(options.method, options.requestClass);
|
|
535
|
+
const referer = caller.get("referer");
|
|
536
|
+
const fetchSite = secFetchSite(options.requestUrl, referer);
|
|
537
|
+
const isNavigation = requestClass === "navigation";
|
|
538
|
+
const values = new Map<string, string>([
|
|
539
|
+
["sec-ch-ua", requiredEmulationHeader(emulation, "sec-ch-ua")],
|
|
540
|
+
["sec-ch-ua-mobile", requiredEmulationHeader(emulation, "sec-ch-ua-mobile")],
|
|
541
|
+
["sec-ch-ua-platform", requiredEmulationHeader(emulation, "sec-ch-ua-platform")],
|
|
542
|
+
["user-agent", requiredEmulationHeader(emulation, "user-agent")],
|
|
543
|
+
[
|
|
544
|
+
"accept-language",
|
|
545
|
+
caller.get("accept-language") ??
|
|
546
|
+
options.acceptLanguage ??
|
|
547
|
+
requiredEmulationHeader(emulation, "accept-language"),
|
|
548
|
+
],
|
|
549
|
+
["accept", isNavigation ? requiredEmulationHeader(emulation, "accept") : "*/*"],
|
|
550
|
+
[
|
|
551
|
+
"accept-encoding",
|
|
552
|
+
caller.has("range") ? "identity" : requiredEmulationHeader(emulation, "accept-encoding"),
|
|
553
|
+
],
|
|
554
|
+
["priority", isNavigation ? requiredEmulationHeader(emulation, "priority") : "u=1, i"],
|
|
555
|
+
["sec-fetch-site", fetchSite],
|
|
556
|
+
["sec-fetch-mode", isNavigation ? "navigate" : "cors"],
|
|
557
|
+
["sec-fetch-dest", isNavigation ? "document" : "empty"],
|
|
558
|
+
]);
|
|
559
|
+
if (isNavigation) {
|
|
560
|
+
values.set("upgrade-insecure-requests", "1");
|
|
561
|
+
values.set("sec-fetch-user", "?1");
|
|
244
562
|
}
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
|
|
563
|
+
for (const [name, value] of callerEntries) values.set(name, value);
|
|
564
|
+
for (const name of ["content-type", "origin", "referer"] as const) {
|
|
565
|
+
const value = caller.get(name);
|
|
566
|
+
if (value !== undefined) values.set(name, value);
|
|
248
567
|
}
|
|
249
|
-
if (
|
|
250
|
-
|
|
251
|
-
|
|
568
|
+
if (requestClass === "post") {
|
|
569
|
+
values.set(
|
|
570
|
+
"content-length",
|
|
571
|
+
caller.get("content-length") ?? String(Buffer.byteLength(options.body ?? "")),
|
|
252
572
|
);
|
|
253
573
|
}
|
|
254
|
-
throw new SDKError(
|
|
255
|
-
`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`,
|
|
256
|
-
);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
function resolveUrl(baseUrl: string, url: string): string {
|
|
260
|
-
return new URL(url, baseUrl).toString();
|
|
261
|
-
}
|
|
262
574
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
const
|
|
271
|
-
for (const
|
|
575
|
+
const isHttp1 = new URL(options.requestUrl).protocol === "http:";
|
|
576
|
+
if (isHttp1) {
|
|
577
|
+
values.set("host", new URL(options.requestUrl).host);
|
|
578
|
+
values.set("connection", "keep-alive");
|
|
579
|
+
}
|
|
580
|
+
const order = chromeHeaderOrder(requestClass, isHttp1, caller);
|
|
581
|
+
const tuples: HeaderTuple[] = [];
|
|
582
|
+
const placed = new Set<string>();
|
|
583
|
+
for (const name of order) {
|
|
584
|
+
const value = values.get(name);
|
|
272
585
|
if (value === undefined) continue;
|
|
273
|
-
|
|
586
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
587
|
+
placed.add(name);
|
|
588
|
+
}
|
|
589
|
+
for (const [name, value] of callerEntries) {
|
|
590
|
+
if (placed.has(name)) continue;
|
|
591
|
+
tuples.push([isHttp1 ? (CHROME_H1_HEADER_NAMES[name] ?? name) : name, value]);
|
|
274
592
|
}
|
|
275
|
-
return
|
|
593
|
+
return tuples;
|
|
276
594
|
}
|
|
277
595
|
|
|
278
596
|
function hasOwn(object: object, key: string): boolean {
|
|
279
597
|
return Object.hasOwn(object, key);
|
|
280
598
|
}
|
|
281
|
-
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
282
|
-
return {
|
|
283
|
-
setCookie(cookie: string, _url: string, cb?: (error?: unknown) => void) {
|
|
284
|
-
cookieJar.setFromCookieStrings([cookie]);
|
|
285
|
-
if (typeof cb === "function") cb();
|
|
286
|
-
},
|
|
287
|
-
getCookieString(_url: string) {
|
|
288
|
-
return cookieJar.toString();
|
|
289
|
-
},
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
599
|
|
|
293
600
|
function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
294
601
|
if (!isRecord(options)) return;
|
|
@@ -300,17 +607,19 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
300
607
|
if (unsupported.length === 0) return;
|
|
301
608
|
|
|
302
609
|
throw new SDKError(
|
|
303
|
-
`ctx.stealth.fetch uses
|
|
610
|
+
`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use stealth.browser and stealth.os instead.`,
|
|
304
611
|
);
|
|
305
612
|
}
|
|
306
613
|
|
|
307
|
-
function responseHeadersToRecord(
|
|
614
|
+
function responseHeadersToRecord(
|
|
615
|
+
headers: StealthTransportHeaders,
|
|
616
|
+
): Record<string, string | string[] | undefined> {
|
|
308
617
|
const record: Record<string, string> = {};
|
|
309
618
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
310
619
|
return record;
|
|
311
620
|
}
|
|
312
621
|
|
|
313
|
-
function setCookieHeadersFromResponse(headers:
|
|
622
|
+
function setCookieHeadersFromResponse(headers: StealthTransportHeaders): string[] {
|
|
314
623
|
const getSetCookie = headers.getSetCookie;
|
|
315
624
|
if (typeof getSetCookie === "function") return getSetCookie.call(headers);
|
|
316
625
|
const setCookie = headers.get("set-cookie");
|
|
@@ -337,10 +646,26 @@ function splitCombinedSetCookieHeader(headerValue: string): string[] {
|
|
|
337
646
|
export async function normalizeResponse(
|
|
338
647
|
response: StealthTransportResponse,
|
|
339
648
|
requestUrl?: string,
|
|
649
|
+
maxBodyBytes?: number,
|
|
650
|
+
): Promise<StealthResponse> {
|
|
651
|
+
return normalizeResponseWithSignal(response, requestUrl, maxBodyBytes);
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async function normalizeResponseWithSignal(
|
|
655
|
+
response: StealthTransportResponse,
|
|
656
|
+
requestUrl?: string,
|
|
657
|
+
maxBodyBytes?: number,
|
|
658
|
+
signal?: AbortSignal,
|
|
340
659
|
): Promise<StealthResponse> {
|
|
341
660
|
const headers = Object.fromEntries(response.headers.entries());
|
|
342
|
-
const cookies = new
|
|
343
|
-
|
|
661
|
+
const cookies = new StealthCookieJar(
|
|
662
|
+
setCookieHeadersFromResponse(response.headers),
|
|
663
|
+
response.url ?? requestUrl,
|
|
664
|
+
);
|
|
665
|
+
const bodyBytes =
|
|
666
|
+
maxBodyBytes === undefined
|
|
667
|
+
? await readResponseArrayBuffer(response, signal)
|
|
668
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes, signal);
|
|
344
669
|
const body = new TextDecoder().decode(bodyBytes);
|
|
345
670
|
|
|
346
671
|
return {
|
|
@@ -368,6 +693,144 @@ export async function normalizeResponse(
|
|
|
368
693
|
};
|
|
369
694
|
}
|
|
370
695
|
|
|
696
|
+
async function readResponseArrayBuffer(
|
|
697
|
+
response: StealthTransportResponse,
|
|
698
|
+
signal?: AbortSignal,
|
|
699
|
+
): Promise<ArrayBuffer> {
|
|
700
|
+
if (!signal) return response.arrayBuffer();
|
|
701
|
+
throwIfAmbientAborted(signal);
|
|
702
|
+
return new Promise((resolve, reject) => {
|
|
703
|
+
let settled = false;
|
|
704
|
+
const settle = (operation: () => void) => {
|
|
705
|
+
if (settled) return;
|
|
706
|
+
settled = true;
|
|
707
|
+
signal.removeEventListener("abort", onAbort);
|
|
708
|
+
operation();
|
|
709
|
+
};
|
|
710
|
+
const onAbort = () => {
|
|
711
|
+
const error = toAmbientCancellationError(signal);
|
|
712
|
+
try {
|
|
713
|
+
void response.body?.cancel().catch(() => undefined);
|
|
714
|
+
} catch {
|
|
715
|
+
// Preserve the cancellation error if accessing or cancelling the body fails.
|
|
716
|
+
}
|
|
717
|
+
settle(() => reject(error));
|
|
718
|
+
};
|
|
719
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
720
|
+
try {
|
|
721
|
+
void response.arrayBuffer().then(
|
|
722
|
+
(body) => settle(() => resolve(body)),
|
|
723
|
+
(error) => settle(() => reject(error)),
|
|
724
|
+
);
|
|
725
|
+
} catch (error) {
|
|
726
|
+
settle(() => reject(error));
|
|
727
|
+
}
|
|
728
|
+
if (signal.aborted) onAbort();
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function responseTooLargeError(maxBodyBytes: number, observedBytes: number): TransportError {
|
|
733
|
+
return new TransportError(
|
|
734
|
+
`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`,
|
|
735
|
+
{
|
|
736
|
+
code: "response_too_large",
|
|
737
|
+
category: "upstream_http",
|
|
738
|
+
retryable: false,
|
|
739
|
+
status: 0,
|
|
740
|
+
},
|
|
741
|
+
);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
function declaredContentLength(headers: StealthTransportHeaders): number | undefined {
|
|
745
|
+
const contentLength = headers.get("content-length")?.trim();
|
|
746
|
+
if (!contentLength || !/^\d+$/.test(contentLength)) return undefined;
|
|
747
|
+
const parsed = Number(contentLength);
|
|
748
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
async function readResponseBodyWithLimit(
|
|
752
|
+
response: StealthTransportResponse,
|
|
753
|
+
maxBodyBytes: number,
|
|
754
|
+
signal?: AbortSignal,
|
|
755
|
+
): Promise<ArrayBuffer> {
|
|
756
|
+
throwIfAmbientAborted(signal);
|
|
757
|
+
const contentLength = declaredContentLength(response.headers);
|
|
758
|
+
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
759
|
+
await response.body?.cancel().catch(() => undefined);
|
|
760
|
+
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (!response.body) {
|
|
764
|
+
throw new TransportError("Response body stream is unavailable", {
|
|
765
|
+
code: "transport_stream_unavailable",
|
|
766
|
+
category: "upstream_http",
|
|
767
|
+
status: 0,
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
const reader = response.body.getReader();
|
|
772
|
+
const chunks: Uint8Array[] = [];
|
|
773
|
+
let receivedBytes = 0;
|
|
774
|
+
try {
|
|
775
|
+
while (true) {
|
|
776
|
+
const { done, value } = await readResponseBodyChunk(reader, signal);
|
|
777
|
+
if (done) break;
|
|
778
|
+
if (!value) continue;
|
|
779
|
+
receivedBytes += value.byteLength;
|
|
780
|
+
if (receivedBytes > maxBodyBytes) {
|
|
781
|
+
await reader.cancel().catch(() => undefined);
|
|
782
|
+
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
783
|
+
}
|
|
784
|
+
chunks.push(value);
|
|
785
|
+
}
|
|
786
|
+
} finally {
|
|
787
|
+
reader.releaseLock();
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
return concatenateResponseBodyChunks(chunks, receivedBytes);
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
function concatenateResponseBodyChunks(
|
|
794
|
+
chunks: readonly Uint8Array[],
|
|
795
|
+
receivedBytes: number,
|
|
796
|
+
): ArrayBuffer {
|
|
797
|
+
const bodyBytes = new Uint8Array(receivedBytes);
|
|
798
|
+
let offset = 0;
|
|
799
|
+
for (const chunk of chunks) {
|
|
800
|
+
bodyBytes.set(chunk, offset);
|
|
801
|
+
offset += chunk.byteLength;
|
|
802
|
+
}
|
|
803
|
+
return bodyBytes.buffer;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
function readResponseBodyChunk(
|
|
807
|
+
reader: ReturnType<StealthTransportBody["getReader"]>,
|
|
808
|
+
signal?: AbortSignal,
|
|
809
|
+
): Promise<{ done: boolean; value?: Uint8Array }> {
|
|
810
|
+
if (!signal) return reader.read();
|
|
811
|
+
throwIfAmbientAborted(signal);
|
|
812
|
+
return new Promise((resolve, reject) => {
|
|
813
|
+
let settled = false;
|
|
814
|
+
const settle = (operation: () => void) => {
|
|
815
|
+
if (settled) return;
|
|
816
|
+
settled = true;
|
|
817
|
+
signal.removeEventListener("abort", onAbort);
|
|
818
|
+
operation();
|
|
819
|
+
};
|
|
820
|
+
const onAbort = () => {
|
|
821
|
+
const error = toAmbientCancellationError(signal);
|
|
822
|
+
void reader.cancel().catch(() => undefined);
|
|
823
|
+
settle(() => reject(error));
|
|
824
|
+
};
|
|
825
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
826
|
+
void reader.read().then(
|
|
827
|
+
(chunk) => settle(() => resolve(chunk)),
|
|
828
|
+
(error) => settle(() => reject(error)),
|
|
829
|
+
);
|
|
830
|
+
if (signal.aborted) onAbort();
|
|
831
|
+
});
|
|
832
|
+
}
|
|
833
|
+
|
|
371
834
|
function normalizeBody(body: StealthFetchOptions["body"]): string {
|
|
372
835
|
if (body === undefined) {
|
|
373
836
|
return "";
|
|
@@ -426,6 +889,7 @@ type ResolvedAttemptProxy = {
|
|
|
426
889
|
url?: string;
|
|
427
890
|
poolIndex?: number;
|
|
428
891
|
proxyHash?: string;
|
|
892
|
+
vendor?: ProxyVendorName;
|
|
429
893
|
};
|
|
430
894
|
|
|
431
895
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -532,8 +996,41 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
532
996
|
});
|
|
533
997
|
}
|
|
534
998
|
|
|
535
|
-
function
|
|
536
|
-
|
|
999
|
+
function toAmbientCancellationError(
|
|
1000
|
+
signal: AbortSignal,
|
|
1001
|
+
error: unknown = signal.reason,
|
|
1002
|
+
): TransportError {
|
|
1003
|
+
if (error instanceof TransportError && error.code === "transport_cancelled") {
|
|
1004
|
+
return error;
|
|
1005
|
+
}
|
|
1006
|
+
return new TransportError("Request cancelled", {
|
|
1007
|
+
code: "transport_cancelled",
|
|
1008
|
+
status: 0,
|
|
1009
|
+
retryable: false,
|
|
1010
|
+
...(error !== undefined
|
|
1011
|
+
? { cause: error instanceof Error ? error : new Error(String(error)) }
|
|
1012
|
+
: {}),
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
function throwIfAmbientAborted(signal: AbortSignal | undefined): void {
|
|
1017
|
+
if (signal?.aborted) throw toAmbientCancellationError(signal);
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
1021
|
+
if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1022
|
+
throwIfAmbientAborted(signal);
|
|
1023
|
+
return new Promise((resolve, reject) => {
|
|
1024
|
+
const onAbort = () => {
|
|
1025
|
+
clearTimeout(timer);
|
|
1026
|
+
reject(toAmbientCancellationError(signal));
|
|
1027
|
+
};
|
|
1028
|
+
const timer = setTimeout(() => {
|
|
1029
|
+
signal.removeEventListener("abort", onAbort);
|
|
1030
|
+
resolve();
|
|
1031
|
+
}, ms);
|
|
1032
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1033
|
+
});
|
|
537
1034
|
}
|
|
538
1035
|
|
|
539
1036
|
function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
@@ -559,16 +1056,6 @@ function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
|
559
1056
|
}
|
|
560
1057
|
}
|
|
561
1058
|
|
|
562
|
-
function isRedirectStatus(status: number): boolean {
|
|
563
|
-
return [301, 302, 303, 307, 308].includes(status);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
function nextRedirectMethod(status: number, method: StealthMethod): StealthMethod {
|
|
567
|
-
if (status === 303 && method !== "HEAD") return "GET";
|
|
568
|
-
if ((status === 301 || status === 302) && method === "POST") return "GET";
|
|
569
|
-
return method;
|
|
570
|
-
}
|
|
571
|
-
|
|
572
1059
|
function locationHeader(headers: Record<string, string>): string | undefined {
|
|
573
1060
|
for (const [name, value] of Object.entries(headers)) {
|
|
574
1061
|
if (name.toLowerCase() === "location") return value;
|
|
@@ -576,56 +1063,260 @@ function locationHeader(headers: Record<string, string>): string | undefined {
|
|
|
576
1063
|
return undefined;
|
|
577
1064
|
}
|
|
578
1065
|
|
|
1066
|
+
function withoutRedirectBodyHeaders(headers: Record<string, string>): Record<string, string> {
|
|
1067
|
+
return Object.fromEntries(
|
|
1068
|
+
Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())),
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function assertStealthRedirectUrl(url: string): void {
|
|
1073
|
+
const protocol = new URL(url).protocol;
|
|
1074
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
1075
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
1076
|
+
code: "transport_invalid_url",
|
|
1077
|
+
status: 0,
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
function discardStealthRedirectBody(response: StealthTransportResponse): void {
|
|
1083
|
+
try {
|
|
1084
|
+
const cancellation = response.body?.cancel();
|
|
1085
|
+
if (cancellation) void cancellation.catch(() => undefined);
|
|
1086
|
+
} catch {
|
|
1087
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
1088
|
+
// failure must not replace or delay that decision.
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
async function fetchStealthRedirectChain(
|
|
1093
|
+
transport: WreqSession,
|
|
1094
|
+
cookieJar: StealthCookieJar,
|
|
1095
|
+
requestUrl: string,
|
|
1096
|
+
method: StealthMethod,
|
|
1097
|
+
options: StealthFetchOptions,
|
|
1098
|
+
signal?: AbortSignal,
|
|
1099
|
+
buildHeaders?: (
|
|
1100
|
+
url: string,
|
|
1101
|
+
method: StealthMethod,
|
|
1102
|
+
body: string | undefined,
|
|
1103
|
+
headers: Record<string, string>,
|
|
1104
|
+
) => HeaderTuple[],
|
|
1105
|
+
): Promise<{ normalized: StealthResponse; response: StealthTransportResponse }> {
|
|
1106
|
+
let currentUrl = requestUrl;
|
|
1107
|
+
let currentMethod = method;
|
|
1108
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
1109
|
+
let currentHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1110
|
+
let followedHops = 0;
|
|
1111
|
+
let response: StealthTransportResponse;
|
|
1112
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
1113
|
+
|
|
1114
|
+
while (true) {
|
|
1115
|
+
throwIfAmbientAborted(signal);
|
|
1116
|
+
const headers = { ...currentHeaders };
|
|
1117
|
+
if (!hasHeader(headers, "cookie")) {
|
|
1118
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
1119
|
+
if (cookieHeader) headers.cookie = cookieHeader;
|
|
1120
|
+
}
|
|
1121
|
+
const requestInit: StealthRequestInit = {
|
|
1122
|
+
headers: buildHeaders
|
|
1123
|
+
? buildHeaders(currentUrl, currentMethod, currentBody, headers)
|
|
1124
|
+
: headers,
|
|
1125
|
+
method: currentMethod,
|
|
1126
|
+
redirect: "manual",
|
|
1127
|
+
...(new URL(currentUrl).protocol === "http:" ? { disableDefaultHeaders: true } : {}),
|
|
1128
|
+
...(signal ? { signal } : {}),
|
|
1129
|
+
};
|
|
1130
|
+
if (currentBody !== undefined) requestInit.body = currentBody;
|
|
1131
|
+
|
|
1132
|
+
await transport.clearCookies();
|
|
1133
|
+
throwIfAmbientAborted(signal);
|
|
1134
|
+
const remainingTimeout =
|
|
1135
|
+
deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
1136
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
1137
|
+
throw new TransportError("Request timed out", {
|
|
1138
|
+
code: "transport_timeout",
|
|
1139
|
+
status: 0,
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
if (remainingTimeout !== undefined) requestInit.timeout = remainingTimeout;
|
|
1143
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
1144
|
+
if (signal?.aborted) {
|
|
1145
|
+
discardStealthRedirectBody(response);
|
|
1146
|
+
throw toAmbientCancellationError(signal);
|
|
1147
|
+
}
|
|
1148
|
+
cookieJar.setFromCookieStrings(
|
|
1149
|
+
setCookieHeadersFromResponse(response.headers),
|
|
1150
|
+
response.url ?? currentUrl,
|
|
1151
|
+
);
|
|
1152
|
+
|
|
1153
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual") break;
|
|
1154
|
+
if (options.redirect === "error") {
|
|
1155
|
+
discardStealthRedirectBody(response);
|
|
1156
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
1157
|
+
code: "transport_network_error",
|
|
1158
|
+
status: 0,
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
const nextUrl = resolveRedirectUrl(
|
|
1163
|
+
response.headers.get("location") ?? undefined,
|
|
1164
|
+
response.url ?? currentUrl,
|
|
1165
|
+
);
|
|
1166
|
+
if (!nextUrl) break;
|
|
1167
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
1168
|
+
discardStealthRedirectBody(response);
|
|
1169
|
+
throw new TransportError(
|
|
1170
|
+
`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`,
|
|
1171
|
+
{ code: "transport_network_error", status: 0 },
|
|
1172
|
+
);
|
|
1173
|
+
}
|
|
1174
|
+
assertStealthRedirectUrl(nextUrl);
|
|
1175
|
+
discardStealthRedirectBody(response);
|
|
1176
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
1177
|
+
if (nextMethod !== currentMethod) {
|
|
1178
|
+
currentBody = undefined;
|
|
1179
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
1180
|
+
}
|
|
1181
|
+
currentMethod = nextMethod;
|
|
1182
|
+
currentUrl = nextUrl;
|
|
1183
|
+
followedHops += 1;
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
const normalized = await normalizeResponseWithSignal(
|
|
1187
|
+
response,
|
|
1188
|
+
currentUrl,
|
|
1189
|
+
options.maxBodyBytes,
|
|
1190
|
+
signal,
|
|
1191
|
+
);
|
|
1192
|
+
if (followedHops > 0) normalized.redirected = true;
|
|
1193
|
+
return { normalized, response };
|
|
1194
|
+
}
|
|
1195
|
+
|
|
579
1196
|
function createSessionFetcher(
|
|
580
1197
|
baseUrl: string,
|
|
581
|
-
defaultProfile:
|
|
1198
|
+
defaultProfile: StealthProfileDescriptor,
|
|
582
1199
|
clientOptions: StealthClientOptions,
|
|
583
1200
|
): StealthSession {
|
|
584
|
-
const clients = new Map<string,
|
|
1201
|
+
const clients = new Map<string, WreqSessionCacheEntry>();
|
|
585
1202
|
let closed = false;
|
|
586
1203
|
let hasWarnedMissingProxy = false;
|
|
587
1204
|
const warn = clientOptions.warn ?? console.warn;
|
|
588
|
-
const cookieJar = new
|
|
589
|
-
const impitCookieJar = toImpitCookieJar(cookieJar);
|
|
1205
|
+
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
590
1206
|
|
|
591
|
-
function
|
|
592
|
-
|
|
1207
|
+
async function getClientEntry(
|
|
1208
|
+
profile: StealthProfileDescriptor,
|
|
593
1209
|
proxyUrl: string | undefined,
|
|
594
1210
|
ignoreTlsErrors: boolean,
|
|
595
|
-
|
|
1211
|
+
defaultHeaders?: HeaderTuple[],
|
|
1212
|
+
): Promise<WreqSessionCacheEntry> {
|
|
596
1213
|
if (closed) {
|
|
597
1214
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
598
1215
|
}
|
|
599
|
-
const
|
|
600
|
-
const
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
1216
|
+
const wreq = await getWreqModule();
|
|
1217
|
+
const { browser, os } = resolveWreqProfile(profile, wreq.getProfiles());
|
|
1218
|
+
const cacheKey = JSON.stringify({
|
|
1219
|
+
browser,
|
|
1220
|
+
os,
|
|
1221
|
+
proxyUrl,
|
|
1222
|
+
ignoreTlsErrors,
|
|
1223
|
+
headerOrder: defaultHeaders?.map(([name]) => name),
|
|
1224
|
+
});
|
|
1225
|
+
let entry = clients.get(cacheKey);
|
|
1226
|
+
if (!entry) {
|
|
1227
|
+
entry = {
|
|
1228
|
+
session: wreq.createSession({
|
|
1229
|
+
browser,
|
|
1230
|
+
os,
|
|
1231
|
+
...(defaultHeaders ? { defaultHeaders } : {}),
|
|
1232
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
1233
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
1234
|
+
timeout: 30_000,
|
|
1235
|
+
}),
|
|
1236
|
+
tail: Promise.resolve(),
|
|
1237
|
+
};
|
|
1238
|
+
clients.set(cacheKey, entry);
|
|
1239
|
+
}
|
|
1240
|
+
return entry;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
async function withClient<T>(
|
|
1244
|
+
profile: StealthProfileDescriptor,
|
|
1245
|
+
proxyUrl: string | undefined,
|
|
1246
|
+
ignoreTlsErrors: boolean,
|
|
1247
|
+
operation: (client: WreqSession) => Promise<T>,
|
|
1248
|
+
signal?: AbortSignal,
|
|
1249
|
+
defaultHeaders?: HeaderTuple[],
|
|
1250
|
+
): Promise<T> {
|
|
1251
|
+
throwIfAmbientAborted(signal);
|
|
1252
|
+
const entry = await getClientEntry(profile, proxyUrl, ignoreTlsErrors, defaultHeaders);
|
|
1253
|
+
throwIfAmbientAborted(signal);
|
|
1254
|
+
const previous = entry.tail;
|
|
1255
|
+
let release!: () => void;
|
|
1256
|
+
entry.tail = new Promise<void>((resolve) => {
|
|
1257
|
+
release = resolve;
|
|
1258
|
+
});
|
|
1259
|
+
let acquired = false;
|
|
1260
|
+
try {
|
|
1261
|
+
await waitForClientTurn(previous, signal);
|
|
1262
|
+
acquired = true;
|
|
1263
|
+
throwIfAmbientAborted(signal);
|
|
1264
|
+
const client = await entry.session;
|
|
1265
|
+
throwIfAmbientAborted(signal);
|
|
1266
|
+
const result = await operation(client);
|
|
1267
|
+
throwIfAmbientAborted(signal);
|
|
1268
|
+
return result;
|
|
1269
|
+
} finally {
|
|
1270
|
+
if (acquired) {
|
|
1271
|
+
release();
|
|
1272
|
+
} else {
|
|
1273
|
+
void previous.then(release, release);
|
|
1274
|
+
}
|
|
611
1275
|
}
|
|
612
|
-
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
async function waitForClientTurn(previous: Promise<void>, signal?: AbortSignal): Promise<void> {
|
|
1279
|
+
if (!signal) {
|
|
1280
|
+
await previous;
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
throwIfAmbientAborted(signal);
|
|
1284
|
+
await new Promise<void>((resolve, reject) => {
|
|
1285
|
+
let settled = false;
|
|
1286
|
+
const settle = (operation: () => void) => {
|
|
1287
|
+
if (settled) return;
|
|
1288
|
+
settled = true;
|
|
1289
|
+
signal.removeEventListener("abort", onAbort);
|
|
1290
|
+
operation();
|
|
1291
|
+
};
|
|
1292
|
+
const onAbort = () => settle(() => reject(toAmbientCancellationError(signal)));
|
|
1293
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1294
|
+
void previous.then(
|
|
1295
|
+
() => settle(resolve),
|
|
1296
|
+
(error) => settle(() => reject(error)),
|
|
1297
|
+
);
|
|
1298
|
+
if (signal.aborted) onAbort();
|
|
1299
|
+
});
|
|
613
1300
|
}
|
|
614
1301
|
|
|
615
1302
|
async function resolveRequestProxy(
|
|
616
1303
|
options?: StealthFetchOptions,
|
|
617
1304
|
proxyAttempt?: number,
|
|
1305
|
+
refreshEpoch?: number,
|
|
618
1306
|
): Promise<ResolvedAttemptProxy> {
|
|
619
1307
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
620
1308
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
621
1309
|
upstream: clientOptions.upstream,
|
|
622
|
-
apifuseConfig: clientOptions.apifuseConfig,
|
|
623
1310
|
affinityKey: clientOptions.affinityKey,
|
|
624
1311
|
proxyAttempt: computeProxyAttemptIndex({
|
|
625
1312
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
626
1313
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
1314
|
retryAttemptOffset: proxyAttempt,
|
|
628
1315
|
}),
|
|
1316
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
1317
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
1318
|
+
transportProtocols: ["http", "socks5"],
|
|
1319
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
629
1320
|
telemetry: clientOptions.telemetry,
|
|
630
1321
|
});
|
|
631
1322
|
|
|
@@ -638,42 +1329,66 @@ function createSessionFetcher(
|
|
|
638
1329
|
url: resolvedProxy.url,
|
|
639
1330
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
640
1331
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
1332
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
641
1333
|
};
|
|
642
1334
|
}
|
|
643
1335
|
|
|
644
1336
|
const session: StealthSession = {
|
|
645
1337
|
async fetch(url, options: StealthFetchOptions = {}) {
|
|
646
|
-
const
|
|
647
|
-
const hasExplicitRetryPolicy =
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
? undefined
|
|
655
|
-
: createDefaultProxyTransportRetryOptions({
|
|
1338
|
+
const requestProfile = resolveStealthProfileSelection(options.stealth, defaultProfile);
|
|
1339
|
+
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
1340
|
+
try {
|
|
1341
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
1342
|
+
normalizedCallerHeaderEntries(options.headers ?? {});
|
|
1343
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
1344
|
+
const stealthRetryOptions =
|
|
1345
|
+
normalizeProxyTransportRetryOptions(options.retry, {
|
|
656
1346
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
657
1347
|
label: "Stealth",
|
|
658
|
-
})
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
1348
|
+
}) ??
|
|
1349
|
+
(hasExplicitRetryPolicy
|
|
1350
|
+
? undefined
|
|
1351
|
+
: createDefaultProxyTransportRetryOptions({
|
|
1352
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
1353
|
+
label: "Stealth",
|
|
1354
|
+
}));
|
|
1355
|
+
if (stealthRetryOptions) {
|
|
1356
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
1357
|
+
}
|
|
1358
|
+
return { hasExplicitRetryPolicy, method, stealthRetryOptions };
|
|
1359
|
+
} catch (error) {
|
|
1360
|
+
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
1361
|
+
}
|
|
1362
|
+
})();
|
|
1363
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
662
1364
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
663
1365
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
1366
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
1367
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
1368
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
1369
|
+
const policyProxy =
|
|
1370
|
+
clientOptions.proxyPolicy ??
|
|
1371
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
1372
|
+
? clientOptions.upstream.proxy
|
|
1373
|
+
: undefined);
|
|
1374
|
+
// The pool span is already bounded by each vendor's max pool size
|
|
1375
|
+
// (smartproxy ≤20, nodemaven ≤50), so the configured span never exceeds
|
|
1376
|
+
// the chain's true maximum — a large NodeMaven pool stays fully
|
|
1377
|
+
// reachable rather than being truncated at an arbitrary ceiling.
|
|
665
1378
|
const policyProxyAttemptCap = Math.max(
|
|
666
1379
|
1,
|
|
667
|
-
|
|
668
|
-
MAX_POLICY_PROXY_RETRY_ATTEMPTS,
|
|
669
|
-
clientOptions.proxyPolicy?.session?.poolSize ??
|
|
670
|
-
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
-
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
-
: undefined) ??
|
|
673
|
-
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
|
-
),
|
|
1380
|
+
policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
675
1381
|
);
|
|
676
|
-
|
|
1382
|
+
// A registry vendor chain (smartproxy/nodemaven) is the only policy whose
|
|
1383
|
+
// successive attempts resolve a *different* endpoint, so it is the only one
|
|
1384
|
+
// that may widen the attempt cap to the pool span, de-duplicate endpoints,
|
|
1385
|
+
// and drive allocator stale-pool refresh. Deprecated custom/decodo policies
|
|
1386
|
+
// have no managed endpoint to rotate or refresh, so they follow the ordinary
|
|
1387
|
+
// transport-retry budget instead.
|
|
1388
|
+
const rotatesRegistryChain =
|
|
1389
|
+
usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
1390
|
+
const maxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
1391
|
+
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
677
1392
|
let lastError: unknown;
|
|
678
1393
|
|
|
679
1394
|
for (
|
|
@@ -686,8 +1401,14 @@ function createSessionFetcher(
|
|
|
686
1401
|
const attemptedProxies = new Set<string>();
|
|
687
1402
|
|
|
688
1403
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
1404
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
689
1405
|
let proxy: string | undefined;
|
|
690
1406
|
let attemptProxy: ResolvedAttemptProxy | undefined;
|
|
1407
|
+
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
1408
|
+
let serializedUrl: ReturnType<typeof serializeRequestUrl> | undefined;
|
|
1409
|
+
let fallbackSensitiveValues: readonly string[] = [];
|
|
1410
|
+
let fallbackRequestUrl: string | undefined;
|
|
1411
|
+
let fallbackRedactedUrl: string | undefined;
|
|
691
1412
|
const attemptStartedAt = Date.now();
|
|
692
1413
|
let attemptRecorded = false;
|
|
693
1414
|
const recordProxyAttempt = (
|
|
@@ -698,7 +1419,7 @@ function createSessionFetcher(
|
|
|
698
1419
|
if (attemptRecorded || !proxy) return;
|
|
699
1420
|
attemptRecorded = true;
|
|
700
1421
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
701
|
-
provider: "smartproxy",
|
|
1422
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
702
1423
|
attempt: attempt + 1,
|
|
703
1424
|
...(attemptProxy?.poolIndex === undefined
|
|
704
1425
|
? {}
|
|
@@ -711,12 +1432,29 @@ function createSessionFetcher(
|
|
|
711
1432
|
});
|
|
712
1433
|
};
|
|
713
1434
|
try {
|
|
1435
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1436
|
+
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
1437
|
+
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
1438
|
+
fallbackSensitiveValues = [
|
|
1439
|
+
...new Set([
|
|
1440
|
+
...Object.values(sensitiveParams ?? {}).map(String),
|
|
1441
|
+
...structural.sensitiveValues,
|
|
1442
|
+
]),
|
|
1443
|
+
].filter((value) => value !== "");
|
|
1444
|
+
fallbackRequestUrl = url;
|
|
1445
|
+
fallbackRedactedUrl = structural.redactedUrl;
|
|
714
1446
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
715
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
1447
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
1448
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
716
1449
|
proxy = attemptProxy.url;
|
|
717
|
-
if (proxy &&
|
|
1450
|
+
if (proxy && dedupeAllocatorEndpoints) {
|
|
1451
|
+
// An under-filled allocation repeats endpoints (via the modulo
|
|
1452
|
+
// pool mapping) before the flat offset crosses into the next
|
|
1453
|
+
// vendor. Skip an already-tried endpoint and advance the offset
|
|
1454
|
+
// rather than breaking — breaking here would strand the request on
|
|
1455
|
+
// the primary vendor and never reach the fallback leg.
|
|
718
1456
|
if (attemptedProxies.has(proxy)) {
|
|
719
|
-
|
|
1457
|
+
continue;
|
|
720
1458
|
}
|
|
721
1459
|
attemptedProxies.add(proxy);
|
|
722
1460
|
}
|
|
@@ -724,28 +1462,65 @@ function createSessionFetcher(
|
|
|
724
1462
|
options.stealth?.insecureSkipVerify ??
|
|
725
1463
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
726
1464
|
);
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
const
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
1465
|
+
serializedUrl = serializeRequestUrl(
|
|
1466
|
+
resolveUrl(baseUrl, url),
|
|
1467
|
+
options.params,
|
|
1468
|
+
sensitiveParams,
|
|
1469
|
+
);
|
|
1470
|
+
const { requestUrl } = serializedUrl;
|
|
1471
|
+
const wreq = await getWreqModule();
|
|
1472
|
+
const mapping = resolveWreqProfile(requestProfile, wreq.getProfiles());
|
|
1473
|
+
const chromeEmulationHeaders = mapping.browser.startsWith("chrome_")
|
|
1474
|
+
? Array.from(
|
|
1475
|
+
wreq.getEmulationHeaders(mapping.browser, mapping.os),
|
|
1476
|
+
([name, value]) => [String(name), String(value)] as HeaderTuple,
|
|
1477
|
+
)
|
|
1478
|
+
: undefined;
|
|
1479
|
+
const buildOrderedHeaders = chromeEmulationHeaders
|
|
1480
|
+
? (
|
|
1481
|
+
currentUrl: string,
|
|
1482
|
+
currentMethod: StealthMethod,
|
|
1483
|
+
currentBody: string | undefined,
|
|
1484
|
+
currentHeaders: Record<string, string>,
|
|
1485
|
+
) =>
|
|
1486
|
+
buildChromeHeaderTuples({
|
|
1487
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1488
|
+
method: currentMethod,
|
|
1489
|
+
body: currentBody,
|
|
1490
|
+
headers: currentHeaders,
|
|
1491
|
+
requestUrl: currentUrl,
|
|
1492
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1493
|
+
requestClass: options.stealth?.requestClass,
|
|
1494
|
+
})
|
|
1495
|
+
: undefined;
|
|
1496
|
+
const initialHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1497
|
+
if (!hasHeader(initialHeaders, "cookie")) {
|
|
1498
|
+
const cookieHeader = cookieJar.toHeader(requestUrl);
|
|
1499
|
+
if (cookieHeader) initialHeaders.cookie = cookieHeader;
|
|
742
1500
|
}
|
|
743
|
-
const
|
|
1501
|
+
const defaultHeaders = buildOrderedHeaders?.(
|
|
744
1502
|
requestUrl,
|
|
745
|
-
|
|
1503
|
+
method,
|
|
1504
|
+
options.body === undefined ? undefined : normalizeBody(options.body),
|
|
1505
|
+
initialHeaders,
|
|
1506
|
+
);
|
|
1507
|
+
const { normalized, response } = await withClient(
|
|
1508
|
+
requestProfile,
|
|
1509
|
+
proxy,
|
|
1510
|
+
ignoreTlsErrors,
|
|
1511
|
+
(transport) =>
|
|
1512
|
+
fetchStealthRedirectChain(
|
|
1513
|
+
transport,
|
|
1514
|
+
cookieJar,
|
|
1515
|
+
requestUrl,
|
|
1516
|
+
method,
|
|
1517
|
+
options,
|
|
1518
|
+
clientOptions.signal,
|
|
1519
|
+
buildOrderedHeaders,
|
|
1520
|
+
),
|
|
1521
|
+
clientOptions.signal,
|
|
1522
|
+
defaultHeaders,
|
|
746
1523
|
);
|
|
747
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
748
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
749
1524
|
|
|
750
1525
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
751
1526
|
throw createProxyConnectFailureError(normalized.body);
|
|
@@ -789,16 +1564,48 @@ function createSessionFetcher(
|
|
|
789
1564
|
recordProxyAttempt("ok", undefined, response.status);
|
|
790
1565
|
return normalized;
|
|
791
1566
|
} catch (error) {
|
|
792
|
-
const
|
|
1567
|
+
const sensitiveValues = serializedUrl?.sensitiveValues ?? fallbackSensitiveValues;
|
|
1568
|
+
let normalizedError: TransportError;
|
|
1569
|
+
try {
|
|
1570
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1571
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
1572
|
+
} catch (normalizationError) {
|
|
1573
|
+
const redactedNormalizationError = redactSensitiveError(
|
|
1574
|
+
normalizationError,
|
|
1575
|
+
sensitiveValues,
|
|
1576
|
+
serializedUrl?.requestUrl ?? fallbackRequestUrl,
|
|
1577
|
+
serializedUrl?.redactedUrl ?? fallbackRedactedUrl,
|
|
1578
|
+
);
|
|
1579
|
+
if (
|
|
1580
|
+
normalizationError instanceof TransportError &&
|
|
1581
|
+
normalizationError.code === "transport_cancelled"
|
|
1582
|
+
) {
|
|
1583
|
+
recordProxyAttempt(
|
|
1584
|
+
"error",
|
|
1585
|
+
proxyAttemptErrorCode(normalizationError),
|
|
1586
|
+
proxyAttemptStatus(normalizationError),
|
|
1587
|
+
);
|
|
1588
|
+
}
|
|
1589
|
+
throw redactedNormalizationError;
|
|
1590
|
+
}
|
|
1591
|
+
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
1592
|
+
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
1593
|
+
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
1594
|
+
normalizedError = redactSensitiveError(
|
|
1595
|
+
normalizedError,
|
|
1596
|
+
sensitiveValues,
|
|
1597
|
+
serializedUrl?.requestUrl ?? fallbackRequestUrl,
|
|
1598
|
+
serializedUrl?.redactedUrl ?? fallbackRedactedUrl,
|
|
1599
|
+
);
|
|
793
1600
|
recordProxyAttempt(
|
|
794
1601
|
"error",
|
|
795
1602
|
proxyAttemptErrorCode(normalizedError),
|
|
796
1603
|
proxyAttemptStatus(normalizedError),
|
|
797
1604
|
);
|
|
798
1605
|
lastError = normalizedError;
|
|
799
|
-
if (proxy &&
|
|
1606
|
+
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
800
1607
|
stalePoolError = normalizedError;
|
|
801
|
-
if (
|
|
1608
|
+
if (runProxyAuthDiagnostic) {
|
|
802
1609
|
stalePoolDiagnosticProxy = proxy;
|
|
803
1610
|
}
|
|
804
1611
|
if (attempt + 1 < maxAttempts) {
|
|
@@ -806,13 +1613,30 @@ function createSessionFetcher(
|
|
|
806
1613
|
}
|
|
807
1614
|
break;
|
|
808
1615
|
}
|
|
1616
|
+
// Cap the number of transport retries. For a policy-allocator chain,
|
|
1617
|
+
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
1618
|
+
// rotates across the concatenated vendor pool spans), so a transport
|
|
1619
|
+
// failure is a signal to advance to the next endpoint — potentially
|
|
1620
|
+
// crossing into the fallback vendor — not to retry the same endpoint.
|
|
1621
|
+
// Truncating that rotation at the per-endpoint retry budget would
|
|
1622
|
+
// strand the request on the primary vendor and never reach the
|
|
1623
|
+
// fallback, since the crossover only happens once the flat attempt
|
|
1624
|
+
// index exceeds the primary vendor's pool size (~10-20).
|
|
1625
|
+
// resolvePolicyTransportAttemptCap widens to the full chain span only
|
|
1626
|
+
// for implicit, safe-method allocator requests; explicit retry
|
|
1627
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
1628
|
+
// static/non-registry vendors keep the per-endpoint retry budget.
|
|
1629
|
+
const transportRetryCap = resolvePolicyTransportAttemptCap({
|
|
1630
|
+
policy: policyProxy,
|
|
1631
|
+
usesPolicyAllocator,
|
|
1632
|
+
retryAttempts: stealthRetryOptions?.attempts ?? 1,
|
|
1633
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
1634
|
+
method,
|
|
1635
|
+
});
|
|
809
1636
|
if (
|
|
810
|
-
attempt + 1 <
|
|
811
|
-
(stealthRetryOptions
|
|
812
|
-
? Math.min(maxAttempts, stealthRetryOptions.attempts)
|
|
813
|
-
: maxAttempts) &&
|
|
1637
|
+
attempt + 1 < transportRetryCap &&
|
|
814
1638
|
shouldRetryProxyTransportAttempt({
|
|
815
|
-
error:
|
|
1639
|
+
error: { code: retryErrorCode },
|
|
816
1640
|
explicitRetry: hasExplicitRetryPolicy,
|
|
817
1641
|
method,
|
|
818
1642
|
options: stealthRetryOptions,
|
|
@@ -820,8 +1644,12 @@ function createSessionFetcher(
|
|
|
820
1644
|
})
|
|
821
1645
|
) {
|
|
822
1646
|
if (stealthRetryOptions) {
|
|
823
|
-
await sleep(
|
|
1647
|
+
await sleep(
|
|
1648
|
+
computeProxyTransportRetryDelayMs(stealthRetryOptions, attempt + 1),
|
|
1649
|
+
clientOptions.signal,
|
|
1650
|
+
);
|
|
824
1651
|
}
|
|
1652
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
825
1653
|
continue;
|
|
826
1654
|
}
|
|
827
1655
|
throw normalizedError;
|
|
@@ -829,24 +1657,23 @@ function createSessionFetcher(
|
|
|
829
1657
|
}
|
|
830
1658
|
|
|
831
1659
|
if (
|
|
832
|
-
|
|
1660
|
+
rotatesRegistryChain &&
|
|
833
1661
|
stalePoolError &&
|
|
834
1662
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES
|
|
835
1663
|
) {
|
|
1664
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
836
1665
|
await invalidateProxyResolutionCacheAsync({
|
|
837
1666
|
proxyPolicy: clientOptions.proxyPolicy,
|
|
838
1667
|
upstream: clientOptions.upstream,
|
|
839
1668
|
affinityKey: clientOptions.affinityKey,
|
|
840
1669
|
});
|
|
1670
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
841
1671
|
continue;
|
|
842
1672
|
}
|
|
843
1673
|
|
|
844
1674
|
const proxyAuthDiagnostic =
|
|
845
1675
|
stalePoolError && stalePoolDiagnosticProxy
|
|
846
|
-
? await classifyProxyAuthDiagnostic(
|
|
847
|
-
options.profile ?? defaultProfile,
|
|
848
|
-
stalePoolDiagnosticProxy,
|
|
849
|
-
)
|
|
1676
|
+
? await classifyProxyAuthDiagnostic(requestProfile, stalePoolDiagnosticProxy)
|
|
850
1677
|
: undefined;
|
|
851
1678
|
if (proxyAuthDiagnostic === "source_ip_denied") {
|
|
852
1679
|
throw createProxyAuthIpDeniedError(
|
|
@@ -873,6 +1700,7 @@ function createSessionFetcher(
|
|
|
873
1700
|
break;
|
|
874
1701
|
}
|
|
875
1702
|
|
|
1703
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
876
1704
|
throw normalizeStealthTransportError(lastError);
|
|
877
1705
|
},
|
|
878
1706
|
cookies: cookieJar,
|
|
@@ -882,25 +1710,99 @@ function createSessionFetcher(
|
|
|
882
1710
|
options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
883
1711
|
? 10
|
|
884
1712
|
: Math.max(0, Math.floor(options.maxHops));
|
|
1713
|
+
const {
|
|
1714
|
+
url: _url,
|
|
1715
|
+
maxHops: _maxHops,
|
|
1716
|
+
stopWhen,
|
|
1717
|
+
params,
|
|
1718
|
+
sensitiveParams,
|
|
1719
|
+
...fetchOptions
|
|
1720
|
+
} = options;
|
|
885
1721
|
const hops: StealthRedirectHop[] = [];
|
|
886
|
-
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
887
1722
|
let method = normalizeMethod(options.method ?? "GET");
|
|
888
1723
|
let body = options.body;
|
|
889
1724
|
let response: StealthResponse | undefined;
|
|
890
1725
|
const visitedRequests = new Set<string>();
|
|
891
|
-
|
|
892
|
-
|
|
1726
|
+
const initialParams = params
|
|
1727
|
+
? Object.fromEntries(
|
|
1728
|
+
Object.entries(params).map(([key, value]) => [
|
|
1729
|
+
key,
|
|
1730
|
+
Array.isArray(value) ? [...value] : value,
|
|
1731
|
+
]),
|
|
1732
|
+
)
|
|
1733
|
+
: undefined;
|
|
1734
|
+
const normalizedSensitiveParams = normalizeSensitiveParams(sensitiveParams);
|
|
1735
|
+
const initialSensitiveParams = normalizedSensitiveParams
|
|
1736
|
+
? { ...normalizedSensitiveParams }
|
|
1737
|
+
: undefined;
|
|
1738
|
+
const sensitiveParamNames = initialSensitiveParams
|
|
1739
|
+
? Object.keys(initialSensitiveParams)
|
|
1740
|
+
: [];
|
|
1741
|
+
const callerStructural = redactUrlQueryParams(options.url, sensitiveParamNames);
|
|
1742
|
+
const sensitiveValues = new Set(
|
|
1743
|
+
[
|
|
1744
|
+
...Object.values(initialSensitiveParams ?? {}),
|
|
1745
|
+
...callerStructural.sensitiveValues,
|
|
1746
|
+
].filter((value) => value !== ""),
|
|
1747
|
+
);
|
|
1748
|
+
const redactRedirectUrl = (value: string): string => {
|
|
1749
|
+
const structural = redactUrlQueryParams(value, [
|
|
1750
|
+
...new Set([...sensitiveParamNames, ...sensitiveQueryParamNames(value)]),
|
|
1751
|
+
]);
|
|
1752
|
+
for (const sensitiveValue of structural.sensitiveValues) {
|
|
1753
|
+
sensitiveValues.add(sensitiveValue);
|
|
1754
|
+
}
|
|
1755
|
+
return redactSensitiveText(structural.redactedUrl, [...sensitiveValues]);
|
|
1756
|
+
};
|
|
1757
|
+
let currentUrl: string;
|
|
1758
|
+
let initialUrl: ReturnType<typeof serializeRequestUrl>;
|
|
1759
|
+
try {
|
|
1760
|
+
currentUrl = resolveUrl(baseUrl, options.url);
|
|
1761
|
+
redactRedirectUrl(currentUrl);
|
|
1762
|
+
initialUrl = serializeRequestUrl(currentUrl, initialParams, initialSensitiveParams);
|
|
1763
|
+
for (const value of initialUrl.sensitiveValues) {
|
|
1764
|
+
if (value !== "") sensitiveValues.add(value);
|
|
1765
|
+
}
|
|
1766
|
+
} catch (error) {
|
|
1767
|
+
throw redactSensitiveError(
|
|
1768
|
+
error,
|
|
1769
|
+
[...sensitiveValues],
|
|
1770
|
+
options.url,
|
|
1771
|
+
redactRedirectUrl(options.url),
|
|
1772
|
+
);
|
|
1773
|
+
}
|
|
893
1774
|
|
|
894
1775
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
1776
|
+
const outboundUrl =
|
|
1777
|
+
hopIndex === 0 ? initialUrl.requestUrl : serializeRequestUrl(currentUrl).requestUrl;
|
|
1778
|
+
// Preserve params-only loop bookkeeping from before sensitiveParams:
|
|
1779
|
+
// the first visited key is the caller's resolved URL, not its expanded query.
|
|
1780
|
+
const visitedUrl = hopIndex === 0 && !initialSensitiveParams ? currentUrl : outboundUrl;
|
|
1781
|
+
visitedRequests.add(`${method} ${visitedUrl}`);
|
|
1782
|
+
try {
|
|
1783
|
+
response = await session.fetch(currentUrl, {
|
|
1784
|
+
...fetchOptions,
|
|
1785
|
+
body,
|
|
1786
|
+
method,
|
|
1787
|
+
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
1788
|
+
...(hopIndex === 0 && initialSensitiveParams
|
|
1789
|
+
? { sensitiveParams: initialSensitiveParams }
|
|
1790
|
+
: {}),
|
|
1791
|
+
redirect: "manual",
|
|
1792
|
+
throwOnHttpError: false,
|
|
1793
|
+
});
|
|
1794
|
+
} catch (error) {
|
|
1795
|
+
throw redactSensitiveError(
|
|
1796
|
+
error,
|
|
1797
|
+
[...sensitiveValues],
|
|
1798
|
+
outboundUrl,
|
|
1799
|
+
redactRedirectUrl(outboundUrl),
|
|
1800
|
+
);
|
|
1801
|
+
}
|
|
1802
|
+
// StealthResponse.url is programmatic metadata and remains raw. Only the
|
|
1803
|
+
// redirect hop emitted below is a diagnostic surface.
|
|
1804
|
+
const responseUrl =
|
|
1805
|
+
response.url ?? (hopIndex === 0 && initialSensitiveParams ? outboundUrl : currentUrl);
|
|
904
1806
|
|
|
905
1807
|
if (!isRedirectStatus(response.status)) {
|
|
906
1808
|
return {
|
|
@@ -908,63 +1810,79 @@ function createSessionFetcher(
|
|
|
908
1810
|
hops,
|
|
909
1811
|
reason: "completed",
|
|
910
1812
|
cookies: cookieJar.snapshot(),
|
|
1813
|
+
cookieStore: cookieJar.serialize(),
|
|
911
1814
|
};
|
|
912
1815
|
}
|
|
913
1816
|
|
|
914
1817
|
const location = locationHeader(response.headers);
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
1818
|
+
const redactedResponseUrl = redactRedirectUrl(responseUrl);
|
|
1819
|
+
const redactedLocation = location ? redactRedirectUrl(location) : undefined;
|
|
1820
|
+
let nextUrl: string | undefined;
|
|
1821
|
+
try {
|
|
1822
|
+
nextUrl = resolveRedirectUrl(location, responseUrl);
|
|
1823
|
+
} catch (error) {
|
|
1824
|
+
throw redactSensitiveError(error, [...sensitiveValues], location, redactedLocation);
|
|
1825
|
+
}
|
|
1826
|
+
const realHop: StealthRedirectHop = {
|
|
1827
|
+
url: responseUrl,
|
|
920
1828
|
status: response.status,
|
|
921
1829
|
method,
|
|
922
1830
|
...(location ? { location } : {}),
|
|
923
1831
|
...(nextUrl ? { nextUrl } : {}),
|
|
924
1832
|
};
|
|
1833
|
+
const hop: StealthRedirectHop = {
|
|
1834
|
+
...realHop,
|
|
1835
|
+
url: redactedResponseUrl,
|
|
1836
|
+
...(redactedLocation ? { location: redactedLocation } : {}),
|
|
1837
|
+
...(nextUrl ? { nextUrl: redactRedirectUrl(nextUrl) } : {}),
|
|
1838
|
+
};
|
|
925
1839
|
hops.push(hop);
|
|
926
1840
|
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1841
|
+
let shouldStop = false;
|
|
1842
|
+
if (stopWhen) {
|
|
1843
|
+
try {
|
|
1844
|
+
shouldStop = await stopWhen(realHop);
|
|
1845
|
+
} catch (error) {
|
|
1846
|
+
let sanitizedError: unknown = error;
|
|
1847
|
+
for (const [rawUrl, safeUrl] of [
|
|
1848
|
+
[responseUrl, redactedResponseUrl],
|
|
1849
|
+
[location, redactedLocation],
|
|
1850
|
+
[nextUrl, nextUrl ? redactRedirectUrl(nextUrl) : undefined],
|
|
1851
|
+
] as const) {
|
|
1852
|
+
if (!rawUrl || !safeUrl) continue;
|
|
1853
|
+
sanitizedError = redactSensitiveError(
|
|
1854
|
+
sanitizedError,
|
|
1855
|
+
[...sensitiveValues],
|
|
1856
|
+
rawUrl,
|
|
1857
|
+
safeUrl,
|
|
1858
|
+
);
|
|
1859
|
+
}
|
|
1860
|
+
throw sanitizedError;
|
|
1861
|
+
}
|
|
943
1862
|
}
|
|
944
|
-
|
|
945
|
-
|
|
1863
|
+
const decision = evaluateRedirectHop({
|
|
1864
|
+
status: response.status,
|
|
1865
|
+
method,
|
|
1866
|
+
nextUrl,
|
|
1867
|
+
shouldStop,
|
|
1868
|
+
redirectCount: hops.length,
|
|
1869
|
+
maxHops,
|
|
1870
|
+
visitedRequests,
|
|
1871
|
+
});
|
|
1872
|
+
if (decision.kind === "stop") {
|
|
946
1873
|
return {
|
|
947
1874
|
final: response,
|
|
948
1875
|
hops,
|
|
949
|
-
reason:
|
|
1876
|
+
reason: decision.reason,
|
|
950
1877
|
cookies: cookieJar.snapshot(),
|
|
1878
|
+
cookieStore: cookieJar.serialize(),
|
|
951
1879
|
};
|
|
952
1880
|
}
|
|
953
|
-
|
|
954
|
-
const nextMethod = nextRedirectMethod(response.status, method);
|
|
955
|
-
if (nextMethod !== method) {
|
|
1881
|
+
if (decision.nextMethod !== method) {
|
|
956
1882
|
body = undefined;
|
|
957
1883
|
}
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
final: response,
|
|
961
|
-
hops,
|
|
962
|
-
reason: "loop",
|
|
963
|
-
cookies: cookieJar.snapshot(),
|
|
964
|
-
};
|
|
965
|
-
}
|
|
966
|
-
method = nextMethod;
|
|
967
|
-
currentUrl = nextUrl;
|
|
1884
|
+
method = decision.nextMethod;
|
|
1885
|
+
currentUrl = decision.nextUrl;
|
|
968
1886
|
}
|
|
969
1887
|
|
|
970
1888
|
if (!response) {
|
|
@@ -982,28 +1900,55 @@ function createSessionFetcher(
|
|
|
982
1900
|
hops,
|
|
983
1901
|
reason: "max_hops",
|
|
984
1902
|
cookies: cookieJar.snapshot(),
|
|
1903
|
+
cookieStore: cookieJar.serialize(),
|
|
985
1904
|
};
|
|
986
1905
|
},
|
|
987
1906
|
},
|
|
988
1907
|
close() {
|
|
989
1908
|
closed = true;
|
|
1909
|
+
for (const client of clients.values()) {
|
|
1910
|
+
void client.session
|
|
1911
|
+
.then((session) => session.close())
|
|
1912
|
+
.catch((error: unknown) => {
|
|
1913
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1914
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1915
|
+
});
|
|
1916
|
+
}
|
|
990
1917
|
clients.clear();
|
|
991
1918
|
},
|
|
992
1919
|
};
|
|
993
1920
|
return session;
|
|
994
1921
|
|
|
995
1922
|
async function classifyProxyAuthDiagnostic(
|
|
996
|
-
|
|
1923
|
+
profile: StealthProfileDescriptor,
|
|
997
1924
|
proxy: string,
|
|
998
1925
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
999
1926
|
try {
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1927
|
+
return await withClient(
|
|
1928
|
+
profile,
|
|
1929
|
+
proxy,
|
|
1930
|
+
false,
|
|
1931
|
+
async (client) => {
|
|
1932
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1933
|
+
await client.clearCookies();
|
|
1934
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1935
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1936
|
+
method: "GET",
|
|
1937
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1938
|
+
...(clientOptions.signal ? { signal: clientOptions.signal } : {}),
|
|
1939
|
+
});
|
|
1940
|
+
const normalized = await normalizeResponseWithSignal(
|
|
1941
|
+
response,
|
|
1942
|
+
undefined,
|
|
1943
|
+
undefined,
|
|
1944
|
+
clientOptions.signal,
|
|
1945
|
+
);
|
|
1946
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1947
|
+
},
|
|
1948
|
+
clientOptions.signal,
|
|
1949
|
+
);
|
|
1006
1950
|
} catch (error) {
|
|
1951
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1007
1952
|
const message =
|
|
1008
1953
|
error instanceof Error
|
|
1009
1954
|
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
@@ -1042,18 +1987,17 @@ function hasHeader(headers: Record<string, string>, name: string): boolean {
|
|
|
1042
1987
|
|
|
1043
1988
|
export function createStealthClient(
|
|
1044
1989
|
baseUrl: string,
|
|
1045
|
-
defaultProfileOrOptions: string | StealthClientOptions = DEFAULT_PROFILE,
|
|
1046
1990
|
clientOptions: StealthClientOptions = {},
|
|
1047
1991
|
): StealthClient {
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1992
|
+
if (typeof clientOptions === "string") {
|
|
1993
|
+
resolveStealthProfileSelection(clientOptions as unknown as StealthProfileSelection);
|
|
1994
|
+
}
|
|
1995
|
+
const defaultProfile = resolveStealthProfileSelection(clientOptions.stealth);
|
|
1052
1996
|
let sharedSession: StealthSession | null = null;
|
|
1053
1997
|
|
|
1054
1998
|
function getSharedSession(): StealthSession {
|
|
1055
1999
|
if (!sharedSession) {
|
|
1056
|
-
sharedSession = createSessionFetcher(baseUrl, defaultProfile,
|
|
2000
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, clientOptions);
|
|
1057
2001
|
}
|
|
1058
2002
|
|
|
1059
2003
|
return sharedSession;
|
|
@@ -1063,9 +2007,9 @@ export function createStealthClient(
|
|
|
1063
2007
|
fetch(url: string, options?: StealthFetchOptions) {
|
|
1064
2008
|
return getSharedSession().fetch(url, options);
|
|
1065
2009
|
},
|
|
1066
|
-
createSession(opts?: {
|
|
1067
|
-
const sessionProfile = opts?.
|
|
1068
|
-
return createSessionFetcher(baseUrl, sessionProfile,
|
|
2010
|
+
createSession(opts?: { stealth?: StealthProfileSelection }) {
|
|
2011
|
+
const sessionProfile = resolveStealthProfileSelection(opts?.stealth, defaultProfile);
|
|
2012
|
+
return createSessionFetcher(baseUrl, sessionProfile, clientOptions);
|
|
1069
2013
|
},
|
|
1070
2014
|
close() {
|
|
1071
2015
|
sharedSession?.close();
|