@apifuse/provider-sdk 2.2.0-beta.4 → 2.2.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +501 -13
- package/CHANGELOG.md +165 -1
- package/README.md +72 -14
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +88 -4
- package/bin/apifuse-dev.ts +38 -5
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +356 -38
- package/bin/apifuse-perf.ts +14 -13
- package/bin/apifuse-record.ts +691 -68
- package/bin/apifuse-submit-check.ts +518 -37
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +2 -1
- package/dist/config/loader.d.ts +177 -16
- package/dist/config/loader.js +424 -127
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +1 -0
- package/dist/contract.js +2 -0
- package/dist/declaration-validation.d.ts +32 -0
- package/dist/declaration-validation.js +207 -0
- package/dist/define.d.ts +51 -25
- package/dist/define.js +752 -38
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +122 -0
- package/dist/errors.d.ts +18 -0
- package/dist/errors.js +40 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +217 -0
- package/dist/health-scenario.d.ts +1842 -0
- package/dist/health-scenario.js +624 -0
- package/dist/index.d.ts +19 -9
- package/dist/index.js +10 -6
- package/dist/lint.d.ts +6 -1
- package/dist/lint.js +362 -3
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +8 -2
- package/dist/provider.js +3 -1
- package/dist/runtime/auth-flow.d.ts +5 -1
- package/dist/runtime/auth-flow.js +6 -0
- package/dist/runtime/browser.d.ts +1 -0
- package/dist/runtime/browser.js +492 -49
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +13 -1
- package/dist/runtime/choice.js +490 -102
- package/dist/runtime/executor.d.ts +2 -2
- package/dist/runtime/executor.js +26 -2
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +515 -53
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +366 -8
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-telemetry.d.ts +80 -1
- package/dist/runtime/proxy-telemetry.js +154 -47
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +22 -0
- package/dist/runtime/resolver-vendors/browser.js +377 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +526 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +407 -0
- package/dist/runtime/resolver-vendors/types.d.ts +94 -0
- package/dist/runtime/resolver-vendors/types.js +96 -0
- package/dist/runtime/resolver.d.ts +60 -0
- package/dist/runtime/resolver.js +737 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +277 -71
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +28 -3
- package/dist/runtime/stealth.js +519 -255
- package/dist/runtime/stt.js +1 -12
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +61 -0
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +5 -3
- package/dist/server/index.js +3 -3
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test.d.ts +114 -0
- package/dist/server/self-test.js +784 -148
- package/dist/server/serve-implementation.d.ts +213 -0
- package/dist/server/serve-implementation.js +2173 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1130
- package/dist/server/trace-output.d.ts +4 -0
- package/dist/server/trace-output.js +20 -0
- package/dist/server/types.d.ts +30 -5
- package/dist/server/types.js +13 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.js +16 -7
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +489 -21
- package/dist/trace-sanitization.d.ts +5 -0
- package/dist/trace-sanitization.js +45 -0
- package/dist/types.d.ts +545 -23
- package/dist/types.js +1 -0
- package/package.json +44 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +2 -1
- package/src/config/loader.ts +665 -163
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +1 -0
- package/src/contract.ts +2 -0
- package/src/declaration-validation.ts +266 -0
- package/src/define.ts +970 -87
- package/src/error-resolution.ts +127 -0
- package/src/errors.ts +52 -0
- package/src/fixture-sanitization.ts +248 -0
- package/src/health-scenario.ts +875 -0
- package/src/index.ts +204 -8
- package/src/lint.ts +408 -4
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +133 -0
- package/src/runtime/auth-flow.ts +12 -0
- package/src/runtime/browser.ts +661 -63
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +631 -120
- package/src/runtime/executor.ts +40 -7
- package/src/runtime/http.ts +641 -61
- package/src/runtime/instrumentation.ts +520 -15
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-telemetry.ts +244 -75
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +20 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +533 -0
- package/src/runtime/resolver-vendors/capsolver.ts +700 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +539 -0
- package/src/runtime/resolver-vendors/types.ts +212 -0
- package/src/runtime/resolver.ts +1103 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +394 -77
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +675 -289
- package/src/runtime/stt.ts +1 -19
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +77 -0
- package/src/serve.ts +6 -1
- package/src/server/index.ts +37 -2
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test.ts +1025 -175
- package/src/server/serve-implementation.ts +3250 -0
- package/src/server/serve.ts +1 -1626
- package/src/server/trace-output.ts +32 -0
- package/src/server/types.ts +13 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +17 -7
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +658 -15
- package/src/trace-sanitization.ts +63 -0
- package/src/types.ts +660 -38
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/runtime/stealth.ts
CHANGED
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import type {
|
|
3
|
-
|
|
2
|
+
import type {
|
|
3
|
+
BrowserProfile,
|
|
4
|
+
EmulationOS,
|
|
5
|
+
RequestInit as WreqRequestInit,
|
|
6
|
+
Session as WreqSession,
|
|
7
|
+
} from "wreq-js";
|
|
4
8
|
|
|
5
|
-
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
9
|
+
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
6
10
|
import {
|
|
7
11
|
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
8
12
|
invalidateProxyResolutionCacheAsync,
|
|
9
13
|
ProxyResolutionError,
|
|
14
|
+
policyResolvesRegistryVendorChain,
|
|
15
|
+
resolvePolicyProxyPoolSpan,
|
|
16
|
+
resolvePolicyTransportAttemptCap,
|
|
10
17
|
resolveProxyConfigAsync,
|
|
11
|
-
|
|
18
|
+
vendorFromResolvedSource,
|
|
12
19
|
} from "../config/loader.js";
|
|
13
20
|
import { SDKError, TransportError } from "../errors.js";
|
|
14
21
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
15
22
|
import type {
|
|
16
|
-
CookieJar,
|
|
17
23
|
HttpMethod,
|
|
18
24
|
StealthClient,
|
|
19
25
|
StealthFetchOptions,
|
|
@@ -21,6 +27,7 @@ import type {
|
|
|
21
27
|
StealthResponse,
|
|
22
28
|
StealthSession,
|
|
23
29
|
} from "../types.js";
|
|
30
|
+
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
24
31
|
import {
|
|
25
32
|
createProxyAuthIpDeniedError,
|
|
26
33
|
createProxyEdgeAuthRejectedError,
|
|
@@ -44,14 +51,27 @@ import {
|
|
|
44
51
|
shouldRetryProxyTransportAttempt,
|
|
45
52
|
validateUnsafeProxyTransportRetryMethods,
|
|
46
53
|
} from "./proxy-retry-policy.js";
|
|
47
|
-
import {
|
|
54
|
+
import {
|
|
55
|
+
evaluateRedirectHop,
|
|
56
|
+
isRedirectStatus,
|
|
57
|
+
nextRedirectMethod,
|
|
58
|
+
resolveRedirectUrl,
|
|
59
|
+
} from "./redirects.js";
|
|
60
|
+
import {
|
|
61
|
+
isSensitiveKey,
|
|
62
|
+
normalizeSensitiveParams,
|
|
63
|
+
redactSensitiveError,
|
|
64
|
+
redactSensitiveRequestError,
|
|
65
|
+
redactSensitiveText,
|
|
66
|
+
redactUrlQueryParams,
|
|
67
|
+
serializeRequestUrl,
|
|
68
|
+
} from "./request-options.js";
|
|
48
69
|
|
|
49
|
-
const DEFAULT_PROFILE = "chrome-146";
|
|
70
|
+
export const DEFAULT_PROFILE = "chrome-146";
|
|
50
71
|
|
|
51
72
|
const MISSING_PROXY_WARNING =
|
|
52
73
|
"[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
53
74
|
|
|
54
|
-
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
55
75
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
56
76
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
57
77
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
@@ -59,6 +79,25 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
59
79
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
60
80
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
61
81
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
82
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
83
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
84
|
+
"content-encoding",
|
|
85
|
+
"content-language",
|
|
86
|
+
"content-location",
|
|
87
|
+
"content-type",
|
|
88
|
+
]);
|
|
89
|
+
|
|
90
|
+
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
91
|
+
return typeof value === "object" && value !== null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function sensitiveQueryParamNames(url: string): string[] {
|
|
95
|
+
const queryStart = url.indexOf("?");
|
|
96
|
+
if (queryStart === -1) return [];
|
|
97
|
+
const fragmentStart = url.indexOf("#", queryStart);
|
|
98
|
+
const query = url.slice(queryStart + 1, fragmentStart === -1 ? undefined : fragmentStart);
|
|
99
|
+
return [...new URLSearchParams(query).keys()].filter(isSensitiveKey);
|
|
100
|
+
}
|
|
62
101
|
|
|
63
102
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
64
103
|
warn?: (message: string) => void;
|
|
@@ -84,183 +123,169 @@ const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
|
84
123
|
"edge-131",
|
|
85
124
|
]);
|
|
86
125
|
|
|
87
|
-
type
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
100: "chrome100",
|
|
92
|
-
101: "chrome101",
|
|
93
|
-
104: "chrome104",
|
|
94
|
-
107: "chrome107",
|
|
95
|
-
110: "chrome110",
|
|
96
|
-
116: "chrome116",
|
|
97
|
-
124: "chrome124",
|
|
98
|
-
125: "chrome125",
|
|
99
|
-
131: "chrome131",
|
|
100
|
-
136: "chrome136",
|
|
101
|
-
142: "chrome142",
|
|
126
|
+
type StealthTransportHeaders = {
|
|
127
|
+
entries(): IterableIterator<[string, string]>;
|
|
128
|
+
get(name: string): string | null;
|
|
129
|
+
getSetCookie?: () => string[];
|
|
102
130
|
};
|
|
103
131
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
132
|
+
type StealthTransportBody = {
|
|
133
|
+
cancel(): Promise<void>;
|
|
134
|
+
getReader(): {
|
|
135
|
+
read(): Promise<{ done: boolean; value?: Uint8Array }>;
|
|
136
|
+
cancel(): Promise<void>;
|
|
137
|
+
releaseLock(): void;
|
|
138
|
+
};
|
|
109
139
|
};
|
|
110
140
|
|
|
111
|
-
type StealthTransportResponse =
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
141
|
+
type StealthTransportResponse = {
|
|
142
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
143
|
+
headers: StealthTransportHeaders;
|
|
144
|
+
status: number;
|
|
145
|
+
body?: StealthTransportBody | null;
|
|
115
146
|
url?: string;
|
|
116
147
|
redirected?: boolean;
|
|
117
148
|
};
|
|
118
149
|
|
|
119
|
-
type StealthMethod =
|
|
120
|
-
type StealthRequestInit =
|
|
150
|
+
type StealthMethod = HttpMethod | "TRACE";
|
|
151
|
+
type StealthRequestInit = WreqRequestInit & {
|
|
121
152
|
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
122
153
|
};
|
|
123
154
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
class CookieJarImpl implements CookieJar {
|
|
129
|
-
private readonly cookies: Record<string, string>;
|
|
130
|
-
|
|
131
|
-
constructor(cookieStrings: string[]) {
|
|
132
|
-
this.cookies = {};
|
|
133
|
-
this.setFromCookieStrings(cookieStrings);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
setFromCookieStrings(cookieStrings: readonly string[]): void {
|
|
137
|
-
for (const cookieString of cookieStrings) {
|
|
138
|
-
const [nameValue] = cookieString.split(";");
|
|
139
|
-
if (!nameValue) {
|
|
140
|
-
continue;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
const separatorIndex = nameValue.indexOf("=");
|
|
144
|
-
if (separatorIndex === -1) {
|
|
145
|
-
continue;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
const name = nameValue.slice(0, separatorIndex).trim();
|
|
149
|
-
const value = nameValue.slice(separatorIndex + 1).trim();
|
|
150
|
-
if (name) this.cookies[name] = value;
|
|
151
|
-
}
|
|
152
|
-
}
|
|
155
|
+
type WreqSessionCacheEntry = {
|
|
156
|
+
session: Promise<WreqSession>;
|
|
157
|
+
tail: Promise<void>;
|
|
158
|
+
};
|
|
153
159
|
|
|
154
|
-
|
|
155
|
-
return this.cookies[name];
|
|
156
|
-
}
|
|
160
|
+
type WreqModule = typeof import("wreq-js");
|
|
157
161
|
|
|
158
|
-
|
|
159
|
-
return { ...this.cookies };
|
|
160
|
-
}
|
|
162
|
+
let wreqModulePromise: Promise<WreqModule> | undefined;
|
|
161
163
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
+
function getWreqModule(): Promise<WreqModule> {
|
|
165
|
+
if (!wreqModulePromise) {
|
|
166
|
+
wreqModulePromise = import("wreq-js").catch((error: unknown) => {
|
|
167
|
+
throw new SDKError(
|
|
168
|
+
`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`,
|
|
169
|
+
{
|
|
170
|
+
code: "stealth_transport_unavailable",
|
|
171
|
+
cause: error instanceof Error ? error : undefined,
|
|
172
|
+
},
|
|
173
|
+
);
|
|
174
|
+
});
|
|
164
175
|
}
|
|
165
176
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
.map(([name, value]) => `${name}=${value}`)
|
|
169
|
-
.join("; ");
|
|
170
|
-
}
|
|
177
|
+
return wreqModulePromise;
|
|
178
|
+
}
|
|
171
179
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
180
|
+
function parseProfileIdentifier(identifier: string): {
|
|
181
|
+
family: string;
|
|
182
|
+
version: number[];
|
|
183
|
+
} | null {
|
|
184
|
+
const match =
|
|
185
|
+
/^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(
|
|
186
|
+
identifier.toLowerCase(),
|
|
187
|
+
);
|
|
188
|
+
if (!match?.[1] || !match[2]) return null;
|
|
189
|
+
return {
|
|
190
|
+
family: match[1],
|
|
191
|
+
version: match[2].split(/[._]/).map(Number),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
175
194
|
|
|
176
|
-
|
|
177
|
-
|
|
195
|
+
function compareVersionDistance(target: number[], left: number[], right: number[]): number {
|
|
196
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
197
|
+
for (let index = 0; index < width; index += 1) {
|
|
198
|
+
const targetPart = target[index] ?? 0;
|
|
199
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
200
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
201
|
+
if (leftDistance !== rightDistance) return leftDistance - rightDistance;
|
|
178
202
|
}
|
|
203
|
+
return 0;
|
|
204
|
+
}
|
|
179
205
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
206
|
+
function closestWreqProfile(
|
|
207
|
+
identifier: string,
|
|
208
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
209
|
+
): BrowserProfile | undefined {
|
|
210
|
+
const requested = parseProfileIdentifier(identifier);
|
|
211
|
+
if (!requested) return undefined;
|
|
212
|
+
|
|
213
|
+
let closest: { name: BrowserProfile; version: number[] } | undefined;
|
|
214
|
+
for (const candidateName of wreqProfiles) {
|
|
215
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
216
|
+
if (!candidate || candidate.family !== requested.family) continue;
|
|
217
|
+
if (
|
|
218
|
+
!closest ||
|
|
219
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0
|
|
220
|
+
) {
|
|
221
|
+
closest = { name: candidateName, version: candidate.version };
|
|
184
222
|
}
|
|
185
223
|
}
|
|
224
|
+
return closest?.name;
|
|
225
|
+
}
|
|
186
226
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
227
|
+
function resolveDefaultWreqProfileMapping(): { identifier: string; os: EmulationOS } {
|
|
228
|
+
let profile: ReturnType<typeof getStealthProfile>;
|
|
229
|
+
try {
|
|
230
|
+
profile = getStealthProfile(DEFAULT_PROFILE);
|
|
231
|
+
} catch (error) {
|
|
232
|
+
throw new SDKError(
|
|
233
|
+
`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`,
|
|
234
|
+
{ cause: error instanceof Error ? error : undefined },
|
|
235
|
+
);
|
|
191
236
|
}
|
|
192
237
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
return undefined;
|
|
238
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
239
|
+
if (!parseProfileIdentifier(identifier)) {
|
|
240
|
+
throw new SDKError(
|
|
241
|
+
`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`,
|
|
242
|
+
);
|
|
202
243
|
}
|
|
244
|
+
return { identifier, os: profile.platform };
|
|
203
245
|
}
|
|
204
246
|
|
|
205
|
-
|
|
206
|
-
major: number,
|
|
207
|
-
candidates: Record<number, ImpitBrowser>,
|
|
208
|
-
): ImpitBrowser {
|
|
209
|
-
let closestMajor: number | undefined;
|
|
210
|
-
let closestBrowser: ImpitBrowser | undefined;
|
|
211
|
-
for (const [candidateMajorText, browser] of Object.entries(candidates)) {
|
|
212
|
-
const candidateMajor = Number(candidateMajorText);
|
|
213
|
-
if (
|
|
214
|
-
closestMajor === undefined ||
|
|
215
|
-
Math.abs(candidateMajor - major) < Math.abs(closestMajor - major)
|
|
216
|
-
) {
|
|
217
|
-
closestMajor = candidateMajor;
|
|
218
|
-
closestBrowser = browser;
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
return closestBrowser ?? "chrome142";
|
|
222
|
-
}
|
|
247
|
+
const DEFAULT_WREQ_PROFILE_MAPPING = resolveDefaultWreqProfileMapping();
|
|
223
248
|
|
|
224
|
-
function
|
|
249
|
+
export function resolveWreqProfile(
|
|
250
|
+
profileName: string,
|
|
251
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
252
|
+
): {
|
|
253
|
+
browser: BrowserProfile;
|
|
254
|
+
os: EmulationOS;
|
|
255
|
+
} {
|
|
225
256
|
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
226
257
|
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
227
258
|
}
|
|
228
259
|
|
|
229
|
-
let
|
|
260
|
+
let identifier: string;
|
|
261
|
+
let os: EmulationOS;
|
|
230
262
|
try {
|
|
231
|
-
profile = getStealthProfile(profileName);
|
|
263
|
+
const profile = getStealthProfile(profileName);
|
|
264
|
+
identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
265
|
+
os = profile.platform;
|
|
232
266
|
} catch {
|
|
233
267
|
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
234
268
|
// profile strings still run with the transport default instead of failing
|
|
235
269
|
// before the request starts. Removed built-in profile aliases above remain
|
|
236
270
|
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
237
|
-
|
|
271
|
+
identifier = DEFAULT_WREQ_PROFILE_MAPPING.identifier;
|
|
272
|
+
os = DEFAULT_WREQ_PROFILE_MAPPING.os;
|
|
238
273
|
}
|
|
239
274
|
|
|
240
|
-
const
|
|
241
|
-
|
|
242
|
-
if (chromeMatch?.[1]) {
|
|
243
|
-
return closestImpitBrowser(Number(chromeMatch[1]), CHROME_IMPIT_BY_MAJOR);
|
|
244
|
-
}
|
|
245
|
-
const firefoxMatch = /^firefox_(\d+)/.exec(identifier);
|
|
246
|
-
if (firefoxMatch?.[1]) {
|
|
247
|
-
return closestImpitBrowser(Number(firefoxMatch[1]), FIREFOX_IMPIT_BY_MAJOR);
|
|
248
|
-
}
|
|
249
|
-
if (identifier.startsWith("safari_")) {
|
|
275
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
276
|
+
if (!browser) {
|
|
250
277
|
throw new SDKError(
|
|
251
|
-
`Stealth profile "${profileName}"
|
|
278
|
+
`Stealth profile "${profileName}" cannot be mapped to a wreq-js browser profile.`,
|
|
252
279
|
);
|
|
253
280
|
}
|
|
254
|
-
|
|
255
|
-
`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`,
|
|
256
|
-
);
|
|
281
|
+
return { browser, os };
|
|
257
282
|
}
|
|
258
283
|
|
|
259
284
|
function resolveUrl(baseUrl: string, url: string): string {
|
|
260
285
|
return new URL(url, baseUrl).toString();
|
|
261
286
|
}
|
|
262
287
|
|
|
263
|
-
function headerEntriesFromHeaders(headers:
|
|
288
|
+
function headerEntriesFromHeaders(headers: StealthTransportHeaders): [string, string][] {
|
|
264
289
|
return Array.from(headers.entries());
|
|
265
290
|
}
|
|
266
291
|
|
|
@@ -278,17 +303,6 @@ function normalizeHeaders(
|
|
|
278
303
|
function hasOwn(object: object, key: string): boolean {
|
|
279
304
|
return Object.hasOwn(object, key);
|
|
280
305
|
}
|
|
281
|
-
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
282
|
-
return {
|
|
283
|
-
setCookie(cookie: string, _url: string, cb?: (error?: unknown) => void) {
|
|
284
|
-
cookieJar.setFromCookieStrings([cookie]);
|
|
285
|
-
if (typeof cb === "function") cb();
|
|
286
|
-
},
|
|
287
|
-
getCookieString(_url: string) {
|
|
288
|
-
return cookieJar.toString();
|
|
289
|
-
},
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
306
|
|
|
293
307
|
function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
294
308
|
if (!isRecord(options)) return;
|
|
@@ -300,17 +314,19 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
300
314
|
if (unsupported.length === 0) return;
|
|
301
315
|
|
|
302
316
|
throw new SDKError(
|
|
303
|
-
`ctx.stealth.fetch uses
|
|
317
|
+
`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use the profile option instead.`,
|
|
304
318
|
);
|
|
305
319
|
}
|
|
306
320
|
|
|
307
|
-
function responseHeadersToRecord(
|
|
321
|
+
function responseHeadersToRecord(
|
|
322
|
+
headers: StealthTransportHeaders,
|
|
323
|
+
): Record<string, string | string[] | undefined> {
|
|
308
324
|
const record: Record<string, string> = {};
|
|
309
325
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
310
326
|
return record;
|
|
311
327
|
}
|
|
312
328
|
|
|
313
|
-
function setCookieHeadersFromResponse(headers:
|
|
329
|
+
function setCookieHeadersFromResponse(headers: StealthTransportHeaders): string[] {
|
|
314
330
|
const getSetCookie = headers.getSetCookie;
|
|
315
331
|
if (typeof getSetCookie === "function") return getSetCookie.call(headers);
|
|
316
332
|
const setCookie = headers.get("set-cookie");
|
|
@@ -337,10 +353,17 @@ function splitCombinedSetCookieHeader(headerValue: string): string[] {
|
|
|
337
353
|
export async function normalizeResponse(
|
|
338
354
|
response: StealthTransportResponse,
|
|
339
355
|
requestUrl?: string,
|
|
356
|
+
maxBodyBytes?: number,
|
|
340
357
|
): Promise<StealthResponse> {
|
|
341
358
|
const headers = Object.fromEntries(response.headers.entries());
|
|
342
|
-
const cookies = new
|
|
343
|
-
|
|
359
|
+
const cookies = new StealthCookieJar(
|
|
360
|
+
setCookieHeadersFromResponse(response.headers),
|
|
361
|
+
response.url ?? requestUrl,
|
|
362
|
+
);
|
|
363
|
+
const bodyBytes =
|
|
364
|
+
maxBodyBytes === undefined
|
|
365
|
+
? await response.arrayBuffer()
|
|
366
|
+
: await readResponseBodyWithLimit(response, maxBodyBytes);
|
|
344
367
|
const body = new TextDecoder().decode(bodyBytes);
|
|
345
368
|
|
|
346
369
|
return {
|
|
@@ -368,6 +391,71 @@ export async function normalizeResponse(
|
|
|
368
391
|
};
|
|
369
392
|
}
|
|
370
393
|
|
|
394
|
+
function responseTooLargeError(maxBodyBytes: number, observedBytes: number): TransportError {
|
|
395
|
+
return new TransportError(
|
|
396
|
+
`Response body exceeded maxBodyBytes limit of ${maxBodyBytes} bytes (observed ${observedBytes} bytes)`,
|
|
397
|
+
{
|
|
398
|
+
code: "response_too_large",
|
|
399
|
+
category: "upstream_http",
|
|
400
|
+
retryable: false,
|
|
401
|
+
status: 0,
|
|
402
|
+
},
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
function declaredContentLength(headers: StealthTransportHeaders): number | undefined {
|
|
407
|
+
const contentLength = headers.get("content-length")?.trim();
|
|
408
|
+
if (!contentLength || !/^\d+$/.test(contentLength)) return undefined;
|
|
409
|
+
const parsed = Number(contentLength);
|
|
410
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
async function readResponseBodyWithLimit(
|
|
414
|
+
response: StealthTransportResponse,
|
|
415
|
+
maxBodyBytes: number,
|
|
416
|
+
): Promise<ArrayBuffer> {
|
|
417
|
+
const contentLength = declaredContentLength(response.headers);
|
|
418
|
+
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
419
|
+
await response.body?.cancel().catch(() => undefined);
|
|
420
|
+
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
if (!response.body) {
|
|
424
|
+
throw new TransportError("Response body stream is unavailable", {
|
|
425
|
+
code: "transport_stream_unavailable",
|
|
426
|
+
category: "upstream_http",
|
|
427
|
+
status: 0,
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const reader = response.body.getReader();
|
|
432
|
+
const chunks: Uint8Array[] = [];
|
|
433
|
+
let receivedBytes = 0;
|
|
434
|
+
try {
|
|
435
|
+
while (true) {
|
|
436
|
+
const { done, value } = await reader.read();
|
|
437
|
+
if (done) break;
|
|
438
|
+
if (!value) continue;
|
|
439
|
+
receivedBytes += value.byteLength;
|
|
440
|
+
if (receivedBytes > maxBodyBytes) {
|
|
441
|
+
await reader.cancel().catch(() => undefined);
|
|
442
|
+
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
443
|
+
}
|
|
444
|
+
chunks.push(value);
|
|
445
|
+
}
|
|
446
|
+
} finally {
|
|
447
|
+
reader.releaseLock();
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const bodyBytes = new Uint8Array(receivedBytes);
|
|
451
|
+
let offset = 0;
|
|
452
|
+
for (const chunk of chunks) {
|
|
453
|
+
bodyBytes.set(chunk, offset);
|
|
454
|
+
offset += chunk.byteLength;
|
|
455
|
+
}
|
|
456
|
+
return bodyBytes.buffer;
|
|
457
|
+
}
|
|
458
|
+
|
|
371
459
|
function normalizeBody(body: StealthFetchOptions["body"]): string {
|
|
372
460
|
if (body === undefined) {
|
|
373
461
|
return "";
|
|
@@ -426,6 +514,7 @@ type ResolvedAttemptProxy = {
|
|
|
426
514
|
url?: string;
|
|
427
515
|
poolIndex?: number;
|
|
428
516
|
proxyHash?: string;
|
|
517
|
+
vendor?: ProxyVendorName;
|
|
429
518
|
};
|
|
430
519
|
|
|
431
520
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -559,16 +648,6 @@ function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
|
559
648
|
}
|
|
560
649
|
}
|
|
561
650
|
|
|
562
|
-
function isRedirectStatus(status: number): boolean {
|
|
563
|
-
return [301, 302, 303, 307, 308].includes(status);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
function nextRedirectMethod(status: number, method: StealthMethod): StealthMethod {
|
|
567
|
-
if (status === 303 && method !== "HEAD") return "GET";
|
|
568
|
-
if ((status === 301 || status === 302) && method === "POST") return "GET";
|
|
569
|
-
return method;
|
|
570
|
-
}
|
|
571
|
-
|
|
572
651
|
function locationHeader(headers: Record<string, string>): string | undefined {
|
|
573
652
|
for (const [name, value] of Object.entries(headers)) {
|
|
574
653
|
if (name.toLowerCase() === "location") return value;
|
|
@@ -576,56 +655,191 @@ function locationHeader(headers: Record<string, string>): string | undefined {
|
|
|
576
655
|
return undefined;
|
|
577
656
|
}
|
|
578
657
|
|
|
658
|
+
function withoutRedirectBodyHeaders(headers: Record<string, string>): Record<string, string> {
|
|
659
|
+
return Object.fromEntries(
|
|
660
|
+
Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())),
|
|
661
|
+
);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
function assertStealthRedirectUrl(url: string): void {
|
|
665
|
+
const protocol = new URL(url).protocol;
|
|
666
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
667
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
668
|
+
code: "transport_invalid_url",
|
|
669
|
+
status: 0,
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function discardStealthRedirectBody(response: StealthTransportResponse): void {
|
|
675
|
+
try {
|
|
676
|
+
const cancellation = response.body?.cancel();
|
|
677
|
+
if (cancellation) void cancellation.catch(() => undefined);
|
|
678
|
+
} catch {
|
|
679
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
680
|
+
// failure must not replace or delay that decision.
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
async function fetchStealthRedirectChain(
|
|
685
|
+
transport: WreqSession,
|
|
686
|
+
cookieJar: StealthCookieJar,
|
|
687
|
+
requestUrl: string,
|
|
688
|
+
method: StealthMethod,
|
|
689
|
+
options: StealthFetchOptions,
|
|
690
|
+
): Promise<{ normalized: StealthResponse; response: StealthTransportResponse }> {
|
|
691
|
+
let currentUrl = requestUrl;
|
|
692
|
+
let currentMethod = method;
|
|
693
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
694
|
+
let currentHeaders = { ...(options.headers ?? {}) };
|
|
695
|
+
let followedHops = 0;
|
|
696
|
+
let response: StealthTransportResponse;
|
|
697
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
698
|
+
|
|
699
|
+
while (true) {
|
|
700
|
+
const headers = { ...currentHeaders };
|
|
701
|
+
if (!hasHeader(headers, "Cookie")) {
|
|
702
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
703
|
+
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
704
|
+
}
|
|
705
|
+
const requestInit: StealthRequestInit = {
|
|
706
|
+
headers: normalizeHeaders(headers),
|
|
707
|
+
method: currentMethod,
|
|
708
|
+
redirect: "manual",
|
|
709
|
+
};
|
|
710
|
+
if (currentBody !== undefined) requestInit.body = currentBody;
|
|
711
|
+
|
|
712
|
+
await transport.clearCookies();
|
|
713
|
+
const remainingTimeout =
|
|
714
|
+
deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
715
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
716
|
+
throw new TransportError("Request timed out", {
|
|
717
|
+
code: "transport_timeout",
|
|
718
|
+
status: 0,
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
if (remainingTimeout !== undefined) requestInit.timeout = remainingTimeout;
|
|
722
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
723
|
+
cookieJar.setFromCookieStrings(
|
|
724
|
+
setCookieHeadersFromResponse(response.headers),
|
|
725
|
+
response.url ?? currentUrl,
|
|
726
|
+
);
|
|
727
|
+
|
|
728
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual") break;
|
|
729
|
+
if (options.redirect === "error") {
|
|
730
|
+
discardStealthRedirectBody(response);
|
|
731
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
732
|
+
code: "transport_network_error",
|
|
733
|
+
status: 0,
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
const nextUrl = resolveRedirectUrl(
|
|
738
|
+
response.headers.get("location") ?? undefined,
|
|
739
|
+
response.url ?? currentUrl,
|
|
740
|
+
);
|
|
741
|
+
if (!nextUrl) break;
|
|
742
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
743
|
+
discardStealthRedirectBody(response);
|
|
744
|
+
throw new TransportError(
|
|
745
|
+
`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`,
|
|
746
|
+
{ code: "transport_network_error", status: 0 },
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
assertStealthRedirectUrl(nextUrl);
|
|
750
|
+
discardStealthRedirectBody(response);
|
|
751
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
752
|
+
if (nextMethod !== currentMethod) {
|
|
753
|
+
currentBody = undefined;
|
|
754
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
755
|
+
}
|
|
756
|
+
currentMethod = nextMethod;
|
|
757
|
+
currentUrl = nextUrl;
|
|
758
|
+
followedHops += 1;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
const normalized = await normalizeResponse(response, currentUrl, options.maxBodyBytes);
|
|
762
|
+
if (followedHops > 0) normalized.redirected = true;
|
|
763
|
+
return { normalized, response };
|
|
764
|
+
}
|
|
765
|
+
|
|
579
766
|
function createSessionFetcher(
|
|
580
767
|
baseUrl: string,
|
|
581
768
|
defaultProfile: string,
|
|
582
769
|
clientOptions: StealthClientOptions,
|
|
583
770
|
): StealthSession {
|
|
584
|
-
const clients = new Map<string,
|
|
771
|
+
const clients = new Map<string, WreqSessionCacheEntry>();
|
|
585
772
|
let closed = false;
|
|
586
773
|
let hasWarnedMissingProxy = false;
|
|
587
774
|
const warn = clientOptions.warn ?? console.warn;
|
|
588
|
-
const cookieJar = new
|
|
589
|
-
const impitCookieJar = toImpitCookieJar(cookieJar);
|
|
775
|
+
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
590
776
|
|
|
591
|
-
function
|
|
777
|
+
async function getClientEntry(
|
|
592
778
|
profileName: string,
|
|
593
779
|
proxyUrl: string | undefined,
|
|
594
780
|
ignoreTlsErrors: boolean,
|
|
595
|
-
):
|
|
781
|
+
): Promise<WreqSessionCacheEntry> {
|
|
596
782
|
if (closed) {
|
|
597
783
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
598
784
|
}
|
|
599
|
-
const
|
|
785
|
+
const wreq = await getWreqModule();
|
|
786
|
+
const { browser, os } = resolveWreqProfile(profileName, wreq.getProfiles());
|
|
600
787
|
const cacheKey = JSON.stringify({ browser, proxyUrl, ignoreTlsErrors });
|
|
601
|
-
let
|
|
602
|
-
if (!
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
788
|
+
let entry = clients.get(cacheKey);
|
|
789
|
+
if (!entry) {
|
|
790
|
+
entry = {
|
|
791
|
+
session: wreq.createSession({
|
|
792
|
+
browser,
|
|
793
|
+
os,
|
|
794
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
795
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
796
|
+
timeout: 30_000,
|
|
797
|
+
}),
|
|
798
|
+
tail: Promise.resolve(),
|
|
799
|
+
};
|
|
800
|
+
clients.set(cacheKey, entry);
|
|
801
|
+
}
|
|
802
|
+
return entry;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
async function withClient<T>(
|
|
806
|
+
profileName: string,
|
|
807
|
+
proxyUrl: string | undefined,
|
|
808
|
+
ignoreTlsErrors: boolean,
|
|
809
|
+
operation: (client: WreqSession) => Promise<T>,
|
|
810
|
+
): Promise<T> {
|
|
811
|
+
const entry = await getClientEntry(profileName, proxyUrl, ignoreTlsErrors);
|
|
812
|
+
const previous = entry.tail;
|
|
813
|
+
let release!: () => void;
|
|
814
|
+
entry.tail = new Promise<void>((resolve) => {
|
|
815
|
+
release = resolve;
|
|
816
|
+
});
|
|
817
|
+
await previous;
|
|
818
|
+
try {
|
|
819
|
+
return await operation(await entry.session);
|
|
820
|
+
} finally {
|
|
821
|
+
release();
|
|
611
822
|
}
|
|
612
|
-
return client;
|
|
613
823
|
}
|
|
614
824
|
|
|
615
825
|
async function resolveRequestProxy(
|
|
616
826
|
options?: StealthFetchOptions,
|
|
617
827
|
proxyAttempt?: number,
|
|
828
|
+
refreshEpoch?: number,
|
|
618
829
|
): Promise<ResolvedAttemptProxy> {
|
|
619
830
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
620
831
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
621
832
|
upstream: clientOptions.upstream,
|
|
622
|
-
apifuseConfig: clientOptions.apifuseConfig,
|
|
623
833
|
affinityKey: clientOptions.affinityKey,
|
|
624
834
|
proxyAttempt: computeProxyAttemptIndex({
|
|
625
835
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
626
836
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
837
|
retryAttemptOffset: proxyAttempt,
|
|
628
838
|
}),
|
|
839
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
840
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
841
|
+
transportProtocols: ["http", "socks5"],
|
|
842
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
629
843
|
telemetry: clientOptions.telemetry,
|
|
630
844
|
});
|
|
631
845
|
|
|
@@ -638,42 +852,63 @@ function createSessionFetcher(
|
|
|
638
852
|
url: resolvedProxy.url,
|
|
639
853
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
640
854
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
855
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
641
856
|
};
|
|
642
857
|
}
|
|
643
858
|
|
|
644
859
|
const session: StealthSession = {
|
|
645
860
|
async fetch(url, options: StealthFetchOptions = {}) {
|
|
646
|
-
const method =
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
}) ??
|
|
653
|
-
(hasExplicitRetryPolicy
|
|
654
|
-
? undefined
|
|
655
|
-
: createDefaultProxyTransportRetryOptions({
|
|
861
|
+
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
862
|
+
try {
|
|
863
|
+
const method = normalizeMethod(options.method ?? "GET");
|
|
864
|
+
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
865
|
+
const stealthRetryOptions =
|
|
866
|
+
normalizeProxyTransportRetryOptions(options.retry, {
|
|
656
867
|
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
657
868
|
label: "Stealth",
|
|
658
|
-
})
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
869
|
+
}) ??
|
|
870
|
+
(hasExplicitRetryPolicy
|
|
871
|
+
? undefined
|
|
872
|
+
: createDefaultProxyTransportRetryOptions({
|
|
873
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
874
|
+
label: "Stealth",
|
|
875
|
+
}));
|
|
876
|
+
if (stealthRetryOptions) {
|
|
877
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
878
|
+
}
|
|
879
|
+
return { hasExplicitRetryPolicy, method, stealthRetryOptions };
|
|
880
|
+
} catch (error) {
|
|
881
|
+
throw redactSensitiveRequestError(error, url, options.sensitiveParams);
|
|
882
|
+
}
|
|
883
|
+
})();
|
|
662
884
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
663
885
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
886
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
887
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
888
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
889
|
+
const policyProxy =
|
|
890
|
+
clientOptions.proxyPolicy ??
|
|
891
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
892
|
+
? clientOptions.upstream.proxy
|
|
893
|
+
: undefined);
|
|
894
|
+
// The pool span is already bounded by each vendor's max pool size
|
|
895
|
+
// (smartproxy ≤20, nodemaven ≤50), so the configured span never exceeds
|
|
896
|
+
// the chain's true maximum — a large NodeMaven pool stays fully
|
|
897
|
+
// reachable rather than being truncated at an arbitrary ceiling.
|
|
665
898
|
const policyProxyAttemptCap = Math.max(
|
|
666
899
|
1,
|
|
667
|
-
|
|
668
|
-
MAX_POLICY_PROXY_RETRY_ATTEMPTS,
|
|
669
|
-
clientOptions.proxyPolicy?.session?.poolSize ??
|
|
670
|
-
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
-
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
-
: undefined) ??
|
|
673
|
-
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
|
-
),
|
|
900
|
+
policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
675
901
|
);
|
|
676
|
-
|
|
902
|
+
// A registry vendor chain (smartproxy/nodemaven) is the only policy whose
|
|
903
|
+
// successive attempts resolve a *different* endpoint, so it is the only one
|
|
904
|
+
// that may widen the attempt cap to the pool span, de-duplicate endpoints,
|
|
905
|
+
// and drive allocator stale-pool refresh. Deprecated custom/decodo policies
|
|
906
|
+
// have no managed endpoint to rotate or refresh, so they follow the ordinary
|
|
907
|
+
// transport-retry budget instead.
|
|
908
|
+
const rotatesRegistryChain =
|
|
909
|
+
usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
910
|
+
const maxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
911
|
+
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
677
912
|
let lastError: unknown;
|
|
678
913
|
|
|
679
914
|
for (
|
|
@@ -688,6 +923,11 @@ function createSessionFetcher(
|
|
|
688
923
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
689
924
|
let proxy: string | undefined;
|
|
690
925
|
let attemptProxy: ResolvedAttemptProxy | undefined;
|
|
926
|
+
// Reuse the exact serialization used by this outbound attempt in its catch path.
|
|
927
|
+
let serializedUrl: ReturnType<typeof serializeRequestUrl> | undefined;
|
|
928
|
+
let fallbackSensitiveValues: readonly string[] = [];
|
|
929
|
+
let fallbackRequestUrl: string | undefined;
|
|
930
|
+
let fallbackRedactedUrl: string | undefined;
|
|
691
931
|
const attemptStartedAt = Date.now();
|
|
692
932
|
let attemptRecorded = false;
|
|
693
933
|
const recordProxyAttempt = (
|
|
@@ -698,7 +938,7 @@ function createSessionFetcher(
|
|
|
698
938
|
if (attemptRecorded || !proxy) return;
|
|
699
939
|
attemptRecorded = true;
|
|
700
940
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
701
|
-
provider: "smartproxy",
|
|
941
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
702
942
|
attempt: attempt + 1,
|
|
703
943
|
...(attemptProxy?.poolIndex === undefined
|
|
704
944
|
? {}
|
|
@@ -711,12 +951,27 @@ function createSessionFetcher(
|
|
|
711
951
|
});
|
|
712
952
|
};
|
|
713
953
|
try {
|
|
954
|
+
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
955
|
+
const structural = redactUrlQueryParams(url, Object.keys(sensitiveParams ?? {}));
|
|
956
|
+
fallbackSensitiveValues = [
|
|
957
|
+
...new Set([
|
|
958
|
+
...Object.values(sensitiveParams ?? {}).map(String),
|
|
959
|
+
...structural.sensitiveValues,
|
|
960
|
+
]),
|
|
961
|
+
].filter((value) => value !== "");
|
|
962
|
+
fallbackRequestUrl = url;
|
|
963
|
+
fallbackRedactedUrl = structural.redactedUrl;
|
|
714
964
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
715
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
965
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
716
966
|
proxy = attemptProxy.url;
|
|
717
|
-
if (proxy &&
|
|
967
|
+
if (proxy && dedupeAllocatorEndpoints) {
|
|
968
|
+
// An under-filled allocation repeats endpoints (via the modulo
|
|
969
|
+
// pool mapping) before the flat offset crosses into the next
|
|
970
|
+
// vendor. Skip an already-tried endpoint and advance the offset
|
|
971
|
+
// rather than breaking — breaking here would strand the request on
|
|
972
|
+
// the primary vendor and never reach the fallback leg.
|
|
718
973
|
if (attemptedProxies.has(proxy)) {
|
|
719
|
-
|
|
974
|
+
continue;
|
|
720
975
|
}
|
|
721
976
|
attemptedProxies.add(proxy);
|
|
722
977
|
}
|
|
@@ -725,27 +980,19 @@ function createSessionFetcher(
|
|
|
725
980
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
726
981
|
);
|
|
727
982
|
const profileName = options.profile ?? defaultProfile;
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
}
|
|
734
|
-
const
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
if (options.body !== undefined) {
|
|
741
|
-
requestInit.body = normalizeBody(options.body);
|
|
742
|
-
}
|
|
743
|
-
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(
|
|
744
|
-
requestUrl,
|
|
745
|
-
requestInit,
|
|
983
|
+
serializedUrl = serializeRequestUrl(
|
|
984
|
+
resolveUrl(baseUrl, url),
|
|
985
|
+
options.params,
|
|
986
|
+
sensitiveParams,
|
|
987
|
+
);
|
|
988
|
+
const { requestUrl } = serializedUrl;
|
|
989
|
+
const { normalized, response } = await withClient(
|
|
990
|
+
profileName,
|
|
991
|
+
proxy,
|
|
992
|
+
ignoreTlsErrors,
|
|
993
|
+
(transport) =>
|
|
994
|
+
fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options),
|
|
746
995
|
);
|
|
747
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
748
|
-
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
749
996
|
|
|
750
997
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
751
998
|
throw createProxyConnectFailureError(normalized.body);
|
|
@@ -789,16 +1036,36 @@ function createSessionFetcher(
|
|
|
789
1036
|
recordProxyAttempt("ok", undefined, response.status);
|
|
790
1037
|
return normalized;
|
|
791
1038
|
} catch (error) {
|
|
792
|
-
const
|
|
1039
|
+
const sensitiveValues = serializedUrl?.sensitiveValues ?? fallbackSensitiveValues;
|
|
1040
|
+
let normalizedError: TransportError;
|
|
1041
|
+
try {
|
|
1042
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
1043
|
+
} catch (normalizationError) {
|
|
1044
|
+
throw redactSensitiveError(
|
|
1045
|
+
normalizationError,
|
|
1046
|
+
sensitiveValues,
|
|
1047
|
+
serializedUrl?.requestUrl ?? fallbackRequestUrl,
|
|
1048
|
+
serializedUrl?.redactedUrl ?? fallbackRedactedUrl,
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
1052
|
+
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
1053
|
+
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
1054
|
+
normalizedError = redactSensitiveError(
|
|
1055
|
+
normalizedError,
|
|
1056
|
+
sensitiveValues,
|
|
1057
|
+
serializedUrl?.requestUrl ?? fallbackRequestUrl,
|
|
1058
|
+
serializedUrl?.redactedUrl ?? fallbackRedactedUrl,
|
|
1059
|
+
);
|
|
793
1060
|
recordProxyAttempt(
|
|
794
1061
|
"error",
|
|
795
1062
|
proxyAttemptErrorCode(normalizedError),
|
|
796
1063
|
proxyAttemptStatus(normalizedError),
|
|
797
1064
|
);
|
|
798
1065
|
lastError = normalizedError;
|
|
799
|
-
if (proxy &&
|
|
1066
|
+
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
800
1067
|
stalePoolError = normalizedError;
|
|
801
|
-
if (
|
|
1068
|
+
if (runProxyAuthDiagnostic) {
|
|
802
1069
|
stalePoolDiagnosticProxy = proxy;
|
|
803
1070
|
}
|
|
804
1071
|
if (attempt + 1 < maxAttempts) {
|
|
@@ -806,13 +1073,30 @@ function createSessionFetcher(
|
|
|
806
1073
|
}
|
|
807
1074
|
break;
|
|
808
1075
|
}
|
|
1076
|
+
// Cap the number of transport retries. For a policy-allocator chain,
|
|
1077
|
+
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
1078
|
+
// rotates across the concatenated vendor pool spans), so a transport
|
|
1079
|
+
// failure is a signal to advance to the next endpoint — potentially
|
|
1080
|
+
// crossing into the fallback vendor — not to retry the same endpoint.
|
|
1081
|
+
// Truncating that rotation at the per-endpoint retry budget would
|
|
1082
|
+
// strand the request on the primary vendor and never reach the
|
|
1083
|
+
// fallback, since the crossover only happens once the flat attempt
|
|
1084
|
+
// index exceeds the primary vendor's pool size (~10-20).
|
|
1085
|
+
// resolvePolicyTransportAttemptCap widens to the full chain span only
|
|
1086
|
+
// for implicit, safe-method allocator requests; explicit retry
|
|
1087
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
1088
|
+
// static/non-registry vendors keep the per-endpoint retry budget.
|
|
1089
|
+
const transportRetryCap = resolvePolicyTransportAttemptCap({
|
|
1090
|
+
policy: policyProxy,
|
|
1091
|
+
usesPolicyAllocator,
|
|
1092
|
+
retryAttempts: stealthRetryOptions?.attempts ?? 1,
|
|
1093
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
1094
|
+
method,
|
|
1095
|
+
});
|
|
809
1096
|
if (
|
|
810
|
-
attempt + 1 <
|
|
811
|
-
(stealthRetryOptions
|
|
812
|
-
? Math.min(maxAttempts, stealthRetryOptions.attempts)
|
|
813
|
-
: maxAttempts) &&
|
|
1097
|
+
attempt + 1 < transportRetryCap &&
|
|
814
1098
|
shouldRetryProxyTransportAttempt({
|
|
815
|
-
error:
|
|
1099
|
+
error: { code: retryErrorCode },
|
|
816
1100
|
explicitRetry: hasExplicitRetryPolicy,
|
|
817
1101
|
method,
|
|
818
1102
|
options: stealthRetryOptions,
|
|
@@ -829,7 +1113,7 @@ function createSessionFetcher(
|
|
|
829
1113
|
}
|
|
830
1114
|
|
|
831
1115
|
if (
|
|
832
|
-
|
|
1116
|
+
rotatesRegistryChain &&
|
|
833
1117
|
stalePoolError &&
|
|
834
1118
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES
|
|
835
1119
|
) {
|
|
@@ -882,25 +1166,99 @@ function createSessionFetcher(
|
|
|
882
1166
|
options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
883
1167
|
? 10
|
|
884
1168
|
: Math.max(0, Math.floor(options.maxHops));
|
|
1169
|
+
const {
|
|
1170
|
+
url: _url,
|
|
1171
|
+
maxHops: _maxHops,
|
|
1172
|
+
stopWhen,
|
|
1173
|
+
params,
|
|
1174
|
+
sensitiveParams,
|
|
1175
|
+
...fetchOptions
|
|
1176
|
+
} = options;
|
|
885
1177
|
const hops: StealthRedirectHop[] = [];
|
|
886
|
-
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
887
1178
|
let method = normalizeMethod(options.method ?? "GET");
|
|
888
1179
|
let body = options.body;
|
|
889
1180
|
let response: StealthResponse | undefined;
|
|
890
1181
|
const visitedRequests = new Set<string>();
|
|
891
|
-
|
|
892
|
-
|
|
1182
|
+
const initialParams = params
|
|
1183
|
+
? Object.fromEntries(
|
|
1184
|
+
Object.entries(params).map(([key, value]) => [
|
|
1185
|
+
key,
|
|
1186
|
+
Array.isArray(value) ? [...value] : value,
|
|
1187
|
+
]),
|
|
1188
|
+
)
|
|
1189
|
+
: undefined;
|
|
1190
|
+
const normalizedSensitiveParams = normalizeSensitiveParams(sensitiveParams);
|
|
1191
|
+
const initialSensitiveParams = normalizedSensitiveParams
|
|
1192
|
+
? { ...normalizedSensitiveParams }
|
|
1193
|
+
: undefined;
|
|
1194
|
+
const sensitiveParamNames = initialSensitiveParams
|
|
1195
|
+
? Object.keys(initialSensitiveParams)
|
|
1196
|
+
: [];
|
|
1197
|
+
const callerStructural = redactUrlQueryParams(options.url, sensitiveParamNames);
|
|
1198
|
+
const sensitiveValues = new Set(
|
|
1199
|
+
[
|
|
1200
|
+
...Object.values(initialSensitiveParams ?? {}),
|
|
1201
|
+
...callerStructural.sensitiveValues,
|
|
1202
|
+
].filter((value) => value !== ""),
|
|
1203
|
+
);
|
|
1204
|
+
const redactRedirectUrl = (value: string): string => {
|
|
1205
|
+
const structural = redactUrlQueryParams(value, [
|
|
1206
|
+
...new Set([...sensitiveParamNames, ...sensitiveQueryParamNames(value)]),
|
|
1207
|
+
]);
|
|
1208
|
+
for (const sensitiveValue of structural.sensitiveValues) {
|
|
1209
|
+
sensitiveValues.add(sensitiveValue);
|
|
1210
|
+
}
|
|
1211
|
+
return redactSensitiveText(structural.redactedUrl, [...sensitiveValues]);
|
|
1212
|
+
};
|
|
1213
|
+
let currentUrl: string;
|
|
1214
|
+
let initialUrl: ReturnType<typeof serializeRequestUrl>;
|
|
1215
|
+
try {
|
|
1216
|
+
currentUrl = resolveUrl(baseUrl, options.url);
|
|
1217
|
+
redactRedirectUrl(currentUrl);
|
|
1218
|
+
initialUrl = serializeRequestUrl(currentUrl, initialParams, initialSensitiveParams);
|
|
1219
|
+
for (const value of initialUrl.sensitiveValues) {
|
|
1220
|
+
if (value !== "") sensitiveValues.add(value);
|
|
1221
|
+
}
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
throw redactSensitiveError(
|
|
1224
|
+
error,
|
|
1225
|
+
[...sensitiveValues],
|
|
1226
|
+
options.url,
|
|
1227
|
+
redactRedirectUrl(options.url),
|
|
1228
|
+
);
|
|
1229
|
+
}
|
|
893
1230
|
|
|
894
1231
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
1232
|
+
const outboundUrl =
|
|
1233
|
+
hopIndex === 0 ? initialUrl.requestUrl : serializeRequestUrl(currentUrl).requestUrl;
|
|
1234
|
+
// Preserve params-only loop bookkeeping from before sensitiveParams:
|
|
1235
|
+
// the first visited key is the caller's resolved URL, not its expanded query.
|
|
1236
|
+
const visitedUrl = hopIndex === 0 && !initialSensitiveParams ? currentUrl : outboundUrl;
|
|
1237
|
+
visitedRequests.add(`${method} ${visitedUrl}`);
|
|
1238
|
+
try {
|
|
1239
|
+
response = await session.fetch(currentUrl, {
|
|
1240
|
+
...fetchOptions,
|
|
1241
|
+
body,
|
|
1242
|
+
method,
|
|
1243
|
+
...(hopIndex === 0 && initialParams ? { params: initialParams } : {}),
|
|
1244
|
+
...(hopIndex === 0 && initialSensitiveParams
|
|
1245
|
+
? { sensitiveParams: initialSensitiveParams }
|
|
1246
|
+
: {}),
|
|
1247
|
+
redirect: "manual",
|
|
1248
|
+
throwOnHttpError: false,
|
|
1249
|
+
});
|
|
1250
|
+
} catch (error) {
|
|
1251
|
+
throw redactSensitiveError(
|
|
1252
|
+
error,
|
|
1253
|
+
[...sensitiveValues],
|
|
1254
|
+
outboundUrl,
|
|
1255
|
+
redactRedirectUrl(outboundUrl),
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1258
|
+
// StealthResponse.url is programmatic metadata and remains raw. Only the
|
|
1259
|
+
// redirect hop emitted below is a diagnostic surface.
|
|
1260
|
+
const responseUrl =
|
|
1261
|
+
response.url ?? (hopIndex === 0 && initialSensitiveParams ? outboundUrl : currentUrl);
|
|
904
1262
|
|
|
905
1263
|
if (!isRedirectStatus(response.status)) {
|
|
906
1264
|
return {
|
|
@@ -908,63 +1266,79 @@ function createSessionFetcher(
|
|
|
908
1266
|
hops,
|
|
909
1267
|
reason: "completed",
|
|
910
1268
|
cookies: cookieJar.snapshot(),
|
|
1269
|
+
cookieStore: cookieJar.serialize(),
|
|
911
1270
|
};
|
|
912
1271
|
}
|
|
913
1272
|
|
|
914
1273
|
const location = locationHeader(response.headers);
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
1274
|
+
const redactedResponseUrl = redactRedirectUrl(responseUrl);
|
|
1275
|
+
const redactedLocation = location ? redactRedirectUrl(location) : undefined;
|
|
1276
|
+
let nextUrl: string | undefined;
|
|
1277
|
+
try {
|
|
1278
|
+
nextUrl = resolveRedirectUrl(location, responseUrl);
|
|
1279
|
+
} catch (error) {
|
|
1280
|
+
throw redactSensitiveError(error, [...sensitiveValues], location, redactedLocation);
|
|
1281
|
+
}
|
|
1282
|
+
const realHop: StealthRedirectHop = {
|
|
1283
|
+
url: responseUrl,
|
|
920
1284
|
status: response.status,
|
|
921
1285
|
method,
|
|
922
1286
|
...(location ? { location } : {}),
|
|
923
1287
|
...(nextUrl ? { nextUrl } : {}),
|
|
924
1288
|
};
|
|
1289
|
+
const hop: StealthRedirectHop = {
|
|
1290
|
+
...realHop,
|
|
1291
|
+
url: redactedResponseUrl,
|
|
1292
|
+
...(redactedLocation ? { location: redactedLocation } : {}),
|
|
1293
|
+
...(nextUrl ? { nextUrl: redactRedirectUrl(nextUrl) } : {}),
|
|
1294
|
+
};
|
|
925
1295
|
hops.push(hop);
|
|
926
1296
|
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1297
|
+
let shouldStop = false;
|
|
1298
|
+
if (stopWhen) {
|
|
1299
|
+
try {
|
|
1300
|
+
shouldStop = await stopWhen(realHop);
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
let sanitizedError: unknown = error;
|
|
1303
|
+
for (const [rawUrl, safeUrl] of [
|
|
1304
|
+
[responseUrl, redactedResponseUrl],
|
|
1305
|
+
[location, redactedLocation],
|
|
1306
|
+
[nextUrl, nextUrl ? redactRedirectUrl(nextUrl) : undefined],
|
|
1307
|
+
] as const) {
|
|
1308
|
+
if (!rawUrl || !safeUrl) continue;
|
|
1309
|
+
sanitizedError = redactSensitiveError(
|
|
1310
|
+
sanitizedError,
|
|
1311
|
+
[...sensitiveValues],
|
|
1312
|
+
rawUrl,
|
|
1313
|
+
safeUrl,
|
|
1314
|
+
);
|
|
1315
|
+
}
|
|
1316
|
+
throw sanitizedError;
|
|
1317
|
+
}
|
|
943
1318
|
}
|
|
944
|
-
|
|
945
|
-
|
|
1319
|
+
const decision = evaluateRedirectHop({
|
|
1320
|
+
status: response.status,
|
|
1321
|
+
method,
|
|
1322
|
+
nextUrl,
|
|
1323
|
+
shouldStop,
|
|
1324
|
+
redirectCount: hops.length,
|
|
1325
|
+
maxHops,
|
|
1326
|
+
visitedRequests,
|
|
1327
|
+
});
|
|
1328
|
+
if (decision.kind === "stop") {
|
|
946
1329
|
return {
|
|
947
1330
|
final: response,
|
|
948
1331
|
hops,
|
|
949
|
-
reason:
|
|
1332
|
+
reason: decision.reason,
|
|
950
1333
|
cookies: cookieJar.snapshot(),
|
|
1334
|
+
cookieStore: cookieJar.serialize(),
|
|
951
1335
|
};
|
|
952
1336
|
}
|
|
953
|
-
|
|
954
|
-
const nextMethod = nextRedirectMethod(response.status, method);
|
|
955
|
-
if (nextMethod !== method) {
|
|
1337
|
+
if (decision.nextMethod !== method) {
|
|
956
1338
|
body = undefined;
|
|
957
1339
|
}
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
final: response,
|
|
961
|
-
hops,
|
|
962
|
-
reason: "loop",
|
|
963
|
-
cookies: cookieJar.snapshot(),
|
|
964
|
-
};
|
|
965
|
-
}
|
|
966
|
-
method = nextMethod;
|
|
967
|
-
currentUrl = nextUrl;
|
|
1340
|
+
method = decision.nextMethod;
|
|
1341
|
+
currentUrl = decision.nextUrl;
|
|
968
1342
|
}
|
|
969
1343
|
|
|
970
1344
|
if (!response) {
|
|
@@ -982,11 +1356,20 @@ function createSessionFetcher(
|
|
|
982
1356
|
hops,
|
|
983
1357
|
reason: "max_hops",
|
|
984
1358
|
cookies: cookieJar.snapshot(),
|
|
1359
|
+
cookieStore: cookieJar.serialize(),
|
|
985
1360
|
};
|
|
986
1361
|
},
|
|
987
1362
|
},
|
|
988
1363
|
close() {
|
|
989
1364
|
closed = true;
|
|
1365
|
+
for (const client of clients.values()) {
|
|
1366
|
+
void client.session
|
|
1367
|
+
.then((session) => session.close())
|
|
1368
|
+
.catch((error: unknown) => {
|
|
1369
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1370
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
990
1373
|
clients.clear();
|
|
991
1374
|
},
|
|
992
1375
|
};
|
|
@@ -997,12 +1380,15 @@ function createSessionFetcher(
|
|
|
997
1380
|
proxy: string,
|
|
998
1381
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
999
1382
|
try {
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1383
|
+
return await withClient(profileName, proxy, false, async (client) => {
|
|
1384
|
+
await client.clearCookies();
|
|
1385
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1386
|
+
method: "GET",
|
|
1387
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1388
|
+
});
|
|
1389
|
+
const normalized = await normalizeResponse(response);
|
|
1390
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1003
1391
|
});
|
|
1004
|
-
const normalized = await normalizeResponse(response);
|
|
1005
|
-
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1006
1392
|
} catch (error) {
|
|
1007
1393
|
const message =
|
|
1008
1394
|
error instanceof Error
|