@apifuse/provider-sdk 2.2.0-beta.1 → 2.2.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +106 -0
- package/CHANGELOG.md +41 -0
- package/README.md +5 -1
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +44 -59
- package/bin/apifuse-create.ts +1 -1
- package/bin/apifuse-dev.ts +27 -52
- package/bin/apifuse-pack-check.ts +14 -0
- package/bin/apifuse-pack-smoke.ts +22 -81
- package/bin/apifuse-pack-types.ts +266 -0
- package/bin/apifuse-perf.ts +45 -127
- package/bin/apifuse-record.ts +53 -70
- package/bin/apifuse-submit-check.ts +510 -45
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/apifuse.ts +1 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/bin/submit-check-xml-semantics.ts +204 -0
- package/bin/submit-check-xml.ts +134 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth.d.ts +2 -2
- package/dist/auth.js +9 -18
- package/dist/ceremonies/index.d.ts +1 -1
- package/dist/ceremonies/index.js +8 -18
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/config/loader.d.ts +149 -8
- package/dist/config/loader.js +378 -89
- package/dist/contract-serialization.d.ts +2 -2
- package/dist/contract-serialization.js +3 -6
- package/dist/contract-types.d.ts +2 -2
- package/dist/contract.d.ts +3 -3
- package/dist/contract.js +4 -6
- package/dist/define.d.ts +8 -1
- package/dist/define.js +119 -122
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/errors.d.ts +4 -1
- package/dist/errors.js +48 -0
- package/dist/i18n/catalog.d.ts +2 -2
- package/dist/i18n/catalog.js +4 -10
- package/dist/i18n/index.d.ts +2 -2
- package/dist/i18n/index.js +2 -2
- package/dist/i18n/keys.d.ts +2 -2
- package/dist/index.d.ts +44 -42
- package/dist/index.js +39 -37
- package/dist/lint.js +8 -15
- package/dist/provider.d.ts +11 -11
- package/dist/provider.js +9 -9
- package/dist/public-schema-field-lint.d.ts +1 -1
- package/dist/recipes/gov-api.js +1 -1
- package/dist/runtime/auth-flow.d.ts +1 -1
- package/dist/runtime/auth-flow.js +3 -3
- package/dist/runtime/browser.d.ts +1 -1
- package/dist/runtime/browser.js +15 -29
- package/dist/runtime/cache.d.ts +1 -1
- package/dist/runtime/cache.js +4 -8
- package/dist/runtime/choice.d.ts +1 -1
- package/dist/runtime/choice.js +31 -35
- package/dist/runtime/credential.d.ts +1 -1
- package/dist/runtime/credential.js +1 -1
- package/dist/runtime/env.d.ts +1 -1
- package/dist/runtime/executor.d.ts +1 -1
- package/dist/runtime/executor.js +15 -3
- package/dist/runtime/http.d.ts +2 -2
- package/dist/runtime/http.js +144 -348
- package/dist/runtime/insights.d.ts +1 -1
- package/dist/runtime/insights.js +6 -13
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +56 -19
- package/dist/runtime/keyring.js +1 -1
- package/dist/runtime/namespace.js +1 -1
- package/dist/runtime/otlp.d.ts +1 -1
- package/dist/runtime/perf.d.ts +1 -1
- package/dist/runtime/provider.d.ts +1 -1
- package/dist/runtime/provider.js +1 -2
- package/dist/runtime/proxy-errors.d.ts +1 -1
- package/dist/runtime/proxy-errors.js +9 -7
- package/dist/runtime/proxy-nodemaven.d.ts +35 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-retry-policy.d.ts +40 -0
- package/dist/runtime/proxy-retry-policy.js +326 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +55 -52
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +2 -4
- package/dist/runtime/request-options.d.ts +1 -1
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +2 -2
- package/dist/runtime/state.js +15 -4
- package/dist/runtime/stealth.d.ts +2 -2
- package/dist/runtime/stealth.js +108 -212
- package/dist/runtime/stt.d.ts +1 -1
- package/dist/runtime/stt.js +11 -15
- package/dist/runtime/trace.d.ts +2 -2
- package/dist/runtime/trace.js +2 -4
- package/dist/runtime/waterfall.d.ts +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +7 -15
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +7 -7
- package/dist/server/index.js +6 -6
- package/dist/server/self-test-redaction.d.ts +1 -1
- package/dist/server/self-test-redaction.js +1 -1
- package/dist/server/self-test.d.ts +104 -3
- package/dist/server/self-test.js +673 -115
- package/dist/server/serve.d.ts +105 -4
- package/dist/server/serve.js +588 -80
- package/dist/server/types.d.ts +10 -9
- package/dist/server/types.js +3 -7
- package/dist/stateful/errors.d.ts +14 -0
- package/dist/stateful/errors.js +14 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +207 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +71 -0
- package/dist/stateful/stateful-provider-session-routing.js +353 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +1 -1
- package/dist/stealth/profiles.js +5 -14
- package/dist/stream.d.ts +1 -1
- package/dist/testing/index.d.ts +2 -2
- package/dist/testing/index.js +2 -2
- package/dist/testing/run.d.ts +1 -1
- package/dist/testing/run.js +7 -14
- package/dist/types.d.ts +87 -1
- package/dist/user-input.d.ts +30 -0
- package/dist/user-input.js +66 -0
- package/package.json +14 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +28 -86
- package/src/ceremonies/index.ts +24 -78
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/config/loader.ts +543 -208
- package/src/contract-serialization.ts +5 -11
- package/src/contract-types.ts +2 -2
- package/src/contract.ts +12 -28
- package/src/define.ts +247 -505
- package/src/dev.ts +4 -9
- package/src/errors.ts +58 -5
- package/src/i18n/catalog.ts +10 -32
- package/src/i18n/index.ts +2 -2
- package/src/i18n/keys.ts +5 -11
- package/src/index.ts +49 -42
- package/src/lint.ts +89 -159
- package/src/provider.ts +15 -11
- package/src/public-schema-field-lint.ts +7 -33
- package/src/recipes/gov-api.ts +2 -5
- package/src/runtime/auth-flow.ts +5 -7
- package/src/runtime/browser.ts +64 -187
- package/src/runtime/cache.ts +20 -67
- package/src/runtime/choice.ts +79 -132
- package/src/runtime/credential.ts +2 -2
- package/src/runtime/env.ts +1 -1
- package/src/runtime/executor.ts +23 -20
- package/src/runtime/http.ts +195 -559
- package/src/runtime/insights.ts +15 -53
- package/src/runtime/instrumentation.ts +78 -65
- package/src/runtime/keyring.ts +7 -19
- package/src/runtime/namespace.ts +2 -7
- package/src/runtime/otlp.ts +12 -23
- package/src/runtime/perf.ts +1 -1
- package/src/runtime/provider.ts +4 -9
- package/src/runtime/proxy-errors.ts +29 -42
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-retry-policy.ts +469 -0
- package/src/runtime/proxy-telemetry.ts +79 -77
- package/src/runtime/redis.ts +4 -12
- package/src/runtime/request-options.ts +4 -13
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +41 -110
- package/src/runtime/stealth.ts +162 -377
- package/src/runtime/stt.ts +38 -94
- package/src/runtime/trace.ts +14 -44
- package/src/runtime/waterfall.ts +5 -18
- package/src/schema.ts +23 -84
- package/src/serve.ts +1 -1
- package/src/server/index.ts +24 -7
- package/src/server/self-test-redaction.ts +2 -2
- package/src/server/self-test.ts +857 -132
- package/src/server/serve.ts +859 -266
- package/src/server/types.ts +7 -19
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +23 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +279 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +555 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +10 -26
- package/src/stream.ts +8 -19
- package/src/testing/index.ts +2 -2
- package/src/testing/run.ts +19 -67
- package/src/types.ts +88 -1
- package/src/user-input.ts +118 -0
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/runtime/stealth.ts
CHANGED
|
@@ -2,27 +2,28 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import type { Browser, ImpitOptions, ImpitResponse, RequestInit } from "impit";
|
|
3
3
|
import { Impit } from "impit";
|
|
4
4
|
|
|
5
|
-
import type { ProxyResolutionOptions } from "../config/loader";
|
|
5
|
+
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
6
6
|
import {
|
|
7
7
|
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
8
8
|
invalidateProxyResolutionCacheAsync,
|
|
9
|
+
policyResolvesRegistryVendorChain,
|
|
9
10
|
ProxyResolutionError,
|
|
11
|
+
resolvePolicyProxyPoolSpan,
|
|
12
|
+
resolvePolicyTransportAttemptCap,
|
|
10
13
|
resolveProxyConfigAsync,
|
|
11
|
-
|
|
12
|
-
} from "../config/loader";
|
|
13
|
-
import {
|
|
14
|
-
import { getStealthProfile } from "../stealth/profiles";
|
|
14
|
+
vendorFromResolvedSource,
|
|
15
|
+
} from "../config/loader.js";
|
|
16
|
+
import { SDKError, TransportError } from "../errors.js";
|
|
17
|
+
import { getStealthProfile } from "../stealth/profiles.js";
|
|
15
18
|
import type {
|
|
16
19
|
CookieJar,
|
|
17
20
|
HttpMethod,
|
|
18
|
-
HttpRetryOptions,
|
|
19
21
|
StealthClient,
|
|
20
22
|
StealthFetchOptions,
|
|
21
23
|
StealthRedirectHop,
|
|
22
24
|
StealthResponse,
|
|
23
25
|
StealthSession,
|
|
24
|
-
} from "../types";
|
|
25
|
-
import { HttpRetryPreset, HttpRetryUnsafeMethodPolicy } from "../types";
|
|
26
|
+
} from "../types.js";
|
|
26
27
|
import {
|
|
27
28
|
createProxyAuthIpDeniedError,
|
|
28
29
|
createProxyEdgeAuthRejectedError,
|
|
@@ -35,49 +36,31 @@ import {
|
|
|
35
36
|
isProxyPoolRefreshableError,
|
|
36
37
|
isProxyPoolStaleMessage,
|
|
37
38
|
isProxyPoolStaleStatus,
|
|
38
|
-
PROXY_AUTH_IP_DENIED_CODE,
|
|
39
39
|
PROXY_EDGE_AUTH_REJECTED_CODE,
|
|
40
40
|
PROXY_POOL_STALE_CODE,
|
|
41
|
-
} from "./proxy-errors";
|
|
42
|
-
import {
|
|
41
|
+
} from "./proxy-errors.js";
|
|
42
|
+
import {
|
|
43
|
+
computeProxyAttemptIndex,
|
|
44
|
+
computeProxyTransportRetryDelayMs,
|
|
45
|
+
createDefaultProxyTransportRetryOptions,
|
|
46
|
+
normalizeProxyTransportRetryOptions,
|
|
47
|
+
shouldRetryProxyTransportAttempt,
|
|
48
|
+
validateUnsafeProxyTransportRetryMethods,
|
|
49
|
+
} from "./proxy-retry-policy.js";
|
|
50
|
+
import { appendQueryParams } from "./request-options.js";
|
|
43
51
|
|
|
44
52
|
const DEFAULT_PROFILE = "chrome-146";
|
|
45
53
|
|
|
46
54
|
const MISSING_PROXY_WARNING =
|
|
47
55
|
"[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
48
56
|
|
|
49
|
-
const MAX_POLICY_PROXY_RETRY_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE;
|
|
50
57
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
51
58
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
52
59
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
53
60
|
/\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
54
61
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
55
62
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
56
|
-
const
|
|
57
|
-
const DEFAULT_STEALTH_RETRY_ERROR_CODES = [
|
|
58
|
-
PROXY_CONNECT_FAILURE_CODE,
|
|
59
|
-
"transport_network_error",
|
|
60
|
-
"transport_timeout",
|
|
61
|
-
] as const;
|
|
62
|
-
const RATE_LIMIT_STEALTH_RETRY_ERROR_CODES = ["transport_timeout"] as const;
|
|
63
|
-
const KNOWN_STEALTH_RETRY_METHODS = new Set([
|
|
64
|
-
"GET",
|
|
65
|
-
"HEAD",
|
|
66
|
-
"POST",
|
|
67
|
-
"PUT",
|
|
68
|
-
"DELETE",
|
|
69
|
-
"OPTIONS",
|
|
70
|
-
"TRACE",
|
|
71
|
-
"PATCH",
|
|
72
|
-
]);
|
|
73
|
-
const UNSAFE_STEALTH_RETRY_METHODS = new Set([
|
|
74
|
-
"POST",
|
|
75
|
-
"PUT",
|
|
76
|
-
"PATCH",
|
|
77
|
-
"DELETE",
|
|
78
|
-
"TRACE",
|
|
79
|
-
]);
|
|
80
|
-
const MAX_STEALTH_RETRY_ATTEMPTS = 8;
|
|
63
|
+
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
81
64
|
|
|
82
65
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
83
66
|
warn?: (message: string) => void;
|
|
@@ -139,12 +122,6 @@ type StealthMethod = NonNullable<ImpitRequestInit["method"]>;
|
|
|
139
122
|
type StealthRequestInit = ImpitRequestInit & {
|
|
140
123
|
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
141
124
|
};
|
|
142
|
-
type NormalizedStealthRetryOptions = {
|
|
143
|
-
attempts: number;
|
|
144
|
-
methods: readonly string[];
|
|
145
|
-
errorCodes: readonly string[];
|
|
146
|
-
unsafeMethodPolicy: HttpRetryOptions["unsafeMethodPolicy"];
|
|
147
|
-
};
|
|
148
125
|
|
|
149
126
|
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
150
127
|
return typeof value === "object" && value !== null;
|
|
@@ -303,9 +280,7 @@ function normalizeHeaders(
|
|
|
303
280
|
function hasOwn(object: object, key: string): boolean {
|
|
304
281
|
return Object.hasOwn(object, key);
|
|
305
282
|
}
|
|
306
|
-
function toImpitCookieJar(
|
|
307
|
-
cookieJar: CookieJarImpl,
|
|
308
|
-
): NonNullable<ImpitOptions["cookieJar"]> {
|
|
283
|
+
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
309
284
|
return {
|
|
310
285
|
setCookie(cookie: string, _url: string, cb?: (error?: unknown) => void) {
|
|
311
286
|
cookieJar.setFromCookieStrings([cookie]);
|
|
@@ -322,10 +297,8 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
322
297
|
const unsupported: string[] = [];
|
|
323
298
|
if (hasOwn(options, "headerOrder")) unsupported.push("headerOrder");
|
|
324
299
|
const stealth = options.stealth;
|
|
325
|
-
if (isRecord(stealth) && hasOwn(stealth, "ja3"))
|
|
326
|
-
|
|
327
|
-
if (isRecord(stealth) && hasOwn(stealth, "h2"))
|
|
328
|
-
unsupported.push("stealth.h2");
|
|
300
|
+
if (isRecord(stealth) && hasOwn(stealth, "ja3")) unsupported.push("stealth.ja3");
|
|
301
|
+
if (isRecord(stealth) && hasOwn(stealth, "h2")) unsupported.push("stealth.h2");
|
|
329
302
|
if (unsupported.length === 0) return;
|
|
330
303
|
|
|
331
304
|
throw new SDKError(
|
|
@@ -333,9 +306,7 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
333
306
|
);
|
|
334
307
|
}
|
|
335
308
|
|
|
336
|
-
function responseHeadersToRecord(
|
|
337
|
-
headers: Headers,
|
|
338
|
-
): Record<string, string | string[] | undefined> {
|
|
309
|
+
function responseHeadersToRecord(headers: Headers): Record<string, string | string[] | undefined> {
|
|
339
310
|
const record: Record<string, string> = {};
|
|
340
311
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
341
312
|
return record;
|
|
@@ -370,9 +341,7 @@ export async function normalizeResponse(
|
|
|
370
341
|
requestUrl?: string,
|
|
371
342
|
): Promise<StealthResponse> {
|
|
372
343
|
const headers = Object.fromEntries(response.headers.entries());
|
|
373
|
-
const cookies = new CookieJarImpl(
|
|
374
|
-
setCookieHeadersFromResponse(response.headers),
|
|
375
|
-
);
|
|
344
|
+
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers));
|
|
376
345
|
const bodyBytes = await response.arrayBuffer();
|
|
377
346
|
const body = new TextDecoder().decode(bodyBytes);
|
|
378
347
|
|
|
@@ -422,189 +391,11 @@ function isPolicyManagedProxy(options: StealthClientOptions): boolean {
|
|
|
422
391
|
return Boolean(policy && typeof policy === "object");
|
|
423
392
|
}
|
|
424
393
|
|
|
425
|
-
function
|
|
426
|
-
return
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
function createStealthRetryOptions(
|
|
430
|
-
preset: HttpRetryPreset,
|
|
431
|
-
): NormalizedStealthRetryOptions {
|
|
432
|
-
switch (preset) {
|
|
433
|
-
case HttpRetryPreset.Off:
|
|
434
|
-
return {
|
|
435
|
-
attempts: 1,
|
|
436
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
437
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
438
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
439
|
-
};
|
|
440
|
-
case HttpRetryPreset.AggressiveRead:
|
|
441
|
-
return {
|
|
442
|
-
attempts: 4,
|
|
443
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
444
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
445
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
446
|
-
};
|
|
447
|
-
case HttpRetryPreset.RateLimitAware:
|
|
448
|
-
return {
|
|
449
|
-
attempts: 3,
|
|
450
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
451
|
-
errorCodes: RATE_LIMIT_STEALTH_RETRY_ERROR_CODES,
|
|
452
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
453
|
-
};
|
|
454
|
-
case HttpRetryPreset.SafeRead:
|
|
455
|
-
case HttpRetryPreset.TransportTransient:
|
|
456
|
-
return {
|
|
457
|
-
attempts: 3,
|
|
458
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
459
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
460
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
461
|
-
};
|
|
462
|
-
}
|
|
463
|
-
throw new ProviderError(`Unknown stealth retry preset: ${preset}`, {
|
|
464
|
-
code: "retry_invalid_policy",
|
|
465
|
-
});
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
function normalizeStealthRetryOptions(
|
|
469
|
-
retry: StealthFetchOptions["retry"],
|
|
470
|
-
): NormalizedStealthRetryOptions | undefined {
|
|
471
|
-
if (retry === undefined) return undefined;
|
|
472
|
-
if (retry === false) return createStealthRetryOptions(HttpRetryPreset.Off);
|
|
473
|
-
if (retry === true)
|
|
474
|
-
return createStealthRetryOptions(HttpRetryPreset.TransportTransient);
|
|
475
|
-
if (typeof retry === "string") {
|
|
476
|
-
if (!Object.values(HttpRetryPreset).includes(retry)) {
|
|
477
|
-
throw new ProviderError(`Unknown stealth retry preset: ${retry}`, {
|
|
478
|
-
code: "retry_invalid_policy",
|
|
479
|
-
});
|
|
480
|
-
}
|
|
481
|
-
return createStealthRetryOptions(retry);
|
|
482
|
-
}
|
|
483
|
-
if (typeof retry !== "object" || retry === null || Array.isArray(retry)) {
|
|
484
|
-
throw new ProviderError("Stealth retry policy must be a plain object", {
|
|
485
|
-
code: "retry_invalid_policy",
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
if (
|
|
489
|
-
retry.unsafeMethodPolicy !== undefined &&
|
|
490
|
-
!Object.values(HttpRetryUnsafeMethodPolicy).includes(
|
|
491
|
-
retry.unsafeMethodPolicy,
|
|
492
|
-
)
|
|
493
|
-
) {
|
|
494
|
-
throw new ProviderError(
|
|
495
|
-
`Unknown stealth retry unsafe method policy: ${String(retry.unsafeMethodPolicy)}`,
|
|
496
|
-
{ code: "retry_invalid_policy" },
|
|
497
|
-
);
|
|
498
|
-
}
|
|
499
|
-
if (retry.methods !== undefined) {
|
|
500
|
-
if (!Array.isArray(retry.methods)) {
|
|
501
|
-
throw new ProviderError("Stealth retry methods must be an array", {
|
|
502
|
-
code: "retry_invalid_policy",
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
const unknownMethods = retry.methods
|
|
506
|
-
.map((method) => (typeof method === "string" ? method.toUpperCase() : ""))
|
|
507
|
-
.filter((method) => !KNOWN_STEALTH_RETRY_METHODS.has(method));
|
|
508
|
-
if (unknownMethods.length > 0) {
|
|
509
|
-
throw new ProviderError(
|
|
510
|
-
`Unknown stealth retry method(s): ${unknownMethods.join(", ")}`,
|
|
511
|
-
{ code: "retry_invalid_policy" },
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
if (retry.errorCodes !== undefined) {
|
|
516
|
-
if (
|
|
517
|
-
!Array.isArray(retry.errorCodes) ||
|
|
518
|
-
retry.errorCodes.some((errorCode) => typeof errorCode !== "string")
|
|
519
|
-
) {
|
|
520
|
-
throw new ProviderError(
|
|
521
|
-
"Stealth retry errorCodes must contain only strings",
|
|
522
|
-
{ code: "retry_invalid_policy" },
|
|
523
|
-
);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
const base = createStealthRetryOptions(
|
|
528
|
-
retry.preset ?? HttpRetryPreset.TransportTransient,
|
|
529
|
-
);
|
|
530
|
-
const attempts =
|
|
531
|
-
retry.attempts === undefined || !Number.isFinite(retry.attempts)
|
|
532
|
-
? base.attempts
|
|
533
|
-
: Math.max(
|
|
534
|
-
1,
|
|
535
|
-
Math.min(MAX_STEALTH_RETRY_ATTEMPTS, Math.floor(retry.attempts)),
|
|
536
|
-
);
|
|
537
|
-
const normalized: NormalizedStealthRetryOptions = {
|
|
538
|
-
attempts,
|
|
539
|
-
methods:
|
|
540
|
-
retry.methods?.map((method) => method.toUpperCase()) ?? base.methods,
|
|
541
|
-
errorCodes: retry.errorCodes ?? base.errorCodes,
|
|
542
|
-
unsafeMethodPolicy: retry.unsafeMethodPolicy ?? base.unsafeMethodPolicy,
|
|
543
|
-
};
|
|
544
|
-
|
|
545
|
-
if (
|
|
546
|
-
normalized.unsafeMethodPolicy !==
|
|
547
|
-
HttpRetryUnsafeMethodPolicy.AllowExplicitUnsafe
|
|
548
|
-
) {
|
|
549
|
-
const unsafeMethods = normalized.methods.filter((method) =>
|
|
550
|
-
UNSAFE_STEALTH_RETRY_METHODS.has(method.toUpperCase()),
|
|
551
|
-
);
|
|
552
|
-
if (unsafeMethods.length > 0) {
|
|
553
|
-
throw new ProviderError(
|
|
554
|
-
`Stealth retry methods include unsafe method(s): ${unsafeMethods.join(", ")}`,
|
|
555
|
-
{ code: "retry_unsafe_method" },
|
|
556
|
-
);
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
return normalized;
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
function isExplicitStealthRetryAllowed(
|
|
564
|
-
method: StealthMethod,
|
|
565
|
-
error: TransportError,
|
|
566
|
-
retryOptions: NormalizedStealthRetryOptions | undefined,
|
|
567
|
-
): boolean {
|
|
568
|
-
if (!retryOptions || retryOptions.attempts <= 1) return false;
|
|
569
|
-
return (
|
|
570
|
-
retryOptions.methods.includes(method.toUpperCase()) &&
|
|
571
|
-
retryOptions.errorCodes.includes(proxyAttemptErrorCode(error))
|
|
572
|
-
);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function isRetryableProxyTransportError(error: unknown): boolean {
|
|
576
|
-
if (error instanceof TransportError) {
|
|
577
|
-
if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
|
|
578
|
-
return false;
|
|
579
|
-
}
|
|
580
|
-
return (
|
|
581
|
-
error.code === PROXY_CONNECT_FAILURE_CODE ||
|
|
582
|
-
error.code === "transport_network_error" ||
|
|
583
|
-
error.code === "transport_timeout"
|
|
584
|
-
);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
if (error instanceof SDKError) {
|
|
588
|
-
return false;
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
592
|
-
return /\bproxy\b|\bnon[\s-]?200\b|\bconnect\b|\btunnel\b/i.test(message);
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
function isProxyConnectFailureResponse(
|
|
596
|
-
response: StealthTransportResponse,
|
|
597
|
-
body: string,
|
|
598
|
-
): boolean {
|
|
599
|
-
return (
|
|
600
|
-
response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "")
|
|
601
|
-
);
|
|
394
|
+
function isProxyConnectFailureResponse(response: StealthTransportResponse, body: string): boolean {
|
|
395
|
+
return response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "");
|
|
602
396
|
}
|
|
603
397
|
|
|
604
|
-
function createProxyConnectFailureError(
|
|
605
|
-
body: string,
|
|
606
|
-
cause?: Error,
|
|
607
|
-
): TransportError {
|
|
398
|
+
function createProxyConnectFailureError(body: string, cause?: Error): TransportError {
|
|
608
399
|
const bodyExcerpt = (body ?? "").trim().slice(0, 1_000);
|
|
609
400
|
if (isProxyAuthIpDeniedMessage(bodyExcerpt)) {
|
|
610
401
|
return createProxyAuthIpDeniedError(cause);
|
|
@@ -613,10 +404,7 @@ function createProxyConnectFailureError(
|
|
|
613
404
|
return createProxyEdgeAuthRejectedError(cause);
|
|
614
405
|
}
|
|
615
406
|
if (isProxyPoolStaleMessage(bodyExcerpt)) {
|
|
616
|
-
return createProxyPoolStaleError(
|
|
617
|
-
bodyExcerpt.includes("512") ? 512 : 509,
|
|
618
|
-
cause,
|
|
619
|
-
);
|
|
407
|
+
return createProxyPoolStaleError(bodyExcerpt.includes("512") ? 512 : 509, cause);
|
|
620
408
|
}
|
|
621
409
|
return new TransportError(bodyExcerpt || "Proxy CONNECT failed", {
|
|
622
410
|
code: PROXY_CONNECT_FAILURE_CODE,
|
|
@@ -640,6 +428,7 @@ type ResolvedAttemptProxy = {
|
|
|
640
428
|
url?: string;
|
|
641
429
|
poolIndex?: number;
|
|
642
430
|
proxyHash?: string;
|
|
431
|
+
vendor?: ProxyVendorName;
|
|
643
432
|
};
|
|
644
433
|
|
|
645
434
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -723,33 +512,20 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
723
512
|
}
|
|
724
513
|
|
|
725
514
|
if (isProxyAuthIpDeniedMessage(message)) {
|
|
726
|
-
return createProxyAuthIpDeniedError(
|
|
727
|
-
error instanceof Error ? error : undefined,
|
|
728
|
-
);
|
|
515
|
+
return createProxyAuthIpDeniedError(error instanceof Error ? error : undefined);
|
|
729
516
|
}
|
|
730
517
|
|
|
731
518
|
if (isProxyEdgeAuthRejectedMessage(message)) {
|
|
732
|
-
return createProxyEdgeAuthRejectedError(
|
|
733
|
-
error instanceof Error ? error : undefined,
|
|
734
|
-
);
|
|
519
|
+
return createProxyEdgeAuthRejectedError(error instanceof Error ? error : undefined);
|
|
735
520
|
}
|
|
736
521
|
|
|
737
522
|
const proxyTunnelStatus = getProxyTunnelStatus(error);
|
|
738
|
-
if (
|
|
739
|
-
proxyTunnelStatus
|
|
740
|
-
isProxyPoolStaleStatus(proxyTunnelStatus)
|
|
741
|
-
) {
|
|
742
|
-
return createProxyPoolStaleError(
|
|
743
|
-
proxyTunnelStatus,
|
|
744
|
-
error instanceof Error ? error : undefined,
|
|
745
|
-
);
|
|
523
|
+
if (proxyTunnelStatus !== undefined && isProxyPoolStaleStatus(proxyTunnelStatus)) {
|
|
524
|
+
return createProxyPoolStaleError(proxyTunnelStatus, error instanceof Error ? error : undefined);
|
|
746
525
|
}
|
|
747
526
|
|
|
748
527
|
if (PROXY_CONNECT_FAILURE_BODY_PATTERN.test(message)) {
|
|
749
|
-
return createProxyConnectFailureError(
|
|
750
|
-
message,
|
|
751
|
-
error instanceof Error ? error : undefined,
|
|
752
|
-
);
|
|
528
|
+
return createProxyConnectFailureError(message, error instanceof Error ? error : undefined);
|
|
753
529
|
}
|
|
754
530
|
|
|
755
531
|
return new TransportError("Network error", {
|
|
@@ -759,6 +535,10 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
759
535
|
});
|
|
760
536
|
}
|
|
761
537
|
|
|
538
|
+
function sleep(ms: number): Promise<void> {
|
|
539
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
540
|
+
}
|
|
541
|
+
|
|
762
542
|
function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
763
543
|
switch (method.toUpperCase()) {
|
|
764
544
|
case "HEAD":
|
|
@@ -786,10 +566,7 @@ function isRedirectStatus(status: number): boolean {
|
|
|
786
566
|
return [301, 302, 303, 307, 308].includes(status);
|
|
787
567
|
}
|
|
788
568
|
|
|
789
|
-
function nextRedirectMethod(
|
|
790
|
-
status: number,
|
|
791
|
-
method: StealthMethod,
|
|
792
|
-
): StealthMethod {
|
|
569
|
+
function nextRedirectMethod(status: number, method: StealthMethod): StealthMethod {
|
|
793
570
|
if (status === 303 && method !== "HEAD") return "GET";
|
|
794
571
|
if ((status === 301 || status === 302) && method === "POST") return "GET";
|
|
795
572
|
return method;
|
|
@@ -841,20 +618,22 @@ function createSessionFetcher(
|
|
|
841
618
|
async function resolveRequestProxy(
|
|
842
619
|
options?: StealthFetchOptions,
|
|
843
620
|
proxyAttempt?: number,
|
|
621
|
+
refreshEpoch?: number,
|
|
844
622
|
): Promise<ResolvedAttemptProxy> {
|
|
845
|
-
const rawProxyAttemptOffset = options?.proxyAttemptOffset ?? 0;
|
|
846
|
-
const proxyAttemptOffset = Number.isFinite(rawProxyAttemptOffset)
|
|
847
|
-
? Math.max(0, Math.floor(rawProxyAttemptOffset))
|
|
848
|
-
: 0;
|
|
849
623
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
850
624
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
851
625
|
upstream: clientOptions.upstream,
|
|
852
626
|
apifuseConfig: clientOptions.apifuseConfig,
|
|
853
627
|
affinityKey: clientOptions.affinityKey,
|
|
854
|
-
proxyAttempt:
|
|
855
|
-
proxyAttempt
|
|
856
|
-
|
|
857
|
-
|
|
628
|
+
proxyAttempt: computeProxyAttemptIndex({
|
|
629
|
+
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
630
|
+
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
631
|
+
retryAttemptOffset: proxyAttempt,
|
|
632
|
+
}),
|
|
633
|
+
// The impit stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
634
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
635
|
+
transportProtocols: ["http", "socks5"],
|
|
636
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
858
637
|
telemetry: clientOptions.telemetry,
|
|
859
638
|
});
|
|
860
639
|
|
|
@@ -867,6 +646,7 @@ function createSessionFetcher(
|
|
|
867
646
|
url: resolvedProxy.url,
|
|
868
647
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
869
648
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
649
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
870
650
|
};
|
|
871
651
|
}
|
|
872
652
|
|
|
@@ -874,23 +654,50 @@ function createSessionFetcher(
|
|
|
874
654
|
async fetch(url, options: StealthFetchOptions = {}) {
|
|
875
655
|
const method = normalizeMethod(options.method ?? "GET");
|
|
876
656
|
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
877
|
-
const stealthRetryOptions =
|
|
657
|
+
const stealthRetryOptions =
|
|
658
|
+
normalizeProxyTransportRetryOptions(options.retry, {
|
|
659
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
660
|
+
label: "Stealth",
|
|
661
|
+
}) ??
|
|
662
|
+
(hasExplicitRetryPolicy
|
|
663
|
+
? undefined
|
|
664
|
+
: createDefaultProxyTransportRetryOptions({
|
|
665
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
666
|
+
label: "Stealth",
|
|
667
|
+
}));
|
|
668
|
+
if (stealthRetryOptions) {
|
|
669
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
670
|
+
}
|
|
878
671
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
879
|
-
const usesPolicyAllocator =
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
672
|
+
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
673
|
+
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
674
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
675
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
676
|
+
const policyProxy =
|
|
677
|
+
clientOptions.proxyPolicy ??
|
|
678
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
679
|
+
? clientOptions.upstream.proxy
|
|
680
|
+
: undefined);
|
|
681
|
+
// The pool span is already bounded by each vendor's max pool size
|
|
682
|
+
// (smartproxy ≤20, nodemaven ≤50), so the configured span never exceeds
|
|
683
|
+
// the chain's true maximum — a large NodeMaven pool stays fully
|
|
684
|
+
// reachable rather than being truncated at an arbitrary ceiling.
|
|
685
|
+
const policyProxyAttemptCap = Math.max(
|
|
686
|
+
1,
|
|
687
|
+
policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
688
|
+
);
|
|
689
|
+
// A registry vendor chain (smartproxy/nodemaven) is the only policy whose
|
|
690
|
+
// successive attempts resolve a *different* endpoint, so it is the only one
|
|
691
|
+
// that may widen the attempt cap to the pool span, de-duplicate endpoints,
|
|
692
|
+
// and drive allocator stale-pool refresh. A static custom/decodo policy
|
|
693
|
+
// resolves the same URL every attempt: widening/refreshing it would resend
|
|
694
|
+
// the request dozens of times (up to maxAttempts × refreshes) and bypass
|
|
695
|
+
// retry:false and unsafe-method controls. Static policies therefore follow
|
|
696
|
+
// the ordinary transport-retry budget instead.
|
|
697
|
+
const rotatesRegistryChain =
|
|
698
|
+
usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
699
|
+
const maxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
700
|
+
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
894
701
|
let lastError: unknown;
|
|
895
702
|
|
|
896
703
|
for (
|
|
@@ -915,14 +722,12 @@ function createSessionFetcher(
|
|
|
915
722
|
if (attemptRecorded || !proxy) return;
|
|
916
723
|
attemptRecorded = true;
|
|
917
724
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
918
|
-
provider: "smartproxy",
|
|
725
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
919
726
|
attempt: attempt + 1,
|
|
920
727
|
...(attemptProxy?.poolIndex === undefined
|
|
921
728
|
? {}
|
|
922
729
|
: { poolIndex: attemptProxy.poolIndex }),
|
|
923
|
-
...(attemptProxy?.proxyHash
|
|
924
|
-
? { proxyHash: attemptProxy.proxyHash }
|
|
925
|
-
: {}),
|
|
730
|
+
...(attemptProxy?.proxyHash ? { proxyHash: attemptProxy.proxyHash } : {}),
|
|
926
731
|
outcome,
|
|
927
732
|
...(errorCode ? { errorCode } : {}),
|
|
928
733
|
...(status === undefined ? {} : { status }),
|
|
@@ -931,25 +736,25 @@ function createSessionFetcher(
|
|
|
931
736
|
};
|
|
932
737
|
try {
|
|
933
738
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
934
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
739
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
935
740
|
proxy = attemptProxy.url;
|
|
936
|
-
if (proxy) {
|
|
741
|
+
if (proxy && dedupeAllocatorEndpoints) {
|
|
742
|
+
// An under-filled allocation repeats endpoints (via the modulo
|
|
743
|
+
// pool mapping) before the flat offset crosses into the next
|
|
744
|
+
// vendor. Skip an already-tried endpoint and advance the offset
|
|
745
|
+
// rather than breaking — breaking here would strand the request on
|
|
746
|
+
// the primary vendor and never reach the fallback leg.
|
|
937
747
|
if (attemptedProxies.has(proxy)) {
|
|
938
|
-
|
|
748
|
+
continue;
|
|
939
749
|
}
|
|
940
750
|
attemptedProxies.add(proxy);
|
|
941
751
|
}
|
|
942
752
|
const ignoreTlsErrors = Boolean(
|
|
943
753
|
options.stealth?.insecureSkipVerify ??
|
|
944
|
-
(!hasPolicyProxy &&
|
|
945
|
-
proxy &&
|
|
946
|
-
clientOptions.proxyStealth?.insecureSkipVerify),
|
|
754
|
+
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
947
755
|
);
|
|
948
756
|
const profileName = options.profile ?? defaultProfile;
|
|
949
|
-
const requestUrl = appendQueryParams(
|
|
950
|
-
resolveUrl(baseUrl, url),
|
|
951
|
-
options.params,
|
|
952
|
-
);
|
|
757
|
+
const requestUrl = appendQueryParams(resolveUrl(baseUrl, url), options.params);
|
|
953
758
|
const headers = { ...(options.headers ?? {}) };
|
|
954
759
|
if (!hasHeader(headers, "Cookie")) {
|
|
955
760
|
const cookieHeader = cookieJar.toString();
|
|
@@ -964,20 +769,14 @@ function createSessionFetcher(
|
|
|
964
769
|
if (options.body !== undefined) {
|
|
965
770
|
requestInit.body = normalizeBody(options.body);
|
|
966
771
|
}
|
|
967
|
-
const response = await getClient(
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
ignoreTlsErrors,
|
|
971
|
-
).fetch(requestUrl, requestInit);
|
|
972
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
973
|
-
cookieJar.setFromCookieStrings(
|
|
974
|
-
setCookieHeadersFromResponse(response.headers),
|
|
772
|
+
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(
|
|
773
|
+
requestUrl,
|
|
774
|
+
requestInit,
|
|
975
775
|
);
|
|
776
|
+
const normalized = await normalizeResponse(response, requestUrl);
|
|
777
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
976
778
|
|
|
977
|
-
if (
|
|
978
|
-
proxy &&
|
|
979
|
-
isProxyConnectFailureResponse(response, normalized.body)
|
|
980
|
-
) {
|
|
779
|
+
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
981
780
|
throw createProxyConnectFailureError(normalized.body);
|
|
982
781
|
}
|
|
983
782
|
|
|
@@ -987,10 +786,9 @@ function createSessionFetcher(
|
|
|
987
786
|
usesPolicyAllocator &&
|
|
988
787
|
isProxyEdgeTlsRejectedResponse(
|
|
989
788
|
response.status,
|
|
990
|
-
[
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
].join("\n"),
|
|
789
|
+
[JSON.stringify(responseHeadersToRecord(response.headers)), normalized.body].join(
|
|
790
|
+
"\n",
|
|
791
|
+
),
|
|
994
792
|
)
|
|
995
793
|
) {
|
|
996
794
|
throw createProxyEdgeTlsRejectedError(response.status);
|
|
@@ -1011,13 +809,10 @@ function createSessionFetcher(
|
|
|
1011
809
|
}
|
|
1012
810
|
|
|
1013
811
|
if (response.status >= 400 && options.throwOnHttpError !== false) {
|
|
1014
|
-
throw new TransportError(
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
status: response.status,
|
|
1019
|
-
},
|
|
1020
|
-
);
|
|
812
|
+
throw new TransportError(`Upstream request failed with status ${response.status}`, {
|
|
813
|
+
code: "upstream_http_error",
|
|
814
|
+
status: response.status,
|
|
815
|
+
});
|
|
1021
816
|
}
|
|
1022
817
|
|
|
1023
818
|
recordProxyAttempt("ok", undefined, response.status);
|
|
@@ -1030,11 +825,7 @@ function createSessionFetcher(
|
|
|
1030
825
|
proxyAttemptStatus(normalizedError),
|
|
1031
826
|
);
|
|
1032
827
|
lastError = normalizedError;
|
|
1033
|
-
if (
|
|
1034
|
-
proxy &&
|
|
1035
|
-
usesPolicyAllocator &&
|
|
1036
|
-
isProxyPoolRefreshableError(normalizedError)
|
|
1037
|
-
) {
|
|
828
|
+
if (proxy && rotatesRegistryChain && isProxyPoolRefreshableError(normalizedError)) {
|
|
1038
829
|
stalePoolError = normalizedError;
|
|
1039
830
|
if (shouldRunProxyAuthDiagnostic(normalizedError)) {
|
|
1040
831
|
stalePoolDiagnosticProxy = proxy;
|
|
@@ -1044,21 +835,39 @@ function createSessionFetcher(
|
|
|
1044
835
|
}
|
|
1045
836
|
break;
|
|
1046
837
|
}
|
|
838
|
+
// Cap the number of transport retries. For a policy-allocator chain,
|
|
839
|
+
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
840
|
+
// rotates across the concatenated vendor pool spans), so a transport
|
|
841
|
+
// failure is a signal to advance to the next endpoint — potentially
|
|
842
|
+
// crossing into the fallback vendor — not to retry the same endpoint.
|
|
843
|
+
// Truncating that rotation at the per-endpoint retry budget would
|
|
844
|
+
// strand the request on the primary vendor and never reach the
|
|
845
|
+
// fallback, since the crossover only happens once the flat attempt
|
|
846
|
+
// index exceeds the primary vendor's pool size (~10-20).
|
|
847
|
+
// resolvePolicyTransportAttemptCap widens to the full chain span only
|
|
848
|
+
// for implicit, safe-method allocator requests; explicit retry
|
|
849
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
850
|
+
// static/non-registry vendors keep the per-endpoint retry budget.
|
|
851
|
+
const transportRetryCap = resolvePolicyTransportAttemptCap({
|
|
852
|
+
policy: policyProxy,
|
|
853
|
+
usesPolicyAllocator,
|
|
854
|
+
retryAttempts: stealthRetryOptions?.attempts ?? 1,
|
|
855
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
856
|
+
method,
|
|
857
|
+
});
|
|
1047
858
|
if (
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
method,
|
|
1057
|
-
normalizedError,
|
|
1058
|
-
stealthRetryOptions,
|
|
1059
|
-
)) &&
|
|
1060
|
-
isRetryableProxyTransportError(normalizedError)
|
|
859
|
+
attempt + 1 < transportRetryCap &&
|
|
860
|
+
shouldRetryProxyTransportAttempt({
|
|
861
|
+
error: normalizedError,
|
|
862
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
863
|
+
method,
|
|
864
|
+
options: stealthRetryOptions,
|
|
865
|
+
proxyUsed: Boolean(proxy),
|
|
866
|
+
})
|
|
1061
867
|
) {
|
|
868
|
+
if (stealthRetryOptions) {
|
|
869
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions!, attempt + 1));
|
|
870
|
+
}
|
|
1062
871
|
continue;
|
|
1063
872
|
}
|
|
1064
873
|
throw normalizedError;
|
|
@@ -1066,7 +875,7 @@ function createSessionFetcher(
|
|
|
1066
875
|
}
|
|
1067
876
|
|
|
1068
877
|
if (
|
|
1069
|
-
|
|
878
|
+
rotatesRegistryChain &&
|
|
1070
879
|
stalePoolError &&
|
|
1071
880
|
refreshAttempt < MAX_POLICY_PROXY_POOL_REFRESHES
|
|
1072
881
|
) {
|
|
@@ -1126,13 +935,7 @@ function createSessionFetcher(
|
|
|
1126
935
|
let response: StealthResponse | undefined;
|
|
1127
936
|
const visitedRequests = new Set<string>();
|
|
1128
937
|
|
|
1129
|
-
const {
|
|
1130
|
-
url: _url,
|
|
1131
|
-
maxHops: _maxHops,
|
|
1132
|
-
stopWhen,
|
|
1133
|
-
params,
|
|
1134
|
-
...fetchOptions
|
|
1135
|
-
} = options;
|
|
938
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, ...fetchOptions } = options;
|
|
1136
939
|
|
|
1137
940
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
1138
941
|
visitedRequests.add(`${method} ${currentUrl}`);
|
|
@@ -1240,22 +1043,16 @@ function createSessionFetcher(
|
|
|
1240
1043
|
proxy: string,
|
|
1241
1044
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
1242
1045
|
try {
|
|
1243
|
-
const response = await getClient(profileName, proxy, false).fetch(
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1248
|
-
},
|
|
1249
|
-
);
|
|
1046
|
+
const response = await getClient(profileName, proxy, false).fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1047
|
+
method: "GET",
|
|
1048
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1049
|
+
});
|
|
1250
1050
|
const normalized = await normalizeResponse(response);
|
|
1251
1051
|
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1252
1052
|
} catch (error) {
|
|
1253
1053
|
const message =
|
|
1254
1054
|
error instanceof Error
|
|
1255
|
-
? [
|
|
1256
|
-
error.message,
|
|
1257
|
-
error.cause instanceof Error ? error.cause.message : "",
|
|
1258
|
-
]
|
|
1055
|
+
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
1259
1056
|
.filter(Boolean)
|
|
1260
1057
|
.join(" ")
|
|
1261
1058
|
: String(error);
|
|
@@ -1295,22 +1092,14 @@ export function createStealthClient(
|
|
|
1295
1092
|
clientOptions: StealthClientOptions = {},
|
|
1296
1093
|
): StealthClient {
|
|
1297
1094
|
const defaultProfile =
|
|
1298
|
-
typeof defaultProfileOrOptions === "string"
|
|
1299
|
-
? defaultProfileOrOptions
|
|
1300
|
-
: DEFAULT_PROFILE;
|
|
1095
|
+
typeof defaultProfileOrOptions === "string" ? defaultProfileOrOptions : DEFAULT_PROFILE;
|
|
1301
1096
|
const resolvedClientOptions =
|
|
1302
|
-
typeof defaultProfileOrOptions === "string"
|
|
1303
|
-
? clientOptions
|
|
1304
|
-
: defaultProfileOrOptions;
|
|
1097
|
+
typeof defaultProfileOrOptions === "string" ? clientOptions : defaultProfileOrOptions;
|
|
1305
1098
|
let sharedSession: StealthSession | null = null;
|
|
1306
1099
|
|
|
1307
1100
|
function getSharedSession(): StealthSession {
|
|
1308
1101
|
if (!sharedSession) {
|
|
1309
|
-
sharedSession = createSessionFetcher(
|
|
1310
|
-
baseUrl,
|
|
1311
|
-
defaultProfile,
|
|
1312
|
-
resolvedClientOptions,
|
|
1313
|
-
);
|
|
1102
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, resolvedClientOptions);
|
|
1314
1103
|
}
|
|
1315
1104
|
|
|
1316
1105
|
return sharedSession;
|
|
@@ -1322,11 +1111,7 @@ export function createStealthClient(
|
|
|
1322
1111
|
},
|
|
1323
1112
|
createSession(opts?: { profile?: string }) {
|
|
1324
1113
|
const sessionProfile = opts?.profile ?? defaultProfile;
|
|
1325
|
-
return createSessionFetcher(
|
|
1326
|
-
baseUrl,
|
|
1327
|
-
sessionProfile,
|
|
1328
|
-
resolvedClientOptions,
|
|
1329
|
-
);
|
|
1114
|
+
return createSessionFetcher(baseUrl, sessionProfile, resolvedClientOptions);
|
|
1330
1115
|
},
|
|
1331
1116
|
close() {
|
|
1332
1117
|
sharedSession?.close();
|