@apifuse/provider-sdk 2.2.0-beta.58 → 2.2.0-beta.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +9 -0
- package/CHANGELOG.md +4 -0
- package/README.md +18 -2
- package/bin/apifuse-pack-types.ts +49 -0
- package/bin/apifuse-record.ts +6 -0
- package/dist/cli/templates/provider/README.md.tpl +15 -3
- package/dist/config/loader.d.ts +7 -0
- package/dist/config/loader.js +30 -6
- package/dist/declaration-validation.d.ts +2 -1
- package/dist/declaration-validation.js +39 -0
- package/dist/define.d.ts +4 -4
- package/dist/define.js +25 -1
- package/dist/engine.d.ts +16 -0
- package/dist/engine.js +56 -1
- package/dist/error-resolution.js +13 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +4 -2
- package/dist/lint.js +2 -2
- package/dist/provider.d.ts +3 -3
- package/dist/provider.js +2 -2
- package/dist/runtime/bounded-expiring-map.d.ts +21 -0
- package/dist/runtime/bounded-expiring-map.js +64 -0
- package/dist/runtime/cache.d.ts +2 -0
- package/dist/runtime/cache.js +8 -1
- package/dist/runtime/egress-lease.d.ts +63 -0
- package/dist/runtime/egress-lease.js +233 -0
- package/dist/runtime/http.js +1 -0
- package/dist/runtime/proxy-retry-policy.d.ts +4 -1
- package/dist/runtime/proxy-retry-policy.js +11 -2
- package/dist/runtime/request-telemetry.d.ts +7 -9
- package/dist/runtime/resolver-config.d.ts +7 -0
- package/dist/runtime/resolver-config.js +8 -0
- package/dist/runtime/resolver-public.d.ts +3 -1
- package/dist/runtime/resolver-public.js +1 -1
- package/dist/runtime/resolver-shared.d.ts +9 -0
- package/dist/runtime/resolver-shared.js +57 -0
- package/dist/runtime/resolver-telemetry.d.ts +134 -0
- package/dist/runtime/resolver-telemetry.js +204 -0
- package/dist/runtime/resolver-usage.d.ts +24 -0
- package/dist/runtime/resolver-usage.js +60 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +5 -0
- package/dist/runtime/resolver-vendors/bindings.js +5 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +2 -2
- package/dist/runtime/resolver-vendors/capsolver.js +21 -9
- package/dist/runtime/resolver-vendors/hypersolutions.d.ts +27 -0
- package/dist/runtime/resolver-vendors/hypersolutions.js +369 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -2
- package/dist/runtime/resolver-vendors/twocaptcha.js +26 -10
- package/dist/runtime/resolver-vendors/types.d.ts +35 -5
- package/dist/runtime/resolver-vendors/types.js +20 -10
- package/dist/runtime/resolver.d.ts +5 -1
- package/dist/runtime/resolver.js +397 -109
- package/dist/runtime/stealth-akamai-sbsd.d.ts +54 -0
- package/dist/runtime/stealth-akamai-sbsd.js +142 -0
- package/dist/runtime/stealth.js +547 -53
- package/dist/schema.d.ts +9 -0
- package/dist/schema.js +16 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +1 -0
- package/dist/server/self-test.d.ts +4 -0
- package/dist/server/self-test.js +10 -1
- package/dist/server/serve-implementation.d.ts +7 -1
- package/dist/server/serve-implementation.js +181 -53
- package/dist/server/types.d.ts +6 -0
- package/dist/server/types.js +14 -0
- package/dist/stealth/profiles.d.ts +7 -1
- package/dist/stealth/profiles.js +9 -0
- package/dist/testing/run.js +6 -0
- package/dist/types.d.ts +106 -10
- package/package.json +1 -1
- package/src/cli/templates/provider/README.md.tpl +15 -3
- package/src/config/loader.ts +34 -6
- package/src/declaration-validation.ts +47 -1
- package/src/define.ts +45 -3
- package/src/engine.ts +75 -1
- package/src/error-resolution.ts +13 -0
- package/src/index.ts +32 -1
- package/src/lint.ts +2 -2
- package/src/provider.ts +9 -0
- package/src/runtime/bounded-expiring-map.ts +67 -0
- package/src/runtime/cache.ts +10 -1
- package/src/runtime/egress-lease.ts +336 -0
- package/src/runtime/http.ts +1 -0
- package/src/runtime/proxy-retry-policy.ts +12 -1
- package/src/runtime/request-telemetry.ts +5 -7
- package/src/runtime/resolver-config.ts +12 -0
- package/src/runtime/resolver-public.ts +21 -0
- package/src/runtime/resolver-shared.ts +75 -0
- package/src/runtime/resolver-telemetry.ts +362 -0
- package/src/runtime/resolver-usage.ts +96 -0
- package/src/runtime/resolver-vendors/bindings.ts +5 -0
- package/src/runtime/resolver-vendors/capsolver.ts +22 -3
- package/src/runtime/resolver-vendors/hypersolutions.ts +480 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +26 -12
- package/src/runtime/resolver-vendors/types.ts +60 -12
- package/src/runtime/resolver.ts +465 -124
- package/src/runtime/stealth-akamai-sbsd.ts +207 -0
- package/src/runtime/stealth.ts +727 -95
- package/src/schema.ts +17 -0
- package/src/server/index.ts +21 -0
- package/src/server/self-test.ts +13 -0
- package/src/server/serve-implementation.ts +268 -48
- package/src/server/types.ts +14 -0
- package/src/stealth/profiles.ts +10 -0
- package/src/testing/run.ts +6 -0
- package/src/types.ts +124 -19
package/src/runtime/stealth.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { Cookie } from "tough-cookie";
|
|
2
3
|
import type {
|
|
3
4
|
BrowserProfile,
|
|
4
5
|
EmulationOS,
|
|
@@ -22,6 +23,7 @@ import {
|
|
|
22
23
|
DEFAULT_STEALTH_BROWSER,
|
|
23
24
|
DEFAULT_STEALTH_OS,
|
|
24
25
|
getStealthProfile,
|
|
26
|
+
resolverClientProfileFamily,
|
|
25
27
|
resolveStealthProfileSelection,
|
|
26
28
|
} from "../stealth/profiles.js";
|
|
27
29
|
import type {
|
|
@@ -35,6 +37,11 @@ import type {
|
|
|
35
37
|
StealthSession,
|
|
36
38
|
} from "../types.js";
|
|
37
39
|
import { chrome149HeaderOrder } from "./chrome149-header-order.js";
|
|
40
|
+
import {
|
|
41
|
+
ENGINE_CEREMONY_EGRESS_LEASE,
|
|
42
|
+
type CeremonyEgressBinding,
|
|
43
|
+
type CeremonyEgressLeaseRuntime,
|
|
44
|
+
} from "./egress-lease.js";
|
|
38
45
|
import {
|
|
39
46
|
createProxyAuthIpDeniedError,
|
|
40
47
|
createProxyEdgeAuthRejectedError,
|
|
@@ -54,6 +61,7 @@ import {
|
|
|
54
61
|
computeProxyAttemptIndex,
|
|
55
62
|
computeProxyTransportRetryDelayMs,
|
|
56
63
|
createDefaultProxyTransportRetryOptions,
|
|
64
|
+
isProxyTransportRetryMethod,
|
|
57
65
|
normalizeProxyTransportRetryOptions,
|
|
58
66
|
shouldRetryProxyTransportAttempt,
|
|
59
67
|
validateUnsafeProxyTransportRetryMethods,
|
|
@@ -73,6 +81,13 @@ import {
|
|
|
73
81
|
redactUrlQueryParams,
|
|
74
82
|
serializeRequestUrl,
|
|
75
83
|
} from "./request-options.js";
|
|
84
|
+
import type { ResolverVendorTransport } from "./resolver-vendors/types.js";
|
|
85
|
+
import {
|
|
86
|
+
type AkamaiSbsdSessionState,
|
|
87
|
+
akamaiSbsdChallengeKey,
|
|
88
|
+
detectAkamaiSbsdChallenge,
|
|
89
|
+
type StealthChallengeRuntime,
|
|
90
|
+
} from "./stealth-akamai-sbsd.js";
|
|
76
91
|
import { StealthCookieJar } from "./stealth-cookies.js";
|
|
77
92
|
|
|
78
93
|
export const DEFAULT_STEALTH_PROFILE: StealthProfileDescriptor = Object.freeze({
|
|
@@ -90,6 +105,21 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
90
105
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
91
106
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
92
107
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
108
|
+
// Failures that mean the bound ceremony endpoint did not carry the request to the origin.
|
|
109
|
+
// Proxy-level refusals are covered by isProxyPoolRefreshableError; PROXY_AUTH_IP_DENIED is
|
|
110
|
+
// deliberately absent because another endpoint of the same vendor is denied just the same.
|
|
111
|
+
const BOUND_EGRESS_TRANSPORT_FAILURE_CODES: ReadonlySet<string> = new Set([
|
|
112
|
+
PROXY_CONNECT_FAILURE_CODE,
|
|
113
|
+
"transport_network_error",
|
|
114
|
+
"transport_timeout",
|
|
115
|
+
]);
|
|
116
|
+
|
|
117
|
+
function isBoundEgressTransportFailure(error: TransportError): boolean {
|
|
118
|
+
return (
|
|
119
|
+
isProxyPoolRefreshableError(error) ||
|
|
120
|
+
(error.code !== undefined && BOUND_EGRESS_TRANSPORT_FAILURE_CODES.has(error.code))
|
|
121
|
+
);
|
|
122
|
+
}
|
|
93
123
|
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
94
124
|
const REDIRECT_BODY_HEADERS = new Set([
|
|
95
125
|
"content-encoding",
|
|
@@ -131,6 +161,35 @@ export type StealthClientOptions = ProxyResolutionOptions & {
|
|
|
131
161
|
proxyStealth?: { insecureSkipVerify?: boolean };
|
|
132
162
|
};
|
|
133
163
|
|
|
164
|
+
/** Server-attached challenge wiring and engine lease; kept off the public StealthClientOptions type. */
|
|
165
|
+
type StealthSessionClientOptions = StealthClientOptions & {
|
|
166
|
+
stealth?: StealthClientOptions["stealth"] & {
|
|
167
|
+
readonly challengeRuntime?: StealthChallengeRuntime;
|
|
168
|
+
};
|
|
169
|
+
readonly [ENGINE_CEREMONY_EGRESS_LEASE]?: CeremonyEgressLeaseRuntime;
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
type ChallengedReplayRecord = {
|
|
173
|
+
consumed: boolean;
|
|
174
|
+
/** Ceremony egress generation the challenged request ran on; a rebind retires the record. */
|
|
175
|
+
readonly generation: number;
|
|
176
|
+
replay(): Promise<StealthResponse>;
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
function assertAkamaiSbsdClientProfile(
|
|
180
|
+
declared: string | undefined,
|
|
181
|
+
session: StealthProfileDescriptor,
|
|
182
|
+
): void {
|
|
183
|
+
if (declared === undefined || resolverClientProfileFamily(declared) === session.browser) return;
|
|
184
|
+
throw new SDKError(
|
|
185
|
+
`Resolver client profile "${declared}" does not match the initiating stealth session browser "${session.browser}"`,
|
|
186
|
+
{
|
|
187
|
+
code: "RESOLVER_CLIENT_PROFILE_MISMATCH",
|
|
188
|
+
fix: "Make resolver.clientProfile name the provider stealth browser family (chrome, firefox, or safari).",
|
|
189
|
+
},
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
134
193
|
type StealthTransportHeaders = {
|
|
135
194
|
entries(): IterableIterator<[string, string]>;
|
|
136
195
|
get(name: string): string | null;
|
|
@@ -256,6 +315,25 @@ function resolveUrl(baseUrl: string, url: string): string {
|
|
|
256
315
|
return new URL(url, baseUrl).toString();
|
|
257
316
|
}
|
|
258
317
|
|
|
318
|
+
function resolveRequestUrl(baseUrl: string, url: string): string {
|
|
319
|
+
try {
|
|
320
|
+
return resolveUrl(baseUrl, url);
|
|
321
|
+
} catch (error) {
|
|
322
|
+
// Both inputs are caller-owned, so an unparsable URL is a programming error, never a
|
|
323
|
+
// transport condition: not retryable, not the upstream's fault. Classify it here rather
|
|
324
|
+
// than in normalizeStealthTransportError, which also sees ERR_INVALID_URL from malformed
|
|
325
|
+
// upstream redirect targets. The fixed message keeps the URL (and any inline query
|
|
326
|
+
// values) out of the diagnostic surface; the runtime TypeError stays in `cause`.
|
|
327
|
+
throw new TransportError("Invalid request URL", {
|
|
328
|
+
code: "transport_invalid_url",
|
|
329
|
+
status: 0,
|
|
330
|
+
category: "provider_error",
|
|
331
|
+
retryable: false,
|
|
332
|
+
...(error instanceof Error ? { cause: error } : {}),
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
259
337
|
function headerEntriesFromHeaders(headers: StealthTransportHeaders): [string, string][] {
|
|
260
338
|
return Array.from(headers.entries());
|
|
261
339
|
}
|
|
@@ -517,11 +595,12 @@ function requiredEmulationHeader(headers: ReadonlyMap<string, string>, name: str
|
|
|
517
595
|
function buildChromeHeaderTuples(options: {
|
|
518
596
|
emulationHeaders: Iterable<[string, string]>;
|
|
519
597
|
method: StealthMethod;
|
|
520
|
-
body?: string;
|
|
598
|
+
body?: string | Buffer;
|
|
521
599
|
headers: Record<string, string>;
|
|
522
600
|
requestUrl: string;
|
|
523
601
|
acceptLanguage?: string;
|
|
524
602
|
requestClass?: ChromeRequestClass;
|
|
603
|
+
userActivation?: boolean;
|
|
525
604
|
}): HeaderTuple[] {
|
|
526
605
|
const callerEntries = normalizedCallerHeaderEntriesFromRecord(options.headers);
|
|
527
606
|
const caller = new Map(callerEntries);
|
|
@@ -558,7 +637,7 @@ function buildChromeHeaderTuples(options: {
|
|
|
558
637
|
]);
|
|
559
638
|
if (isNavigation) {
|
|
560
639
|
values.set("upgrade-insecure-requests", "1");
|
|
561
|
-
values.set("sec-fetch-user", "?1");
|
|
640
|
+
if (options.userActivation !== false) values.set("sec-fetch-user", "?1");
|
|
562
641
|
}
|
|
563
642
|
for (const [name, value] of callerEntries) values.set(name, value);
|
|
564
643
|
for (const name of ["content-type", "origin", "referer"] as const) {
|
|
@@ -626,6 +705,33 @@ function setCookieHeadersFromResponse(headers: StealthTransportHeaders): string[
|
|
|
626
705
|
return setCookie ? splitCombinedSetCookieHeader(setCookie) : [];
|
|
627
706
|
}
|
|
628
707
|
|
|
708
|
+
function resolverCookiesFromResponse(
|
|
709
|
+
response: StealthTransportResponse,
|
|
710
|
+
requestUrl: string,
|
|
711
|
+
): Awaited<ReturnType<ResolverVendorTransport["fetch"]>>["cookies"] {
|
|
712
|
+
const responseUrl = response.url ?? requestUrl;
|
|
713
|
+
return setCookieHeadersFromResponse(response.headers).flatMap((cookieString) => {
|
|
714
|
+
const cookie = Cookie.parse(cookieString);
|
|
715
|
+
if (!cookie?.key) return [];
|
|
716
|
+
const expires =
|
|
717
|
+
cookie.expires instanceof Date && Number.isFinite(cookie.expires.getTime())
|
|
718
|
+
? cookie.expires.getTime() / 1_000
|
|
719
|
+
: undefined;
|
|
720
|
+
return [
|
|
721
|
+
{
|
|
722
|
+
name: cookie.key,
|
|
723
|
+
value: cookie.value,
|
|
724
|
+
...(expires === undefined ? {} : { expires }),
|
|
725
|
+
httpOnly: cookie.httpOnly,
|
|
726
|
+
secure: cookie.secure,
|
|
727
|
+
...(cookie.domain ? { domain: cookie.domain } : { domain: new URL(responseUrl).hostname }),
|
|
728
|
+
...(cookie.path ? { path: cookie.path } : {}),
|
|
729
|
+
...(cookie.sameSite ? { sameSite: cookie.sameSite } : {}),
|
|
730
|
+
},
|
|
731
|
+
];
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
|
|
629
735
|
function splitCombinedSetCookieHeader(headerValue: string): string[] {
|
|
630
736
|
const cookieNamePattern = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+=/;
|
|
631
737
|
const cookieStrings: string[] = [];
|
|
@@ -831,9 +937,9 @@ function readResponseBodyChunk(
|
|
|
831
937
|
});
|
|
832
938
|
}
|
|
833
939
|
|
|
834
|
-
function normalizeBody(body:
|
|
940
|
+
function normalizeBody(body: unknown): string | Buffer | undefined {
|
|
835
941
|
if (body === undefined) {
|
|
836
|
-
return
|
|
942
|
+
return undefined;
|
|
837
943
|
}
|
|
838
944
|
|
|
839
945
|
if (typeof body === "string") {
|
|
@@ -841,10 +947,15 @@ function normalizeBody(body: StealthFetchOptions["body"]): string {
|
|
|
841
947
|
}
|
|
842
948
|
|
|
843
949
|
if (Buffer.isBuffer(body)) {
|
|
844
|
-
return
|
|
950
|
+
return Buffer.from(body);
|
|
845
951
|
}
|
|
846
952
|
|
|
847
|
-
|
|
953
|
+
// `StealthFetchOptions.body` is `string | Buffer`; anything else is a provider fault,
|
|
954
|
+
// caught before the first request so a later replay can send the same bytes.
|
|
955
|
+
throw new SDKError("Stealth request bodies must be a string or Buffer", {
|
|
956
|
+
code: "STEALTH_BODY_UNSUPPORTED",
|
|
957
|
+
fix: "Supply a string or Buffer body; streams and typed arrays are not snapshotted.",
|
|
958
|
+
});
|
|
848
959
|
}
|
|
849
960
|
|
|
850
961
|
function isPolicyManagedProxy(options: StealthClientOptions): boolean {
|
|
@@ -890,6 +1001,8 @@ type ResolvedAttemptProxy = {
|
|
|
890
1001
|
poolIndex?: number;
|
|
891
1002
|
proxyHash?: string;
|
|
892
1003
|
vendor?: ProxyVendorName;
|
|
1004
|
+
refreshEpoch?: number;
|
|
1005
|
+
lifetimeMinutes?: number;
|
|
893
1006
|
};
|
|
894
1007
|
|
|
895
1008
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -902,6 +1015,14 @@ function proxyPoolIndexFromDiagnostics(
|
|
|
902
1015
|
return Math.floor(value);
|
|
903
1016
|
}
|
|
904
1017
|
|
|
1018
|
+
function positiveDiagnosticNumber(
|
|
1019
|
+
diagnostics: Record<string, string | number | boolean> | undefined,
|
|
1020
|
+
name: string,
|
|
1021
|
+
): number | undefined {
|
|
1022
|
+
const value = diagnostics?.[name];
|
|
1023
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
|
1024
|
+
}
|
|
1025
|
+
|
|
905
1026
|
function proxyEndpointHash(proxyUrl: string | undefined): string | undefined {
|
|
906
1027
|
if (!proxyUrl) return undefined;
|
|
907
1028
|
try {
|
|
@@ -1099,7 +1220,7 @@ async function fetchStealthRedirectChain(
|
|
|
1099
1220
|
buildHeaders?: (
|
|
1100
1221
|
url: string,
|
|
1101
1222
|
method: StealthMethod,
|
|
1102
|
-
body: string | undefined,
|
|
1223
|
+
body: string | Buffer | undefined,
|
|
1103
1224
|
headers: Record<string, string>,
|
|
1104
1225
|
) => HeaderTuple[],
|
|
1105
1226
|
): Promise<{ normalized: StealthResponse; response: StealthTransportResponse }> {
|
|
@@ -1196,13 +1317,81 @@ async function fetchStealthRedirectChain(
|
|
|
1196
1317
|
function createSessionFetcher(
|
|
1197
1318
|
baseUrl: string,
|
|
1198
1319
|
defaultProfile: StealthProfileDescriptor,
|
|
1199
|
-
clientOptions:
|
|
1320
|
+
clientOptions: StealthSessionClientOptions,
|
|
1200
1321
|
): StealthSession {
|
|
1201
1322
|
const clients = new Map<string, WreqSessionCacheEntry>();
|
|
1202
1323
|
let closed = false;
|
|
1203
1324
|
let hasWarnedMissingProxy = false;
|
|
1204
1325
|
const warn = clientOptions.warn ?? console.warn;
|
|
1205
1326
|
const cookieJar = new StealthCookieJar([], baseUrl);
|
|
1327
|
+
const akamaiSbsdState: AkamaiSbsdSessionState = { transactions: new Map() };
|
|
1328
|
+
const challengedReplays = new WeakMap<StealthResponse, ChallengedReplayRecord>();
|
|
1329
|
+
const ceremonyEgressLease = clientOptions[ENGINE_CEREMONY_EGRESS_LEASE];
|
|
1330
|
+
// The lease is shared by every session of this client; this is the lease generation this
|
|
1331
|
+
// session's challenge state and replay records were built against.
|
|
1332
|
+
let observedEgressGeneration = ceremonyEgressLease?.generation ?? 0;
|
|
1333
|
+
const automaticChallengeRefetchPolicy = {
|
|
1334
|
+
...createDefaultProxyTransportRetryOptions({ label: "Stealth" }),
|
|
1335
|
+
methods: ["GET"],
|
|
1336
|
+
};
|
|
1337
|
+
|
|
1338
|
+
function clearAkamaiSbsdState(): void {
|
|
1339
|
+
akamaiSbsdState.rememberedScript = undefined;
|
|
1340
|
+
akamaiSbsdState.transactions.clear();
|
|
1341
|
+
akamaiSbsdState.completedSuccessKey = undefined;
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
function currentEgressGeneration(): number {
|
|
1345
|
+
return ceremonyEgressLease?.generation ?? 0;
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/**
|
|
1349
|
+
* Whoever dropped the binding (this session by expiry or release, or another session of
|
|
1350
|
+
* the same client), the challenge state that was only valid on the retired endpoint goes
|
|
1351
|
+
* with it, and replay records made against the old generation stop matching.
|
|
1352
|
+
*/
|
|
1353
|
+
function syncCeremonyEgress(): void {
|
|
1354
|
+
const generation = currentEgressGeneration();
|
|
1355
|
+
if (generation === observedEgressGeneration) return;
|
|
1356
|
+
observedEgressGeneration = generation;
|
|
1357
|
+
clearAkamaiSbsdState();
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/**
|
|
1361
|
+
* A binding past its vendor session lifetime is dropped, together with the challenge
|
|
1362
|
+
* state that was only valid on that endpoint, so this request selects and binds afresh.
|
|
1363
|
+
*/
|
|
1364
|
+
function expireCeremonyEgressLease(): void {
|
|
1365
|
+
ceremonyEgressLease?.dropExpiredBinding();
|
|
1366
|
+
syncCeremonyEgress();
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
/**
|
|
1370
|
+
* The lease is best effort (ADR-0010: a raw Smartproxy endpoint is not a vendor-guaranteed
|
|
1371
|
+
* lease). When the attempt that ran on the bound endpoint fails at the transport (the
|
|
1372
|
+
* proxy refused it, the tunnel never opened, the socket died or timed out) the binding is
|
|
1373
|
+
* released so the ceremony can leave a dead endpoint before its lifetime elapses. Origin
|
|
1374
|
+
* responses, however bad, keep the binding: the endpoint is alive. Returns true when a
|
|
1375
|
+
* binding was released.
|
|
1376
|
+
*/
|
|
1377
|
+
function releaseFailedCeremonyEgress(
|
|
1378
|
+
attemptProxyUrl: string | undefined,
|
|
1379
|
+
error: TransportError,
|
|
1380
|
+
): boolean {
|
|
1381
|
+
const bound = ceremonyEgressLease?.binding;
|
|
1382
|
+
if (
|
|
1383
|
+
!ceremonyEgressLease ||
|
|
1384
|
+
!bound ||
|
|
1385
|
+
!attemptProxyUrl ||
|
|
1386
|
+
attemptProxyUrl !== bound.proxyUrl ||
|
|
1387
|
+
!isBoundEgressTransportFailure(error)
|
|
1388
|
+
) {
|
|
1389
|
+
return false;
|
|
1390
|
+
}
|
|
1391
|
+
ceremonyEgressLease.dropBinding();
|
|
1392
|
+
syncCeremonyEgress();
|
|
1393
|
+
return true;
|
|
1394
|
+
}
|
|
1206
1395
|
|
|
1207
1396
|
async function getClientEntry(
|
|
1208
1397
|
profile: StealthProfileDescriptor,
|
|
@@ -1304,6 +1493,26 @@ function createSessionFetcher(
|
|
|
1304
1493
|
proxyAttempt?: number,
|
|
1305
1494
|
refreshEpoch?: number,
|
|
1306
1495
|
): Promise<ResolvedAttemptProxy> {
|
|
1496
|
+
const bound = ceremonyEgressLease?.binding;
|
|
1497
|
+
if (bound) {
|
|
1498
|
+
if (options?.proxy !== undefined || options?.proxyAttemptOffset !== undefined) {
|
|
1499
|
+
throw new SDKError(
|
|
1500
|
+
"A ceremony-bound egress cannot be overridden by provider request options",
|
|
1501
|
+
{
|
|
1502
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1503
|
+
fix: "Remove `proxy` and `proxyAttemptOffset` from stealth requests made inside an auth ceremony.",
|
|
1504
|
+
},
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1507
|
+
return {
|
|
1508
|
+
url: bound.proxyUrl,
|
|
1509
|
+
poolIndex: bound.poolIndex,
|
|
1510
|
+
proxyHash: proxyEndpointHash(bound.proxyUrl),
|
|
1511
|
+
vendor: bound.vendor,
|
|
1512
|
+
refreshEpoch: bound.refreshEpoch,
|
|
1513
|
+
lifetimeMinutes: bound.lifetimeMinutes,
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1307
1516
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
1308
1517
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
1309
1518
|
upstream: clientOptions.upstream,
|
|
@@ -1318,6 +1527,7 @@ function createSessionFetcher(
|
|
|
1318
1527
|
transportProtocols: ["http", "socks5"],
|
|
1319
1528
|
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
1320
1529
|
telemetry: clientOptions.telemetry,
|
|
1530
|
+
engineCredentials: clientOptions.engineCredentials,
|
|
1321
1531
|
});
|
|
1322
1532
|
|
|
1323
1533
|
if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
|
|
@@ -1330,12 +1540,77 @@ function createSessionFetcher(
|
|
|
1330
1540
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
1331
1541
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
1332
1542
|
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
1543
|
+
refreshEpoch: refreshEpoch ?? 0,
|
|
1544
|
+
...(positiveDiagnosticNumber(resolvedProxy.diagnostics, "lifetimeMinutes") === undefined
|
|
1545
|
+
? {}
|
|
1546
|
+
: {
|
|
1547
|
+
lifetimeMinutes: positiveDiagnosticNumber(resolvedProxy.diagnostics, "lifetimeMinutes"),
|
|
1548
|
+
}),
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
function bindCeremonyEgress(attemptProxy: ResolvedAttemptProxy | undefined): void {
|
|
1553
|
+
const lease = ceremonyEgressLease;
|
|
1554
|
+
if (!lease || !attemptProxy?.url || !attemptProxy.vendor) return;
|
|
1555
|
+
if (lease.binding) {
|
|
1556
|
+
// A concurrent request on this session may have bound the ceremony while this one
|
|
1557
|
+
// was in flight on an endpoint it selected before the binding existed (pool
|
|
1558
|
+
// rotation after a transport failure). Its response reached the origin from an
|
|
1559
|
+
// egress the handle does not name; accepting it would split the ceremony identity.
|
|
1560
|
+
if (attemptProxy.url !== lease.binding.proxyUrl) {
|
|
1561
|
+
throw new SDKError("The response arrived on an egress other than the ceremony's bound endpoint", {
|
|
1562
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
return;
|
|
1566
|
+
}
|
|
1567
|
+
if (
|
|
1568
|
+
attemptProxy.poolIndex === undefined ||
|
|
1569
|
+
attemptProxy.lifetimeMinutes === undefined ||
|
|
1570
|
+
!clientOptions.affinityKey
|
|
1571
|
+
) {
|
|
1572
|
+
// Registry vendors always report pool index and session lifetime; missing either
|
|
1573
|
+
// is an SDK fault, not something the caller can correct.
|
|
1574
|
+
throw new SDKError("The selected egress cannot be represented by a ceremony lease", {
|
|
1575
|
+
code: "EGRESS_LEASE_BINDING_INVALID",
|
|
1576
|
+
});
|
|
1577
|
+
}
|
|
1578
|
+
const binding: CeremonyEgressBinding = {
|
|
1579
|
+
vendor: attemptProxy.vendor,
|
|
1580
|
+
proxyUrl: attemptProxy.url,
|
|
1581
|
+
poolIndex: attemptProxy.poolIndex,
|
|
1582
|
+
affinityKey: clientOptions.affinityKey,
|
|
1583
|
+
refreshEpoch: attemptProxy.refreshEpoch ?? 0,
|
|
1584
|
+
lifetimeMinutes: attemptProxy.lifetimeMinutes,
|
|
1333
1585
|
};
|
|
1586
|
+
lease.bind(binding);
|
|
1334
1587
|
}
|
|
1335
1588
|
|
|
1336
1589
|
const session: StealthSession = {
|
|
1337
|
-
async fetch(url,
|
|
1590
|
+
async fetch(url, callerOptions: StealthFetchOptions = {}) {
|
|
1591
|
+
expireCeremonyEgressLease();
|
|
1592
|
+
const requestBody = normalizeBody(callerOptions.body);
|
|
1593
|
+
// Snapshot the caller's headers with the body before the first send: the first
|
|
1594
|
+
// request and an explicit replay both use this copy, so a caller mutating its own
|
|
1595
|
+
// objects afterwards cannot make the two sends differ.
|
|
1596
|
+
const options: StealthFetchOptions = {
|
|
1597
|
+
...callerOptions,
|
|
1598
|
+
...(callerOptions.headers
|
|
1599
|
+
? {
|
|
1600
|
+
headers: Object.fromEntries(
|
|
1601
|
+
Object.entries(callerOptions.headers).map(([name, value]) => [
|
|
1602
|
+
name,
|
|
1603
|
+
Array.isArray(value) ? [...value] : value,
|
|
1604
|
+
]),
|
|
1605
|
+
),
|
|
1606
|
+
}
|
|
1607
|
+
: {}),
|
|
1608
|
+
...(requestBody === undefined ? {} : { body: requestBody }),
|
|
1609
|
+
};
|
|
1338
1610
|
const requestProfile = resolveStealthProfileSelection(options.stealth, defaultProfile);
|
|
1611
|
+
let challengeSolveAttempted = false;
|
|
1612
|
+
let challengeRefetchAttempted = false;
|
|
1613
|
+
let challengeSolveFailure: { readonly error: unknown } | undefined;
|
|
1339
1614
|
const { hasExplicitRetryPolicy, method, stealthRetryOptions } = (() => {
|
|
1340
1615
|
try {
|
|
1341
1616
|
const method = normalizeMethod(options.method ?? "GET");
|
|
@@ -1387,7 +1662,10 @@ function createSessionFetcher(
|
|
|
1387
1662
|
// transport-retry budget instead.
|
|
1388
1663
|
const rotatesRegistryChain =
|
|
1389
1664
|
usesPolicyAllocator && policyResolvesRegistryVendorChain(policyProxy);
|
|
1390
|
-
const
|
|
1665
|
+
const unboundMaxAttempts = rotatesRegistryChain ? policyProxyAttemptCap : retryAttemptCap;
|
|
1666
|
+
// A bound ceremony has exactly one endpoint to try; releasing the binding after a
|
|
1667
|
+
// transport failure widens the budget back to the ordinary rotation span.
|
|
1668
|
+
let maxAttempts = ceremonyEgressLease?.binding ? 1 : unboundMaxAttempts;
|
|
1391
1669
|
const dedupeAllocatorEndpoints = rotatesRegistryChain;
|
|
1392
1670
|
let lastError: unknown;
|
|
1393
1671
|
|
|
@@ -1409,8 +1687,9 @@ function createSessionFetcher(
|
|
|
1409
1687
|
let fallbackSensitiveValues: readonly string[] = [];
|
|
1410
1688
|
let fallbackRequestUrl: string | undefined;
|
|
1411
1689
|
let fallbackRedactedUrl: string | undefined;
|
|
1412
|
-
|
|
1690
|
+
let attemptStartedAt = Date.now();
|
|
1413
1691
|
let attemptRecorded = false;
|
|
1692
|
+
const attemptEgressGeneration = currentEgressGeneration();
|
|
1414
1693
|
const recordProxyAttempt = (
|
|
1415
1694
|
outcome: "ok" | "error",
|
|
1416
1695
|
errorCode?: string,
|
|
@@ -1431,6 +1710,68 @@ function createSessionFetcher(
|
|
|
1431
1710
|
durationMs: Date.now() - attemptStartedAt,
|
|
1432
1711
|
});
|
|
1433
1712
|
};
|
|
1713
|
+
// Shared tail of the first fetch, the automatic refetch, and an explicit replay: a
|
|
1714
|
+
// classified challenge is returned whatever `throwOnHttpError` says (documented on
|
|
1715
|
+
// StealthFetchOptions), any other non-2xx honours it, and the proxy attempt is
|
|
1716
|
+
// recorded once per delivered response.
|
|
1717
|
+
const finalizeAttempt = (
|
|
1718
|
+
attemptResponse: StealthTransportResponse,
|
|
1719
|
+
attemptNormalized: StealthResponse,
|
|
1720
|
+
): StealthResponse => {
|
|
1721
|
+
if (
|
|
1722
|
+
!attemptNormalized.challenge &&
|
|
1723
|
+
attemptResponse.status >= 400 &&
|
|
1724
|
+
options.throwOnHttpError !== false
|
|
1725
|
+
) {
|
|
1726
|
+
throw new TransportError(
|
|
1727
|
+
`Upstream request failed with status ${attemptResponse.status}`,
|
|
1728
|
+
{ code: "upstream_http_error", status: attemptResponse.status },
|
|
1729
|
+
);
|
|
1730
|
+
}
|
|
1731
|
+
recordProxyAttempt("ok", undefined, attemptResponse.status);
|
|
1732
|
+
return attemptNormalized;
|
|
1733
|
+
};
|
|
1734
|
+
const redactAttemptError = <T>(error: T): T =>
|
|
1735
|
+
redactSensitiveError(
|
|
1736
|
+
error,
|
|
1737
|
+
serializedUrl?.sensitiveValues ?? fallbackSensitiveValues,
|
|
1738
|
+
serializedUrl?.requestUrl ?? fallbackRequestUrl,
|
|
1739
|
+
serializedUrl?.redactedUrl ?? fallbackRedactedUrl,
|
|
1740
|
+
);
|
|
1741
|
+
// Cancellation and resolver failures leave the transport path here, redacted; any
|
|
1742
|
+
// other failure becomes the TransportError the retry decision reasons about.
|
|
1743
|
+
const normalizeAttemptError = (error: unknown): TransportError => {
|
|
1744
|
+
let normalizedError: TransportError;
|
|
1745
|
+
try {
|
|
1746
|
+
throwIfAmbientAborted(clientOptions.signal);
|
|
1747
|
+
normalizedError = normalizeStealthTransportError(error);
|
|
1748
|
+
} catch (normalizationError) {
|
|
1749
|
+
if (
|
|
1750
|
+
normalizationError instanceof TransportError &&
|
|
1751
|
+
normalizationError.code === "transport_cancelled"
|
|
1752
|
+
) {
|
|
1753
|
+
recordProxyAttempt(
|
|
1754
|
+
"error",
|
|
1755
|
+
proxyAttemptErrorCode(normalizationError),
|
|
1756
|
+
proxyAttemptStatus(normalizationError),
|
|
1757
|
+
);
|
|
1758
|
+
}
|
|
1759
|
+
throw redactAttemptError(normalizationError);
|
|
1760
|
+
}
|
|
1761
|
+
if (challengeSolveFailure && error === challengeSolveFailure) {
|
|
1762
|
+
throw redactAttemptError(challengeSolveFailure.error);
|
|
1763
|
+
}
|
|
1764
|
+
return normalizedError;
|
|
1765
|
+
};
|
|
1766
|
+
const recordAttemptFailure = (normalizedError: TransportError): TransportError => {
|
|
1767
|
+
const redactedError = redactAttemptError(normalizedError);
|
|
1768
|
+
recordProxyAttempt(
|
|
1769
|
+
"error",
|
|
1770
|
+
proxyAttemptErrorCode(redactedError),
|
|
1771
|
+
proxyAttemptStatus(redactedError),
|
|
1772
|
+
);
|
|
1773
|
+
return redactedError;
|
|
1774
|
+
};
|
|
1434
1775
|
try {
|
|
1435
1776
|
throwIfAmbientAborted(clientOptions.signal);
|
|
1436
1777
|
const sensitiveParams = normalizeSensitiveParams(options.sensitiveParams);
|
|
@@ -1463,24 +1804,25 @@ function createSessionFetcher(
|
|
|
1463
1804
|
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
1464
1805
|
);
|
|
1465
1806
|
serializedUrl = serializeRequestUrl(
|
|
1466
|
-
|
|
1807
|
+
resolveRequestUrl(baseUrl, url),
|
|
1467
1808
|
options.params,
|
|
1468
1809
|
sensitiveParams,
|
|
1469
1810
|
);
|
|
1470
1811
|
const { requestUrl } = serializedUrl;
|
|
1471
1812
|
const wreq = await getWreqModule();
|
|
1472
1813
|
const mapping = resolveWreqProfile(requestProfile, wreq.getProfiles());
|
|
1814
|
+
const emulationHeaders = Array.from(
|
|
1815
|
+
wreq.getEmulationHeaders(mapping.browser, mapping.os),
|
|
1816
|
+
([name, value]) => [String(name), String(value)] as HeaderTuple,
|
|
1817
|
+
);
|
|
1473
1818
|
const chromeEmulationHeaders = mapping.browser.startsWith("chrome_")
|
|
1474
|
-
?
|
|
1475
|
-
wreq.getEmulationHeaders(mapping.browser, mapping.os),
|
|
1476
|
-
([name, value]) => [String(name), String(value)] as HeaderTuple,
|
|
1477
|
-
)
|
|
1819
|
+
? emulationHeaders
|
|
1478
1820
|
: undefined;
|
|
1479
1821
|
const buildOrderedHeaders = chromeEmulationHeaders
|
|
1480
1822
|
? (
|
|
1481
1823
|
currentUrl: string,
|
|
1482
1824
|
currentMethod: StealthMethod,
|
|
1483
|
-
currentBody: string | undefined,
|
|
1825
|
+
currentBody: string | Buffer | undefined,
|
|
1484
1826
|
currentHeaders: Record<string, string>,
|
|
1485
1827
|
) =>
|
|
1486
1828
|
buildChromeHeaderTuples({
|
|
@@ -1491,6 +1833,7 @@ function createSessionFetcher(
|
|
|
1491
1833
|
requestUrl: currentUrl,
|
|
1492
1834
|
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1493
1835
|
requestClass: options.stealth?.requestClass,
|
|
1836
|
+
userActivation: options.stealth?.userActivation,
|
|
1494
1837
|
})
|
|
1495
1838
|
: undefined;
|
|
1496
1839
|
const initialHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
@@ -1501,110 +1844,358 @@ function createSessionFetcher(
|
|
|
1501
1844
|
const defaultHeaders = buildOrderedHeaders?.(
|
|
1502
1845
|
requestUrl,
|
|
1503
1846
|
method,
|
|
1504
|
-
|
|
1847
|
+
requestBody,
|
|
1505
1848
|
initialHeaders,
|
|
1506
1849
|
);
|
|
1507
|
-
const
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1850
|
+
const fetchOnBoundSession = (
|
|
1851
|
+
fetchUrl: string,
|
|
1852
|
+
fetchMethod: StealthMethod,
|
|
1853
|
+
fetchOptions: StealthFetchOptions,
|
|
1854
|
+
fetchSignal: AbortSignal | undefined,
|
|
1855
|
+
orderedHeaders = buildOrderedHeaders,
|
|
1856
|
+
sessionDefaultHeaders = defaultHeaders,
|
|
1857
|
+
) =>
|
|
1858
|
+
withClient(
|
|
1859
|
+
requestProfile,
|
|
1860
|
+
proxy,
|
|
1861
|
+
ignoreTlsErrors,
|
|
1862
|
+
(transport) =>
|
|
1863
|
+
fetchStealthRedirectChain(
|
|
1864
|
+
transport,
|
|
1865
|
+
cookieJar,
|
|
1866
|
+
fetchUrl,
|
|
1867
|
+
fetchMethod,
|
|
1868
|
+
fetchOptions,
|
|
1869
|
+
fetchSignal,
|
|
1870
|
+
orderedHeaders,
|
|
1871
|
+
),
|
|
1872
|
+
fetchSignal,
|
|
1873
|
+
sessionDefaultHeaders,
|
|
1874
|
+
);
|
|
1875
|
+
const throwProxyTransportFault = (
|
|
1876
|
+
faultResponse: StealthTransportResponse,
|
|
1877
|
+
faultBody: string,
|
|
1878
|
+
) => {
|
|
1879
|
+
if (!proxy) return;
|
|
1880
|
+
if (isProxyConnectFailureResponse(faultResponse, faultBody)) {
|
|
1881
|
+
throw createProxyConnectFailureError(faultBody);
|
|
1882
|
+
}
|
|
1883
|
+
if (faultResponse.status < 400) return;
|
|
1530
1884
|
if (
|
|
1531
|
-
proxy &&
|
|
1532
1885
|
usesPolicyAllocator &&
|
|
1533
1886
|
isProxyEdgeTlsRejectedResponse(
|
|
1534
|
-
|
|
1535
|
-
[JSON.stringify(responseHeadersToRecord(
|
|
1887
|
+
faultResponse.status,
|
|
1888
|
+
[JSON.stringify(responseHeadersToRecord(faultResponse.headers)), faultBody].join(
|
|
1536
1889
|
"\n",
|
|
1537
1890
|
),
|
|
1538
1891
|
)
|
|
1539
1892
|
) {
|
|
1540
|
-
throw createProxyEdgeTlsRejectedError(
|
|
1893
|
+
throw createProxyEdgeTlsRejectedError(faultResponse.status);
|
|
1541
1894
|
}
|
|
1542
|
-
if (
|
|
1895
|
+
if (isProxyAuthIpDeniedMessage(faultBody)) {
|
|
1543
1896
|
throw createProxyAuthIpDeniedError();
|
|
1544
1897
|
}
|
|
1545
|
-
if (
|
|
1898
|
+
if (isProxyEdgeAuthRejectedMessage(faultBody)) {
|
|
1546
1899
|
throw createProxyEdgeAuthRejectedError();
|
|
1547
1900
|
}
|
|
1548
1901
|
if (
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
isProxyPoolStaleMessage(normalized.body)
|
|
1902
|
+
isProxyPoolStaleStatus(faultResponse.status) &&
|
|
1903
|
+
isProxyPoolStaleMessage(faultBody)
|
|
1552
1904
|
) {
|
|
1553
|
-
throw createProxyPoolStaleError(
|
|
1905
|
+
throw createProxyPoolStaleError(faultResponse.status);
|
|
1554
1906
|
}
|
|
1555
|
-
}
|
|
1907
|
+
};
|
|
1908
|
+
let { normalized, response } = await fetchOnBoundSession(
|
|
1909
|
+
requestUrl,
|
|
1910
|
+
method,
|
|
1911
|
+
options,
|
|
1912
|
+
clientOptions.signal,
|
|
1913
|
+
);
|
|
1914
|
+
throwProxyTransportFault(response, normalized.body);
|
|
1556
1915
|
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
});
|
|
1562
|
-
}
|
|
1916
|
+
// Commit only after the selected endpoint reached the origin. A failed
|
|
1917
|
+
// proxy attempt may rotate pool index; the successful exact endpoint is
|
|
1918
|
+
// the one sealed into the ceremony handle.
|
|
1919
|
+
bindCeremonyEgress(attemptProxy);
|
|
1563
1920
|
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1921
|
+
const akamaiSbsd = clientOptions.stealth?.challengeRuntime?.akamaiSbsd;
|
|
1922
|
+
const detected = akamaiSbsd
|
|
1923
|
+
? detectAkamaiSbsdChallenge(
|
|
1924
|
+
normalized,
|
|
1925
|
+
requestUrl,
|
|
1926
|
+
cookieJar,
|
|
1927
|
+
akamaiSbsd.allowedHosts,
|
|
1928
|
+
akamaiSbsdState,
|
|
1929
|
+
)
|
|
1930
|
+
: undefined;
|
|
1931
|
+
if (detected && akamaiSbsd) {
|
|
1932
|
+
if (challengeSolveAttempted || challengeRefetchAttempted) {
|
|
1933
|
+
normalized.challenge = {
|
|
1934
|
+
challenge: detected,
|
|
1935
|
+
outcome: "challenge_persisted",
|
|
1936
|
+
};
|
|
1937
|
+
return finalizeAttempt(response, normalized);
|
|
1938
|
+
}
|
|
1939
|
+
const automaticReplayEligible = isProxyTransportRetryMethod(
|
|
1940
|
+
method,
|
|
1941
|
+
automaticChallengeRefetchPolicy,
|
|
1942
|
+
{ body: requestBody, headers: options.headers },
|
|
1578
1943
|
);
|
|
1579
|
-
if (
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1944
|
+
if (automaticReplayEligible && !akamaiSbsd.solve) {
|
|
1945
|
+
normalized.challenge = {
|
|
1946
|
+
challenge: detected,
|
|
1947
|
+
outcome: "resolver_unavailable",
|
|
1948
|
+
};
|
|
1949
|
+
return finalizeAttempt(response, normalized);
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
const emulationHeaderMap = new Map(
|
|
1953
|
+
emulationHeaders.map(([name, value]) => [name.toLowerCase(), value] as const),
|
|
1954
|
+
);
|
|
1955
|
+
const initiatingHeaders = normalizeHeaders({ ...(options.headers ?? {}) });
|
|
1956
|
+
// Mirror what the protected request actually sent: `stealth.acceptLanguage` is
|
|
1957
|
+
// applied by the Chrome header builder only; Safari/Firefox sessions send the
|
|
1958
|
+
// emulation default.
|
|
1959
|
+
const sessionHeaders = {
|
|
1960
|
+
"User-Agent": requiredEmulationHeader(emulationHeaderMap, "user-agent"),
|
|
1961
|
+
"Accept-Language":
|
|
1962
|
+
initiatingHeaders["accept-language"] ??
|
|
1963
|
+
(chromeEmulationHeaders ? clientOptions.stealth?.acceptLanguage : undefined) ??
|
|
1964
|
+
requiredEmulationHeader(emulationHeaderMap, "accept-language"),
|
|
1965
|
+
};
|
|
1966
|
+
const resolverBuildHeaders = chromeEmulationHeaders
|
|
1967
|
+
? (
|
|
1968
|
+
currentUrl: string,
|
|
1969
|
+
currentMethod: StealthMethod,
|
|
1970
|
+
currentBody: string | Buffer | undefined,
|
|
1971
|
+
currentHeaders: Record<string, string>,
|
|
1972
|
+
) =>
|
|
1973
|
+
buildChromeHeaderTuples({
|
|
1974
|
+
emulationHeaders: chromeEmulationHeaders,
|
|
1975
|
+
method: currentMethod,
|
|
1976
|
+
body: currentBody,
|
|
1977
|
+
headers: currentHeaders,
|
|
1978
|
+
requestUrl: currentUrl,
|
|
1979
|
+
acceptLanguage: clientOptions.stealth?.acceptLanguage,
|
|
1980
|
+
})
|
|
1981
|
+
: undefined;
|
|
1982
|
+
const resolverTransport: ResolverVendorTransport = {
|
|
1983
|
+
sessionHeaders,
|
|
1984
|
+
getCookie(name, cookieUrl) {
|
|
1985
|
+
return cookieJar.get(name, cookieUrl);
|
|
1986
|
+
},
|
|
1987
|
+
async fetch(transportUrl, init) {
|
|
1988
|
+
const boundSignal = clientOptions.signal
|
|
1989
|
+
? AbortSignal.any([clientOptions.signal, init.signal])
|
|
1990
|
+
: init.signal;
|
|
1991
|
+
const transportHeaders = { ...(init.headers ?? {}) };
|
|
1992
|
+
if (resolverBuildHeaders) {
|
|
1993
|
+
for (const name of Object.keys(transportHeaders)) {
|
|
1994
|
+
if (name.toLowerCase() === "user-agent") {
|
|
1995
|
+
delete transportHeaders[name];
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
// Session defaults come from this request's own shape, never from the
|
|
2000
|
+
// initiating request: its defaults carry the upstream Cookie header,
|
|
2001
|
+
// which wreq would otherwise merge into the Hyper /ip call.
|
|
2002
|
+
const result = await fetchOnBoundSession(
|
|
2003
|
+
transportUrl,
|
|
2004
|
+
init.method,
|
|
2005
|
+
{
|
|
2006
|
+
method: init.method,
|
|
2007
|
+
headers: transportHeaders,
|
|
2008
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
2009
|
+
redirect: "manual",
|
|
2010
|
+
throwOnHttpError: false,
|
|
2011
|
+
...(init.maxBodyBytes === undefined
|
|
2012
|
+
? {}
|
|
2013
|
+
: { maxBodyBytes: init.maxBodyBytes }),
|
|
2014
|
+
},
|
|
2015
|
+
boundSignal,
|
|
2016
|
+
resolverBuildHeaders,
|
|
2017
|
+
resolverBuildHeaders?.(
|
|
2018
|
+
transportUrl,
|
|
2019
|
+
init.method,
|
|
2020
|
+
init.body,
|
|
2021
|
+
normalizeHeaders(transportHeaders),
|
|
2022
|
+
),
|
|
2023
|
+
);
|
|
2024
|
+
return {
|
|
2025
|
+
status: result.normalized.status,
|
|
2026
|
+
headers: result.normalized.headers,
|
|
2027
|
+
body: result.normalized.body,
|
|
2028
|
+
cookies: resolverCookiesFromResponse(result.response, transportUrl),
|
|
2029
|
+
};
|
|
2030
|
+
},
|
|
2031
|
+
};
|
|
2032
|
+
// `options` is already the fetch-start snapshot; the replay gets its own copy
|
|
2033
|
+
// of the body bytes so the transport cannot alias the first send's buffer.
|
|
2034
|
+
const replayOptions: StealthFetchOptions = {
|
|
2035
|
+
...options,
|
|
2036
|
+
...(Buffer.isBuffer(requestBody) ? { body: Buffer.from(requestBody) } : {}),
|
|
2037
|
+
};
|
|
2038
|
+
const transactionKey = akamaiSbsdChallengeKey(detected, {
|
|
2039
|
+
wreqBrowser: mapping.browser,
|
|
2040
|
+
wreqOs: mapping.os,
|
|
2041
|
+
proxyUrl: proxy,
|
|
2042
|
+
});
|
|
2043
|
+
const challenged = { normalized, response };
|
|
2044
|
+
const solveAndReplay = async (
|
|
2045
|
+
explicitReplay: boolean,
|
|
2046
|
+
): Promise<{ normalized: StealthResponse; response: StealthTransportResponse }> => {
|
|
2047
|
+
if (!akamaiSbsd.solve) {
|
|
2048
|
+
throw new SDKError("No resolver is available for the challenged request", {
|
|
2049
|
+
code: "RESOLVER_UNAVAILABLE",
|
|
2050
|
+
});
|
|
2051
|
+
}
|
|
2052
|
+
// Only a solve that will actually run on this session needs the profile match.
|
|
2053
|
+
assertAkamaiSbsdClientProfile(akamaiSbsd.clientProfile, requestProfile);
|
|
2054
|
+
// A safe request that already solved this exact challenge on this identity
|
|
2055
|
+
// left the session cookies valid: the explicit replay reuses them once
|
|
2056
|
+
// instead of paying for a second solve.
|
|
2057
|
+
const reuseCompletedSuccess =
|
|
2058
|
+
explicitReplay && akamaiSbsdState.completedSuccessKey === transactionKey;
|
|
2059
|
+
if (reuseCompletedSuccess) akamaiSbsdState.completedSuccessKey = undefined;
|
|
2060
|
+
let ownsTransaction = false;
|
|
2061
|
+
if (!reuseCompletedSuccess) {
|
|
2062
|
+
let transaction = akamaiSbsdState.transactions.get(transactionKey);
|
|
2063
|
+
if (!transaction) {
|
|
2064
|
+
akamaiSbsdState.completedSuccessKey = undefined;
|
|
2065
|
+
challengeSolveAttempted = true;
|
|
2066
|
+
ownsTransaction = true;
|
|
2067
|
+
// The solve spans several round trips, so it runs under the client's ambient
|
|
2068
|
+
// signal and the resolver's own timeouts, not this fetch's per-request `timeout`.
|
|
2069
|
+
transaction = {
|
|
2070
|
+
result: akamaiSbsd
|
|
2071
|
+
.solve(
|
|
2072
|
+
detected,
|
|
2073
|
+
resolverTransport,
|
|
2074
|
+
clientOptions.signal ?? new AbortController().signal,
|
|
2075
|
+
)
|
|
2076
|
+
.then(
|
|
2077
|
+
() => ({ solved: true }) as const,
|
|
2078
|
+
(error: unknown) => ({ solved: false, error }) as const,
|
|
2079
|
+
)
|
|
2080
|
+
.finally(() => {
|
|
2081
|
+
akamaiSbsdState.transactions.delete(transactionKey);
|
|
2082
|
+
}),
|
|
2083
|
+
};
|
|
2084
|
+
akamaiSbsdState.transactions.set(transactionKey, transaction);
|
|
2085
|
+
}
|
|
2086
|
+
const transactionResult = await transaction.result;
|
|
2087
|
+
if (!transactionResult.solved) {
|
|
2088
|
+
if (ownsTransaction) {
|
|
2089
|
+
// The proxy delivered the challenged response; the resolver failed.
|
|
2090
|
+
// Surface that failure as-is: it is not a transport fault to normalize
|
|
2091
|
+
// or retry.
|
|
2092
|
+
if (!explicitReplay) {
|
|
2093
|
+
recordProxyAttempt("ok", undefined, challenged.response.status);
|
|
2094
|
+
}
|
|
2095
|
+
challengeSolveFailure = { error: transactionResult.error };
|
|
2096
|
+
throw challengeSolveFailure;
|
|
2097
|
+
}
|
|
2098
|
+
return {
|
|
2099
|
+
normalized: {
|
|
2100
|
+
...challenged.normalized,
|
|
2101
|
+
challenge: { challenge: detected, outcome: "solve_failed" },
|
|
2102
|
+
},
|
|
2103
|
+
response: challenged.response,
|
|
2104
|
+
};
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
challengeRefetchAttempted = true;
|
|
2108
|
+
// An explicit replay is a new transport exchange on the bound endpoint and
|
|
2109
|
+
// gets its own proxy-attempt record timed from here, not from the initiating
|
|
2110
|
+
// fetch (the caller's think time in between is not transport duration); the
|
|
2111
|
+
// automatic refetch stays within the initiating attempt's record.
|
|
2112
|
+
if (explicitReplay) {
|
|
2113
|
+
// The solve above may have taken seconds; if another session of this client
|
|
2114
|
+
// moved the ceremony meanwhile, the mutation must not go out on the retired
|
|
2115
|
+
// endpoint.
|
|
2116
|
+
if (currentEgressGeneration() !== attemptEgressGeneration) {
|
|
2117
|
+
throw new SDKError(
|
|
2118
|
+
"The ceremony's egress binding changed while the replay was being solved",
|
|
2119
|
+
{ code: "REPLAY_SESSION_MISMATCH" },
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2122
|
+
attemptRecorded = false;
|
|
2123
|
+
attemptStartedAt = Date.now();
|
|
2124
|
+
}
|
|
2125
|
+
const replayed = await fetchOnBoundSession(
|
|
2126
|
+
requestUrl,
|
|
2127
|
+
method,
|
|
2128
|
+
replayOptions,
|
|
2129
|
+
clientOptions.signal,
|
|
1587
2130
|
);
|
|
2131
|
+
throwProxyTransportFault(replayed.response, replayed.normalized.body);
|
|
2132
|
+
const persisted = detectAkamaiSbsdChallenge(
|
|
2133
|
+
replayed.normalized,
|
|
2134
|
+
requestUrl,
|
|
2135
|
+
cookieJar,
|
|
2136
|
+
akamaiSbsd.allowedHosts,
|
|
2137
|
+
akamaiSbsdState,
|
|
2138
|
+
);
|
|
2139
|
+
if (persisted) {
|
|
2140
|
+
replayed.normalized.challenge = {
|
|
2141
|
+
challenge: persisted,
|
|
2142
|
+
outcome: "challenge_persisted",
|
|
2143
|
+
};
|
|
2144
|
+
} else if (ownsTransaction && !explicitReplay) {
|
|
2145
|
+
akamaiSbsdState.completedSuccessKey = transactionKey;
|
|
2146
|
+
}
|
|
2147
|
+
return replayed;
|
|
2148
|
+
};
|
|
2149
|
+
|
|
2150
|
+
if (!automaticReplayEligible) {
|
|
2151
|
+
normalized.challenge = {
|
|
2152
|
+
challenge: detected,
|
|
2153
|
+
outcome: "replay_required",
|
|
2154
|
+
};
|
|
2155
|
+
challengedReplays.set(normalized, {
|
|
2156
|
+
consumed: false,
|
|
2157
|
+
generation: attemptEgressGeneration,
|
|
2158
|
+
async replay() {
|
|
2159
|
+
try {
|
|
2160
|
+
const replayed = await solveAndReplay(true);
|
|
2161
|
+
return finalizeAttempt(replayed.response, replayed.normalized);
|
|
2162
|
+
} catch (error) {
|
|
2163
|
+
const normalizedError = normalizeAttemptError(error);
|
|
2164
|
+
releaseFailedCeremonyEgress(proxy, normalizedError);
|
|
2165
|
+
throw recordAttemptFailure(normalizedError);
|
|
1588
2166
|
}
|
|
1589
|
-
|
|
2167
|
+
},
|
|
2168
|
+
});
|
|
2169
|
+
return finalizeAttempt(response, normalized);
|
|
2170
|
+
}
|
|
2171
|
+
({ normalized, response } = await solveAndReplay(false));
|
|
1590
2172
|
}
|
|
2173
|
+
|
|
2174
|
+
return finalizeAttempt(response, normalized);
|
|
2175
|
+
} catch (error) {
|
|
2176
|
+
const normalizedError = normalizeAttemptError(error);
|
|
1591
2177
|
const retryErrorCode = proxyAttemptErrorCode(normalizedError);
|
|
1592
2178
|
const refreshableProxyError = isProxyPoolRefreshableError(normalizedError);
|
|
1593
2179
|
const runProxyAuthDiagnostic = shouldRunProxyAuthDiagnostic(normalizedError);
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
2180
|
+
const redactedError = recordAttemptFailure(normalizedError);
|
|
2181
|
+
lastError = redactedError;
|
|
2182
|
+
const releasedBoundEgress = releaseFailedCeremonyEgress(proxy, normalizedError);
|
|
2183
|
+
if (releasedBoundEgress) {
|
|
2184
|
+
// The request now proceeds as an unbound one with the full rotation budget;
|
|
2185
|
+
// the attempt spent on the dead endpoint does not count against it.
|
|
2186
|
+
maxAttempts = attempt + 1 + unboundMaxAttempts;
|
|
2187
|
+
}
|
|
2188
|
+
if (challengeSolveAttempted || challengeRefetchAttempted) {
|
|
2189
|
+
// The single refetch is spent (owner or waiter of a shared solve): another
|
|
2190
|
+
// transport attempt would replay the original request and may rotate the
|
|
2191
|
+
// proxy the solved jar is bound to.
|
|
2192
|
+
throw redactedError;
|
|
2193
|
+
}
|
|
1606
2194
|
if (proxy && rotatesRegistryChain && refreshableProxyError) {
|
|
1607
|
-
|
|
2195
|
+
// Also the path a released bound endpoint takes after a proxy-level refusal:
|
|
2196
|
+
// the stale-pool bookkeeping here is what drives allocator invalidation and the
|
|
2197
|
+
// refresh pass when the fresh selection lands on the same endpoint again.
|
|
2198
|
+
stalePoolError = redactedError;
|
|
1608
2199
|
if (runProxyAuthDiagnostic) {
|
|
1609
2200
|
stalePoolDiagnosticProxy = proxy;
|
|
1610
2201
|
}
|
|
@@ -1613,6 +2204,17 @@ function createSessionFetcher(
|
|
|
1613
2204
|
}
|
|
1614
2205
|
break;
|
|
1615
2206
|
}
|
|
2207
|
+
if (
|
|
2208
|
+
releasedBoundEgress &&
|
|
2209
|
+
attempt + 1 < maxAttempts &&
|
|
2210
|
+
retryErrorCode === PROXY_CONNECT_FAILURE_CODE
|
|
2211
|
+
) {
|
|
2212
|
+
// The tunnel to the released endpoint never opened, so the origin never saw the
|
|
2213
|
+
// request: selecting and binding a fresh endpoint in this request is safe for
|
|
2214
|
+
// any method. Ambiguous failures (reset, timeout) fall through to the ordinary
|
|
2215
|
+
// method-aware retry decision below with the binding already released.
|
|
2216
|
+
continue;
|
|
2217
|
+
}
|
|
1616
2218
|
// Cap the number of transport retries. For a policy-allocator chain,
|
|
1617
2219
|
// every attempt resolves a *different* endpoint/vendor (poolIndex
|
|
1618
2220
|
// rotates across the concatenated vendor pool spans), so a transport
|
|
@@ -1652,7 +2254,7 @@ function createSessionFetcher(
|
|
|
1652
2254
|
throwIfAmbientAborted(clientOptions.signal);
|
|
1653
2255
|
continue;
|
|
1654
2256
|
}
|
|
1655
|
-
throw
|
|
2257
|
+
throw redactedError;
|
|
1656
2258
|
}
|
|
1657
2259
|
}
|
|
1658
2260
|
|
|
@@ -1703,6 +2305,35 @@ function createSessionFetcher(
|
|
|
1703
2305
|
throwIfAmbientAborted(clientOptions.signal);
|
|
1704
2306
|
throw normalizeStealthTransportError(lastError);
|
|
1705
2307
|
},
|
|
2308
|
+
async replayChallenged(challengedResponse) {
|
|
2309
|
+
// Lease expiry is evaluated when a fetch starts, not here: the replay belongs to the
|
|
2310
|
+
// exchange that was challenged on the bound endpoint (Akamai state is IP-bound), so a
|
|
2311
|
+
// lifetime boundary crossed during the solve does not move it elsewhere. The next
|
|
2312
|
+
// fetch drops the expired binding and rebinds.
|
|
2313
|
+
const replay = challengedReplays.get(challengedResponse);
|
|
2314
|
+
if (!replay || challengedResponse.challenge?.outcome !== "replay_required") {
|
|
2315
|
+
throw new SDKError("The challenged response does not belong to this stealth session", {
|
|
2316
|
+
code: "REPLAY_SESSION_MISMATCH",
|
|
2317
|
+
});
|
|
2318
|
+
}
|
|
2319
|
+
syncCeremonyEgress();
|
|
2320
|
+
if (replay.generation !== currentEgressGeneration()) {
|
|
2321
|
+
// The ceremony rebound to another egress after this request ran (in this session or
|
|
2322
|
+
// another of the same client); solving and replaying on the retired endpoint would
|
|
2323
|
+
// split the identity the handle names.
|
|
2324
|
+
throw new SDKError(
|
|
2325
|
+
"The challenged response predates the ceremony's current egress binding",
|
|
2326
|
+
{ code: "REPLAY_SESSION_MISMATCH" },
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
if (replay.consumed) {
|
|
2330
|
+
throw new SDKError("The challenged request has already used its one replay budget", {
|
|
2331
|
+
code: "REPLAY_ALREADY_ATTEMPTED",
|
|
2332
|
+
});
|
|
2333
|
+
}
|
|
2334
|
+
replay.consumed = true;
|
|
2335
|
+
return replay.replay();
|
|
2336
|
+
},
|
|
1706
2337
|
cookies: cookieJar,
|
|
1707
2338
|
redirects: {
|
|
1708
2339
|
async run(options) {
|
|
@@ -1906,6 +2537,7 @@ function createSessionFetcher(
|
|
|
1906
2537
|
},
|
|
1907
2538
|
close() {
|
|
1908
2539
|
closed = true;
|
|
2540
|
+
clearAkamaiSbsdState();
|
|
1909
2541
|
for (const client of clients.values()) {
|
|
1910
2542
|
void client.session
|
|
1911
2543
|
.then((session) => session.close())
|