@apifuse/provider-sdk 2.2.0-beta.1 → 2.2.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/dist/runtime/http.js +28 -338
- package/dist/runtime/proxy-retry-policy.d.ts +40 -0
- package/dist/runtime/proxy-retry-policy.js +326 -0
- package/dist/runtime/stealth.js +57 -203
- package/dist/server/serve.js +5 -2
- package/dist/server/types.d.ts +1 -0
- package/dist/server/types.js +1 -0
- package/package.json +1 -1
- package/src/runtime/http.ts +60 -547
- package/src/runtime/proxy-retry-policy.ts +469 -0
- package/src/runtime/stealth.ts +100 -361
- package/src/server/serve.ts +8 -2
- package/src/server/types.ts +1 -0
package/src/runtime/stealth.ts
CHANGED
|
@@ -10,19 +10,17 @@ import {
|
|
|
10
10
|
resolveProxyConfigAsync,
|
|
11
11
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
12
12
|
} from "../config/loader";
|
|
13
|
-
import {
|
|
13
|
+
import { SDKError, TransportError } from "../errors";
|
|
14
14
|
import { getStealthProfile } from "../stealth/profiles";
|
|
15
15
|
import type {
|
|
16
16
|
CookieJar,
|
|
17
17
|
HttpMethod,
|
|
18
|
-
HttpRetryOptions,
|
|
19
18
|
StealthClient,
|
|
20
19
|
StealthFetchOptions,
|
|
21
20
|
StealthRedirectHop,
|
|
22
21
|
StealthResponse,
|
|
23
22
|
StealthSession,
|
|
24
23
|
} from "../types";
|
|
25
|
-
import { HttpRetryPreset, HttpRetryUnsafeMethodPolicy } from "../types";
|
|
26
24
|
import {
|
|
27
25
|
createProxyAuthIpDeniedError,
|
|
28
26
|
createProxyEdgeAuthRejectedError,
|
|
@@ -35,10 +33,17 @@ import {
|
|
|
35
33
|
isProxyPoolRefreshableError,
|
|
36
34
|
isProxyPoolStaleMessage,
|
|
37
35
|
isProxyPoolStaleStatus,
|
|
38
|
-
PROXY_AUTH_IP_DENIED_CODE,
|
|
39
36
|
PROXY_EDGE_AUTH_REJECTED_CODE,
|
|
40
37
|
PROXY_POOL_STALE_CODE,
|
|
41
38
|
} from "./proxy-errors";
|
|
39
|
+
import {
|
|
40
|
+
computeProxyAttemptIndex,
|
|
41
|
+
computeProxyTransportRetryDelayMs,
|
|
42
|
+
createDefaultProxyTransportRetryOptions,
|
|
43
|
+
normalizeProxyTransportRetryOptions,
|
|
44
|
+
shouldRetryProxyTransportAttempt,
|
|
45
|
+
validateUnsafeProxyTransportRetryMethods,
|
|
46
|
+
} from "./proxy-retry-policy";
|
|
42
47
|
import { appendQueryParams } from "./request-options";
|
|
43
48
|
|
|
44
49
|
const DEFAULT_PROFILE = "chrome-146";
|
|
@@ -53,31 +58,7 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
53
58
|
/\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
54
59
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
55
60
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
56
|
-
const
|
|
57
|
-
const DEFAULT_STEALTH_RETRY_ERROR_CODES = [
|
|
58
|
-
PROXY_CONNECT_FAILURE_CODE,
|
|
59
|
-
"transport_network_error",
|
|
60
|
-
"transport_timeout",
|
|
61
|
-
] as const;
|
|
62
|
-
const RATE_LIMIT_STEALTH_RETRY_ERROR_CODES = ["transport_timeout"] as const;
|
|
63
|
-
const KNOWN_STEALTH_RETRY_METHODS = new Set([
|
|
64
|
-
"GET",
|
|
65
|
-
"HEAD",
|
|
66
|
-
"POST",
|
|
67
|
-
"PUT",
|
|
68
|
-
"DELETE",
|
|
69
|
-
"OPTIONS",
|
|
70
|
-
"TRACE",
|
|
71
|
-
"PATCH",
|
|
72
|
-
]);
|
|
73
|
-
const UNSAFE_STEALTH_RETRY_METHODS = new Set([
|
|
74
|
-
"POST",
|
|
75
|
-
"PUT",
|
|
76
|
-
"PATCH",
|
|
77
|
-
"DELETE",
|
|
78
|
-
"TRACE",
|
|
79
|
-
]);
|
|
80
|
-
const MAX_STEALTH_RETRY_ATTEMPTS = 8;
|
|
61
|
+
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
81
62
|
|
|
82
63
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
83
64
|
warn?: (message: string) => void;
|
|
@@ -139,12 +120,6 @@ type StealthMethod = NonNullable<ImpitRequestInit["method"]>;
|
|
|
139
120
|
type StealthRequestInit = ImpitRequestInit & {
|
|
140
121
|
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
141
122
|
};
|
|
142
|
-
type NormalizedStealthRetryOptions = {
|
|
143
|
-
attempts: number;
|
|
144
|
-
methods: readonly string[];
|
|
145
|
-
errorCodes: readonly string[];
|
|
146
|
-
unsafeMethodPolicy: HttpRetryOptions["unsafeMethodPolicy"];
|
|
147
|
-
};
|
|
148
123
|
|
|
149
124
|
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
150
125
|
return typeof value === "object" && value !== null;
|
|
@@ -303,9 +278,7 @@ function normalizeHeaders(
|
|
|
303
278
|
function hasOwn(object: object, key: string): boolean {
|
|
304
279
|
return Object.hasOwn(object, key);
|
|
305
280
|
}
|
|
306
|
-
function toImpitCookieJar(
|
|
307
|
-
cookieJar: CookieJarImpl,
|
|
308
|
-
): NonNullable<ImpitOptions["cookieJar"]> {
|
|
281
|
+
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
309
282
|
return {
|
|
310
283
|
setCookie(cookie: string, _url: string, cb?: (error?: unknown) => void) {
|
|
311
284
|
cookieJar.setFromCookieStrings([cookie]);
|
|
@@ -322,10 +295,8 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
322
295
|
const unsupported: string[] = [];
|
|
323
296
|
if (hasOwn(options, "headerOrder")) unsupported.push("headerOrder");
|
|
324
297
|
const stealth = options.stealth;
|
|
325
|
-
if (isRecord(stealth) && hasOwn(stealth, "ja3"))
|
|
326
|
-
|
|
327
|
-
if (isRecord(stealth) && hasOwn(stealth, "h2"))
|
|
328
|
-
unsupported.push("stealth.h2");
|
|
298
|
+
if (isRecord(stealth) && hasOwn(stealth, "ja3")) unsupported.push("stealth.ja3");
|
|
299
|
+
if (isRecord(stealth) && hasOwn(stealth, "h2")) unsupported.push("stealth.h2");
|
|
329
300
|
if (unsupported.length === 0) return;
|
|
330
301
|
|
|
331
302
|
throw new SDKError(
|
|
@@ -333,9 +304,7 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
333
304
|
);
|
|
334
305
|
}
|
|
335
306
|
|
|
336
|
-
function responseHeadersToRecord(
|
|
337
|
-
headers: Headers,
|
|
338
|
-
): Record<string, string | string[] | undefined> {
|
|
307
|
+
function responseHeadersToRecord(headers: Headers): Record<string, string | string[] | undefined> {
|
|
339
308
|
const record: Record<string, string> = {};
|
|
340
309
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
341
310
|
return record;
|
|
@@ -370,9 +339,7 @@ export async function normalizeResponse(
|
|
|
370
339
|
requestUrl?: string,
|
|
371
340
|
): Promise<StealthResponse> {
|
|
372
341
|
const headers = Object.fromEntries(response.headers.entries());
|
|
373
|
-
const cookies = new CookieJarImpl(
|
|
374
|
-
setCookieHeadersFromResponse(response.headers),
|
|
375
|
-
);
|
|
342
|
+
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers));
|
|
376
343
|
const bodyBytes = await response.arrayBuffer();
|
|
377
344
|
const body = new TextDecoder().decode(bodyBytes);
|
|
378
345
|
|
|
@@ -422,189 +389,11 @@ function isPolicyManagedProxy(options: StealthClientOptions): boolean {
|
|
|
422
389
|
return Boolean(policy && typeof policy === "object");
|
|
423
390
|
}
|
|
424
391
|
|
|
425
|
-
function
|
|
426
|
-
return
|
|
392
|
+
function isProxyConnectFailureResponse(response: StealthTransportResponse, body: string): boolean {
|
|
393
|
+
return response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "");
|
|
427
394
|
}
|
|
428
395
|
|
|
429
|
-
function
|
|
430
|
-
preset: HttpRetryPreset,
|
|
431
|
-
): NormalizedStealthRetryOptions {
|
|
432
|
-
switch (preset) {
|
|
433
|
-
case HttpRetryPreset.Off:
|
|
434
|
-
return {
|
|
435
|
-
attempts: 1,
|
|
436
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
437
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
438
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
439
|
-
};
|
|
440
|
-
case HttpRetryPreset.AggressiveRead:
|
|
441
|
-
return {
|
|
442
|
-
attempts: 4,
|
|
443
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
444
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
445
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
446
|
-
};
|
|
447
|
-
case HttpRetryPreset.RateLimitAware:
|
|
448
|
-
return {
|
|
449
|
-
attempts: 3,
|
|
450
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
451
|
-
errorCodes: RATE_LIMIT_STEALTH_RETRY_ERROR_CODES,
|
|
452
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
453
|
-
};
|
|
454
|
-
case HttpRetryPreset.SafeRead:
|
|
455
|
-
case HttpRetryPreset.TransportTransient:
|
|
456
|
-
return {
|
|
457
|
-
attempts: 3,
|
|
458
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
459
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
460
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
461
|
-
};
|
|
462
|
-
}
|
|
463
|
-
throw new ProviderError(`Unknown stealth retry preset: ${preset}`, {
|
|
464
|
-
code: "retry_invalid_policy",
|
|
465
|
-
});
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
function normalizeStealthRetryOptions(
|
|
469
|
-
retry: StealthFetchOptions["retry"],
|
|
470
|
-
): NormalizedStealthRetryOptions | undefined {
|
|
471
|
-
if (retry === undefined) return undefined;
|
|
472
|
-
if (retry === false) return createStealthRetryOptions(HttpRetryPreset.Off);
|
|
473
|
-
if (retry === true)
|
|
474
|
-
return createStealthRetryOptions(HttpRetryPreset.TransportTransient);
|
|
475
|
-
if (typeof retry === "string") {
|
|
476
|
-
if (!Object.values(HttpRetryPreset).includes(retry)) {
|
|
477
|
-
throw new ProviderError(`Unknown stealth retry preset: ${retry}`, {
|
|
478
|
-
code: "retry_invalid_policy",
|
|
479
|
-
});
|
|
480
|
-
}
|
|
481
|
-
return createStealthRetryOptions(retry);
|
|
482
|
-
}
|
|
483
|
-
if (typeof retry !== "object" || retry === null || Array.isArray(retry)) {
|
|
484
|
-
throw new ProviderError("Stealth retry policy must be a plain object", {
|
|
485
|
-
code: "retry_invalid_policy",
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
if (
|
|
489
|
-
retry.unsafeMethodPolicy !== undefined &&
|
|
490
|
-
!Object.values(HttpRetryUnsafeMethodPolicy).includes(
|
|
491
|
-
retry.unsafeMethodPolicy,
|
|
492
|
-
)
|
|
493
|
-
) {
|
|
494
|
-
throw new ProviderError(
|
|
495
|
-
`Unknown stealth retry unsafe method policy: ${String(retry.unsafeMethodPolicy)}`,
|
|
496
|
-
{ code: "retry_invalid_policy" },
|
|
497
|
-
);
|
|
498
|
-
}
|
|
499
|
-
if (retry.methods !== undefined) {
|
|
500
|
-
if (!Array.isArray(retry.methods)) {
|
|
501
|
-
throw new ProviderError("Stealth retry methods must be an array", {
|
|
502
|
-
code: "retry_invalid_policy",
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
const unknownMethods = retry.methods
|
|
506
|
-
.map((method) => (typeof method === "string" ? method.toUpperCase() : ""))
|
|
507
|
-
.filter((method) => !KNOWN_STEALTH_RETRY_METHODS.has(method));
|
|
508
|
-
if (unknownMethods.length > 0) {
|
|
509
|
-
throw new ProviderError(
|
|
510
|
-
`Unknown stealth retry method(s): ${unknownMethods.join(", ")}`,
|
|
511
|
-
{ code: "retry_invalid_policy" },
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
if (retry.errorCodes !== undefined) {
|
|
516
|
-
if (
|
|
517
|
-
!Array.isArray(retry.errorCodes) ||
|
|
518
|
-
retry.errorCodes.some((errorCode) => typeof errorCode !== "string")
|
|
519
|
-
) {
|
|
520
|
-
throw new ProviderError(
|
|
521
|
-
"Stealth retry errorCodes must contain only strings",
|
|
522
|
-
{ code: "retry_invalid_policy" },
|
|
523
|
-
);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
const base = createStealthRetryOptions(
|
|
528
|
-
retry.preset ?? HttpRetryPreset.TransportTransient,
|
|
529
|
-
);
|
|
530
|
-
const attempts =
|
|
531
|
-
retry.attempts === undefined || !Number.isFinite(retry.attempts)
|
|
532
|
-
? base.attempts
|
|
533
|
-
: Math.max(
|
|
534
|
-
1,
|
|
535
|
-
Math.min(MAX_STEALTH_RETRY_ATTEMPTS, Math.floor(retry.attempts)),
|
|
536
|
-
);
|
|
537
|
-
const normalized: NormalizedStealthRetryOptions = {
|
|
538
|
-
attempts,
|
|
539
|
-
methods:
|
|
540
|
-
retry.methods?.map((method) => method.toUpperCase()) ?? base.methods,
|
|
541
|
-
errorCodes: retry.errorCodes ?? base.errorCodes,
|
|
542
|
-
unsafeMethodPolicy: retry.unsafeMethodPolicy ?? base.unsafeMethodPolicy,
|
|
543
|
-
};
|
|
544
|
-
|
|
545
|
-
if (
|
|
546
|
-
normalized.unsafeMethodPolicy !==
|
|
547
|
-
HttpRetryUnsafeMethodPolicy.AllowExplicitUnsafe
|
|
548
|
-
) {
|
|
549
|
-
const unsafeMethods = normalized.methods.filter((method) =>
|
|
550
|
-
UNSAFE_STEALTH_RETRY_METHODS.has(method.toUpperCase()),
|
|
551
|
-
);
|
|
552
|
-
if (unsafeMethods.length > 0) {
|
|
553
|
-
throw new ProviderError(
|
|
554
|
-
`Stealth retry methods include unsafe method(s): ${unsafeMethods.join(", ")}`,
|
|
555
|
-
{ code: "retry_unsafe_method" },
|
|
556
|
-
);
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
return normalized;
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
function isExplicitStealthRetryAllowed(
|
|
564
|
-
method: StealthMethod,
|
|
565
|
-
error: TransportError,
|
|
566
|
-
retryOptions: NormalizedStealthRetryOptions | undefined,
|
|
567
|
-
): boolean {
|
|
568
|
-
if (!retryOptions || retryOptions.attempts <= 1) return false;
|
|
569
|
-
return (
|
|
570
|
-
retryOptions.methods.includes(method.toUpperCase()) &&
|
|
571
|
-
retryOptions.errorCodes.includes(proxyAttemptErrorCode(error))
|
|
572
|
-
);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function isRetryableProxyTransportError(error: unknown): boolean {
|
|
576
|
-
if (error instanceof TransportError) {
|
|
577
|
-
if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
|
|
578
|
-
return false;
|
|
579
|
-
}
|
|
580
|
-
return (
|
|
581
|
-
error.code === PROXY_CONNECT_FAILURE_CODE ||
|
|
582
|
-
error.code === "transport_network_error" ||
|
|
583
|
-
error.code === "transport_timeout"
|
|
584
|
-
);
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
if (error instanceof SDKError) {
|
|
588
|
-
return false;
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
592
|
-
return /\bproxy\b|\bnon[\s-]?200\b|\bconnect\b|\btunnel\b/i.test(message);
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
function isProxyConnectFailureResponse(
|
|
596
|
-
response: StealthTransportResponse,
|
|
597
|
-
body: string,
|
|
598
|
-
): boolean {
|
|
599
|
-
return (
|
|
600
|
-
response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "")
|
|
601
|
-
);
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
function createProxyConnectFailureError(
|
|
605
|
-
body: string,
|
|
606
|
-
cause?: Error,
|
|
607
|
-
): TransportError {
|
|
396
|
+
function createProxyConnectFailureError(body: string, cause?: Error): TransportError {
|
|
608
397
|
const bodyExcerpt = (body ?? "").trim().slice(0, 1_000);
|
|
609
398
|
if (isProxyAuthIpDeniedMessage(bodyExcerpt)) {
|
|
610
399
|
return createProxyAuthIpDeniedError(cause);
|
|
@@ -613,10 +402,7 @@ function createProxyConnectFailureError(
|
|
|
613
402
|
return createProxyEdgeAuthRejectedError(cause);
|
|
614
403
|
}
|
|
615
404
|
if (isProxyPoolStaleMessage(bodyExcerpt)) {
|
|
616
|
-
return createProxyPoolStaleError(
|
|
617
|
-
bodyExcerpt.includes("512") ? 512 : 509,
|
|
618
|
-
cause,
|
|
619
|
-
);
|
|
405
|
+
return createProxyPoolStaleError(bodyExcerpt.includes("512") ? 512 : 509, cause);
|
|
620
406
|
}
|
|
621
407
|
return new TransportError(bodyExcerpt || "Proxy CONNECT failed", {
|
|
622
408
|
code: PROXY_CONNECT_FAILURE_CODE,
|
|
@@ -723,33 +509,20 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
723
509
|
}
|
|
724
510
|
|
|
725
511
|
if (isProxyAuthIpDeniedMessage(message)) {
|
|
726
|
-
return createProxyAuthIpDeniedError(
|
|
727
|
-
error instanceof Error ? error : undefined,
|
|
728
|
-
);
|
|
512
|
+
return createProxyAuthIpDeniedError(error instanceof Error ? error : undefined);
|
|
729
513
|
}
|
|
730
514
|
|
|
731
515
|
if (isProxyEdgeAuthRejectedMessage(message)) {
|
|
732
|
-
return createProxyEdgeAuthRejectedError(
|
|
733
|
-
error instanceof Error ? error : undefined,
|
|
734
|
-
);
|
|
516
|
+
return createProxyEdgeAuthRejectedError(error instanceof Error ? error : undefined);
|
|
735
517
|
}
|
|
736
518
|
|
|
737
519
|
const proxyTunnelStatus = getProxyTunnelStatus(error);
|
|
738
|
-
if (
|
|
739
|
-
proxyTunnelStatus
|
|
740
|
-
isProxyPoolStaleStatus(proxyTunnelStatus)
|
|
741
|
-
) {
|
|
742
|
-
return createProxyPoolStaleError(
|
|
743
|
-
proxyTunnelStatus,
|
|
744
|
-
error instanceof Error ? error : undefined,
|
|
745
|
-
);
|
|
520
|
+
if (proxyTunnelStatus !== undefined && isProxyPoolStaleStatus(proxyTunnelStatus)) {
|
|
521
|
+
return createProxyPoolStaleError(proxyTunnelStatus, error instanceof Error ? error : undefined);
|
|
746
522
|
}
|
|
747
523
|
|
|
748
524
|
if (PROXY_CONNECT_FAILURE_BODY_PATTERN.test(message)) {
|
|
749
|
-
return createProxyConnectFailureError(
|
|
750
|
-
message,
|
|
751
|
-
error instanceof Error ? error : undefined,
|
|
752
|
-
);
|
|
525
|
+
return createProxyConnectFailureError(message, error instanceof Error ? error : undefined);
|
|
753
526
|
}
|
|
754
527
|
|
|
755
528
|
return new TransportError("Network error", {
|
|
@@ -759,6 +532,10 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
759
532
|
});
|
|
760
533
|
}
|
|
761
534
|
|
|
535
|
+
function sleep(ms: number): Promise<void> {
|
|
536
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
537
|
+
}
|
|
538
|
+
|
|
762
539
|
function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
763
540
|
switch (method.toUpperCase()) {
|
|
764
541
|
case "HEAD":
|
|
@@ -786,10 +563,7 @@ function isRedirectStatus(status: number): boolean {
|
|
|
786
563
|
return [301, 302, 303, 307, 308].includes(status);
|
|
787
564
|
}
|
|
788
565
|
|
|
789
|
-
function nextRedirectMethod(
|
|
790
|
-
status: number,
|
|
791
|
-
method: StealthMethod,
|
|
792
|
-
): StealthMethod {
|
|
566
|
+
function nextRedirectMethod(status: number, method: StealthMethod): StealthMethod {
|
|
793
567
|
if (status === 303 && method !== "HEAD") return "GET";
|
|
794
568
|
if ((status === 301 || status === 302) && method === "POST") return "GET";
|
|
795
569
|
return method;
|
|
@@ -842,19 +616,16 @@ function createSessionFetcher(
|
|
|
842
616
|
options?: StealthFetchOptions,
|
|
843
617
|
proxyAttempt?: number,
|
|
844
618
|
): Promise<ResolvedAttemptProxy> {
|
|
845
|
-
const rawProxyAttemptOffset = options?.proxyAttemptOffset ?? 0;
|
|
846
|
-
const proxyAttemptOffset = Number.isFinite(rawProxyAttemptOffset)
|
|
847
|
-
? Math.max(0, Math.floor(rawProxyAttemptOffset))
|
|
848
|
-
: 0;
|
|
849
619
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
850
620
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
851
621
|
upstream: clientOptions.upstream,
|
|
852
622
|
apifuseConfig: clientOptions.apifuseConfig,
|
|
853
623
|
affinityKey: clientOptions.affinityKey,
|
|
854
|
-
proxyAttempt:
|
|
855
|
-
proxyAttempt
|
|
856
|
-
|
|
857
|
-
|
|
624
|
+
proxyAttempt: computeProxyAttemptIndex({
|
|
625
|
+
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
626
|
+
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
|
+
retryAttemptOffset: proxyAttempt,
|
|
628
|
+
}),
|
|
858
629
|
telemetry: clientOptions.telemetry,
|
|
859
630
|
});
|
|
860
631
|
|
|
@@ -874,23 +645,35 @@ function createSessionFetcher(
|
|
|
874
645
|
async fetch(url, options: StealthFetchOptions = {}) {
|
|
875
646
|
const method = normalizeMethod(options.method ?? "GET");
|
|
876
647
|
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
877
|
-
const stealthRetryOptions =
|
|
648
|
+
const stealthRetryOptions =
|
|
649
|
+
normalizeProxyTransportRetryOptions(options.retry, {
|
|
650
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
651
|
+
label: "Stealth",
|
|
652
|
+
}) ??
|
|
653
|
+
(hasExplicitRetryPolicy
|
|
654
|
+
? undefined
|
|
655
|
+
: createDefaultProxyTransportRetryOptions({
|
|
656
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
657
|
+
label: "Stealth",
|
|
658
|
+
}));
|
|
659
|
+
if (stealthRetryOptions) {
|
|
660
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
661
|
+
}
|
|
878
662
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
879
|
-
const usesPolicyAllocator =
|
|
880
|
-
|
|
881
|
-
const
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
: 1;
|
|
663
|
+
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
|
+
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
665
|
+
const policyProxyAttemptCap = Math.max(
|
|
666
|
+
1,
|
|
667
|
+
Math.min(
|
|
668
|
+
MAX_POLICY_PROXY_RETRY_ATTEMPTS,
|
|
669
|
+
clientOptions.proxyPolicy?.session?.poolSize ??
|
|
670
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
+
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
+
: undefined) ??
|
|
673
|
+
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
|
+
),
|
|
675
|
+
);
|
|
676
|
+
const maxAttempts = usesPolicyAllocator ? policyProxyAttemptCap : retryAttemptCap;
|
|
894
677
|
let lastError: unknown;
|
|
895
678
|
|
|
896
679
|
for (
|
|
@@ -920,9 +703,7 @@ function createSessionFetcher(
|
|
|
920
703
|
...(attemptProxy?.poolIndex === undefined
|
|
921
704
|
? {}
|
|
922
705
|
: { poolIndex: attemptProxy.poolIndex }),
|
|
923
|
-
...(attemptProxy?.proxyHash
|
|
924
|
-
? { proxyHash: attemptProxy.proxyHash }
|
|
925
|
-
: {}),
|
|
706
|
+
...(attemptProxy?.proxyHash ? { proxyHash: attemptProxy.proxyHash } : {}),
|
|
926
707
|
outcome,
|
|
927
708
|
...(errorCode ? { errorCode } : {}),
|
|
928
709
|
...(status === undefined ? {} : { status }),
|
|
@@ -933,7 +714,7 @@ function createSessionFetcher(
|
|
|
933
714
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
934
715
|
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
935
716
|
proxy = attemptProxy.url;
|
|
936
|
-
if (proxy) {
|
|
717
|
+
if (proxy && usesPolicyAllocator) {
|
|
937
718
|
if (attemptedProxies.has(proxy)) {
|
|
938
719
|
break;
|
|
939
720
|
}
|
|
@@ -941,15 +722,10 @@ function createSessionFetcher(
|
|
|
941
722
|
}
|
|
942
723
|
const ignoreTlsErrors = Boolean(
|
|
943
724
|
options.stealth?.insecureSkipVerify ??
|
|
944
|
-
(!hasPolicyProxy &&
|
|
945
|
-
proxy &&
|
|
946
|
-
clientOptions.proxyStealth?.insecureSkipVerify),
|
|
725
|
+
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
947
726
|
);
|
|
948
727
|
const profileName = options.profile ?? defaultProfile;
|
|
949
|
-
const requestUrl = appendQueryParams(
|
|
950
|
-
resolveUrl(baseUrl, url),
|
|
951
|
-
options.params,
|
|
952
|
-
);
|
|
728
|
+
const requestUrl = appendQueryParams(resolveUrl(baseUrl, url), options.params);
|
|
953
729
|
const headers = { ...(options.headers ?? {}) };
|
|
954
730
|
if (!hasHeader(headers, "Cookie")) {
|
|
955
731
|
const cookieHeader = cookieJar.toString();
|
|
@@ -964,20 +740,14 @@ function createSessionFetcher(
|
|
|
964
740
|
if (options.body !== undefined) {
|
|
965
741
|
requestInit.body = normalizeBody(options.body);
|
|
966
742
|
}
|
|
967
|
-
const response = await getClient(
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
ignoreTlsErrors,
|
|
971
|
-
).fetch(requestUrl, requestInit);
|
|
972
|
-
const normalized = await normalizeResponse(response, requestUrl);
|
|
973
|
-
cookieJar.setFromCookieStrings(
|
|
974
|
-
setCookieHeadersFromResponse(response.headers),
|
|
743
|
+
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(
|
|
744
|
+
requestUrl,
|
|
745
|
+
requestInit,
|
|
975
746
|
);
|
|
747
|
+
const normalized = await normalizeResponse(response, requestUrl);
|
|
748
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
976
749
|
|
|
977
|
-
if (
|
|
978
|
-
proxy &&
|
|
979
|
-
isProxyConnectFailureResponse(response, normalized.body)
|
|
980
|
-
) {
|
|
750
|
+
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
981
751
|
throw createProxyConnectFailureError(normalized.body);
|
|
982
752
|
}
|
|
983
753
|
|
|
@@ -987,10 +757,9 @@ function createSessionFetcher(
|
|
|
987
757
|
usesPolicyAllocator &&
|
|
988
758
|
isProxyEdgeTlsRejectedResponse(
|
|
989
759
|
response.status,
|
|
990
|
-
[
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
].join("\n"),
|
|
760
|
+
[JSON.stringify(responseHeadersToRecord(response.headers)), normalized.body].join(
|
|
761
|
+
"\n",
|
|
762
|
+
),
|
|
994
763
|
)
|
|
995
764
|
) {
|
|
996
765
|
throw createProxyEdgeTlsRejectedError(response.status);
|
|
@@ -1011,13 +780,10 @@ function createSessionFetcher(
|
|
|
1011
780
|
}
|
|
1012
781
|
|
|
1013
782
|
if (response.status >= 400 && options.throwOnHttpError !== false) {
|
|
1014
|
-
throw new TransportError(
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
status: response.status,
|
|
1019
|
-
},
|
|
1020
|
-
);
|
|
783
|
+
throw new TransportError(`Upstream request failed with status ${response.status}`, {
|
|
784
|
+
code: "upstream_http_error",
|
|
785
|
+
status: response.status,
|
|
786
|
+
});
|
|
1021
787
|
}
|
|
1022
788
|
|
|
1023
789
|
recordProxyAttempt("ok", undefined, response.status);
|
|
@@ -1030,11 +796,7 @@ function createSessionFetcher(
|
|
|
1030
796
|
proxyAttemptStatus(normalizedError),
|
|
1031
797
|
);
|
|
1032
798
|
lastError = normalizedError;
|
|
1033
|
-
if (
|
|
1034
|
-
proxy &&
|
|
1035
|
-
usesPolicyAllocator &&
|
|
1036
|
-
isProxyPoolRefreshableError(normalizedError)
|
|
1037
|
-
) {
|
|
799
|
+
if (proxy && usesPolicyAllocator && isProxyPoolRefreshableError(normalizedError)) {
|
|
1038
800
|
stalePoolError = normalizedError;
|
|
1039
801
|
if (shouldRunProxyAuthDiagnostic(normalizedError)) {
|
|
1040
802
|
stalePoolDiagnosticProxy = proxy;
|
|
@@ -1045,20 +807,21 @@ function createSessionFetcher(
|
|
|
1045
807
|
break;
|
|
1046
808
|
}
|
|
1047
809
|
if (
|
|
1048
|
-
proxy &&
|
|
1049
810
|
attempt + 1 <
|
|
1050
811
|
(stealthRetryOptions
|
|
1051
812
|
? Math.min(maxAttempts, stealthRetryOptions.attempts)
|
|
1052
813
|
: maxAttempts) &&
|
|
1053
|
-
(
|
|
1054
|
-
|
|
1055
|
-
:
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
isRetryableProxyTransportError(normalizedError)
|
|
814
|
+
shouldRetryProxyTransportAttempt({
|
|
815
|
+
error: normalizedError,
|
|
816
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
817
|
+
method,
|
|
818
|
+
options: stealthRetryOptions,
|
|
819
|
+
proxyUsed: Boolean(proxy),
|
|
820
|
+
})
|
|
1061
821
|
) {
|
|
822
|
+
if (stealthRetryOptions) {
|
|
823
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions!, attempt + 1));
|
|
824
|
+
}
|
|
1062
825
|
continue;
|
|
1063
826
|
}
|
|
1064
827
|
throw normalizedError;
|
|
@@ -1126,13 +889,7 @@ function createSessionFetcher(
|
|
|
1126
889
|
let response: StealthResponse | undefined;
|
|
1127
890
|
const visitedRequests = new Set<string>();
|
|
1128
891
|
|
|
1129
|
-
const {
|
|
1130
|
-
url: _url,
|
|
1131
|
-
maxHops: _maxHops,
|
|
1132
|
-
stopWhen,
|
|
1133
|
-
params,
|
|
1134
|
-
...fetchOptions
|
|
1135
|
-
} = options;
|
|
892
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, ...fetchOptions } = options;
|
|
1136
893
|
|
|
1137
894
|
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
1138
895
|
visitedRequests.add(`${method} ${currentUrl}`);
|
|
@@ -1240,22 +997,16 @@ function createSessionFetcher(
|
|
|
1240
997
|
proxy: string,
|
|
1241
998
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
1242
999
|
try {
|
|
1243
|
-
const response = await getClient(profileName, proxy, false).fetch(
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1248
|
-
},
|
|
1249
|
-
);
|
|
1000
|
+
const response = await getClient(profileName, proxy, false).fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1001
|
+
method: "GET",
|
|
1002
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1003
|
+
});
|
|
1250
1004
|
const normalized = await normalizeResponse(response);
|
|
1251
1005
|
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1252
1006
|
} catch (error) {
|
|
1253
1007
|
const message =
|
|
1254
1008
|
error instanceof Error
|
|
1255
|
-
? [
|
|
1256
|
-
error.message,
|
|
1257
|
-
error.cause instanceof Error ? error.cause.message : "",
|
|
1258
|
-
]
|
|
1009
|
+
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
1259
1010
|
.filter(Boolean)
|
|
1260
1011
|
.join(" ")
|
|
1261
1012
|
: String(error);
|
|
@@ -1295,22 +1046,14 @@ export function createStealthClient(
|
|
|
1295
1046
|
clientOptions: StealthClientOptions = {},
|
|
1296
1047
|
): StealthClient {
|
|
1297
1048
|
const defaultProfile =
|
|
1298
|
-
typeof defaultProfileOrOptions === "string"
|
|
1299
|
-
? defaultProfileOrOptions
|
|
1300
|
-
: DEFAULT_PROFILE;
|
|
1049
|
+
typeof defaultProfileOrOptions === "string" ? defaultProfileOrOptions : DEFAULT_PROFILE;
|
|
1301
1050
|
const resolvedClientOptions =
|
|
1302
|
-
typeof defaultProfileOrOptions === "string"
|
|
1303
|
-
? clientOptions
|
|
1304
|
-
: defaultProfileOrOptions;
|
|
1051
|
+
typeof defaultProfileOrOptions === "string" ? clientOptions : defaultProfileOrOptions;
|
|
1305
1052
|
let sharedSession: StealthSession | null = null;
|
|
1306
1053
|
|
|
1307
1054
|
function getSharedSession(): StealthSession {
|
|
1308
1055
|
if (!sharedSession) {
|
|
1309
|
-
sharedSession = createSessionFetcher(
|
|
1310
|
-
baseUrl,
|
|
1311
|
-
defaultProfile,
|
|
1312
|
-
resolvedClientOptions,
|
|
1313
|
-
);
|
|
1056
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, resolvedClientOptions);
|
|
1314
1057
|
}
|
|
1315
1058
|
|
|
1316
1059
|
return sharedSession;
|
|
@@ -1322,11 +1065,7 @@ export function createStealthClient(
|
|
|
1322
1065
|
},
|
|
1323
1066
|
createSession(opts?: { profile?: string }) {
|
|
1324
1067
|
const sessionProfile = opts?.profile ?? defaultProfile;
|
|
1325
|
-
return createSessionFetcher(
|
|
1326
|
-
baseUrl,
|
|
1327
|
-
sessionProfile,
|
|
1328
|
-
resolvedClientOptions,
|
|
1329
|
-
);
|
|
1068
|
+
return createSessionFetcher(baseUrl, sessionProfile, resolvedClientOptions);
|
|
1330
1069
|
},
|
|
1331
1070
|
close() {
|
|
1332
1071
|
sharedSession?.close();
|