@apifuse/provider-sdk 2.1.0-beta.9 → 2.2.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +134 -0
- package/CHANGELOG.md +61 -0
- package/README.md +21 -9
- package/SUBMISSION.md +10 -11
- package/bin/apifuse-pack-check.ts +22 -0
- package/bin/apifuse-submit-check.ts +1747 -388
- package/dist/auth-turn/auth-turn.v1.schema.json +89 -0
- package/dist/auth-turn/fixtures/invalid/empty-kind.json +4 -0
- package/dist/auth-turn/fixtures/invalid/expires-at-not-string.json +5 -0
- package/dist/auth-turn/fixtures/invalid/missing-kind.json +3 -0
- package/dist/auth-turn/fixtures/invalid/missing-turn-id.json +3 -0
- package/dist/auth-turn/fixtures/invalid/timing-unknown-field.json +7 -0
- package/dist/auth-turn/fixtures/invalid/turn-id-snake-case.json +4 -0
- package/dist/auth-turn/fixtures/invalid/unknown-top-level-field.json +5 -0
- package/dist/auth-turn/fixtures/valid/abort.json +8 -0
- package/dist/auth-turn/fixtures/valid/challenge.json +17 -0
- package/dist/auth-turn/fixtures/valid/complete.json +13 -0
- package/dist/auth-turn/fixtures/valid/form.json +14 -0
- package/dist/auth-turn/fixtures/valid/message.json +13 -0
- package/dist/auth-turn/fixtures/valid/multi_choice.json +15 -0
- package/dist/auth-turn/fixtures/valid/pending.json +5 -0
- package/dist/auth-turn/fixtures/valid/poll.json +9 -0
- package/dist/auth-turn/fixtures/valid/redirect.json +16 -0
- package/dist/auth-turn/fixtures/valid/retry.json +8 -0
- package/dist/auth-turn/fixtures/valid/unknown-kind.json +7 -0
- package/dist/auth-turn/index.d.ts +195 -0
- package/dist/auth-turn/index.js +133 -0
- package/dist/auth.d.ts +76 -0
- package/dist/auth.js +436 -0
- package/dist/ceremonies/index.js +7 -31
- package/dist/cli/create.js +45 -30
- package/dist/cli/templates/provider/.dockerignore.tpl +22 -0
- package/dist/cli/templates/provider/.gitignore.tpl +22 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +87 -0
- package/dist/cli/templates/provider/CLAUDE.md.tpl +1 -0
- package/dist/cli/templates/provider/Dockerfile.tpl +7 -0
- package/dist/cli/templates/provider/README.md.tpl +163 -0
- package/dist/cli/templates/provider/dev.ts.tpl +5 -0
- package/dist/cli/templates/provider/domain/README.md.tpl +3 -0
- package/dist/cli/templates/provider/index.test.ts.tpl +13 -0
- package/dist/cli/templates/provider/index.ts.tpl +15 -0
- package/dist/cli/templates/provider/mappers/README.md.tpl +3 -0
- package/dist/cli/templates/provider/meta.ts.tpl +7 -0
- package/dist/cli/templates/provider/operations/index.ts.tpl +5 -0
- package/dist/cli/templates/provider/operations/ping.ts.tpl +24 -0
- package/dist/cli/templates/provider/schemas/ping.ts.tpl +24 -0
- package/dist/cli/templates/provider/skills/fixtures-and-recording/SKILL.md.tpl +58 -0
- package/dist/cli/templates/provider/skills/health-checks-and-fail-closed/SKILL.md.tpl +65 -0
- package/dist/cli/templates/provider/skills/normalization-standards/SKILL.md.tpl +57 -0
- package/dist/cli/templates/provider/skills/pagination-and-counts/SKILL.md.tpl +52 -0
- package/dist/cli/templates/provider/skills/upstream-contract-verification/SKILL.md.tpl +45 -0
- package/dist/cli/templates/provider/skills/upstream-notes/README.md.tpl +13 -0
- package/dist/cli/templates/provider/start.ts.tpl +5 -0
- package/dist/cli/templates/provider/upstream/README.md.tpl +3 -0
- package/dist/contract.js +1 -0
- package/dist/define.d.ts +6 -1
- package/dist/define.js +140 -70
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/lint.d.ts +1 -0
- package/dist/lint.js +27 -0
- package/dist/provider.d.ts +4 -2
- package/dist/provider.js +2 -1
- package/dist/runtime/auth-flow.js +2 -0
- package/dist/runtime/browser.js +203 -0
- package/dist/runtime/http.js +46 -336
- package/dist/runtime/proxy-retry-policy.d.ts +40 -0
- package/dist/runtime/proxy-retry-policy.js +326 -0
- package/dist/runtime/stealth.d.ts +5 -2
- package/dist/runtime/stealth.js +206 -206
- package/dist/server/index.d.ts +4 -0
- package/dist/server/index.js +4 -0
- package/dist/server/self-test-input-tokens.d.ts +1 -0
- package/dist/server/self-test-input-tokens.js +37 -0
- package/dist/server/self-test-redaction.d.ts +20 -0
- package/dist/server/self-test-redaction.js +70 -0
- package/dist/server/self-test-token.d.ts +30 -0
- package/dist/server/self-test-token.js +50 -0
- package/dist/server/self-test.d.ts +98 -0
- package/dist/server/self-test.js +555 -0
- package/dist/server/serve.d.ts +6 -0
- package/dist/server/serve.js +38 -10
- package/dist/server/types.d.ts +1 -0
- package/dist/server/types.js +1 -0
- package/dist/testing/run.js +5 -1
- package/dist/types.d.ts +150 -0
- package/package.json +9 -2
- package/src/auth-turn/auth-turn.v1.schema.json +89 -0
- package/src/auth-turn/fixtures/invalid/empty-kind.json +4 -0
- package/src/auth-turn/fixtures/invalid/expires-at-not-string.json +5 -0
- package/src/auth-turn/fixtures/invalid/missing-kind.json +3 -0
- package/src/auth-turn/fixtures/invalid/missing-turn-id.json +3 -0
- package/src/auth-turn/fixtures/invalid/timing-unknown-field.json +7 -0
- package/src/auth-turn/fixtures/invalid/turn-id-snake-case.json +4 -0
- package/src/auth-turn/fixtures/invalid/unknown-top-level-field.json +5 -0
- package/src/auth-turn/fixtures/valid/abort.json +8 -0
- package/src/auth-turn/fixtures/valid/challenge.json +17 -0
- package/src/auth-turn/fixtures/valid/complete.json +13 -0
- package/src/auth-turn/fixtures/valid/form.json +14 -0
- package/src/auth-turn/fixtures/valid/message.json +13 -0
- package/src/auth-turn/fixtures/valid/multi_choice.json +15 -0
- package/src/auth-turn/fixtures/valid/pending.json +5 -0
- package/src/auth-turn/fixtures/valid/poll.json +9 -0
- package/src/auth-turn/fixtures/valid/redirect.json +16 -0
- package/src/auth-turn/fixtures/valid/retry.json +8 -0
- package/src/auth-turn/fixtures/valid/unknown-kind.json +7 -0
- package/src/auth-turn/index.ts +177 -0
- package/src/auth.ts +786 -0
- package/src/ceremonies/index.ts +9 -43
- package/src/cli/create.ts +60 -97
- package/src/cli/templates/provider/AGENTS.md.tpl +87 -0
- package/src/cli/templates/provider/CLAUDE.md.tpl +1 -0
- package/src/cli/templates/provider/README.md.tpl +7 -4
- package/src/cli/templates/provider/skills/fixtures-and-recording/SKILL.md.tpl +58 -0
- package/src/cli/templates/provider/skills/health-checks-and-fail-closed/SKILL.md.tpl +65 -0
- package/src/cli/templates/provider/skills/normalization-standards/SKILL.md.tpl +57 -0
- package/src/cli/templates/provider/skills/pagination-and-counts/SKILL.md.tpl +52 -0
- package/src/cli/templates/provider/skills/upstream-contract-verification/SKILL.md.tpl +45 -0
- package/src/cli/templates/provider/skills/upstream-notes/README.md.tpl +13 -0
- package/src/contract.ts +1 -0
- package/src/define.ts +198 -71
- package/src/index.ts +16 -0
- package/src/lint.ts +33 -0
- package/src/provider.ts +27 -0
- package/src/runtime/auth-flow.ts +2 -0
- package/src/runtime/browser.ts +293 -1
- package/src/runtime/http.ts +84 -530
- package/src/runtime/proxy-retry-policy.ts +469 -0
- package/src/runtime/stealth.ts +269 -353
- package/src/server/index.ts +36 -0
- package/src/server/self-test-input-tokens.ts +46 -0
- package/src/server/self-test-redaction.ts +97 -0
- package/src/server/self-test-token.ts +70 -0
- package/src/server/self-test.ts +725 -0
- package/src/server/serve.ts +75 -6
- package/src/server/types.ts +1 -0
- package/src/testing/run.ts +9 -1
- package/src/types.ts +186 -0
package/src/runtime/stealth.ts
CHANGED
|
@@ -10,18 +10,17 @@ import {
|
|
|
10
10
|
resolveProxyConfigAsync,
|
|
11
11
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
12
12
|
} from "../config/loader";
|
|
13
|
-
import {
|
|
13
|
+
import { SDKError, TransportError } from "../errors";
|
|
14
14
|
import { getStealthProfile } from "../stealth/profiles";
|
|
15
15
|
import type {
|
|
16
16
|
CookieJar,
|
|
17
17
|
HttpMethod,
|
|
18
|
-
HttpRetryOptions,
|
|
19
18
|
StealthClient,
|
|
20
19
|
StealthFetchOptions,
|
|
20
|
+
StealthRedirectHop,
|
|
21
21
|
StealthResponse,
|
|
22
22
|
StealthSession,
|
|
23
23
|
} from "../types";
|
|
24
|
-
import { HttpRetryPreset, HttpRetryUnsafeMethodPolicy } from "../types";
|
|
25
24
|
import {
|
|
26
25
|
createProxyAuthIpDeniedError,
|
|
27
26
|
createProxyEdgeAuthRejectedError,
|
|
@@ -34,10 +33,17 @@ import {
|
|
|
34
33
|
isProxyPoolRefreshableError,
|
|
35
34
|
isProxyPoolStaleMessage,
|
|
36
35
|
isProxyPoolStaleStatus,
|
|
37
|
-
PROXY_AUTH_IP_DENIED_CODE,
|
|
38
36
|
PROXY_EDGE_AUTH_REJECTED_CODE,
|
|
39
37
|
PROXY_POOL_STALE_CODE,
|
|
40
38
|
} from "./proxy-errors";
|
|
39
|
+
import {
|
|
40
|
+
computeProxyAttemptIndex,
|
|
41
|
+
computeProxyTransportRetryDelayMs,
|
|
42
|
+
createDefaultProxyTransportRetryOptions,
|
|
43
|
+
normalizeProxyTransportRetryOptions,
|
|
44
|
+
shouldRetryProxyTransportAttempt,
|
|
45
|
+
validateUnsafeProxyTransportRetryMethods,
|
|
46
|
+
} from "./proxy-retry-policy";
|
|
41
47
|
import { appendQueryParams } from "./request-options";
|
|
42
48
|
|
|
43
49
|
const DEFAULT_PROFILE = "chrome-146";
|
|
@@ -52,31 +58,7 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
52
58
|
/\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
53
59
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
54
60
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
55
|
-
const
|
|
56
|
-
const DEFAULT_STEALTH_RETRY_ERROR_CODES = [
|
|
57
|
-
PROXY_CONNECT_FAILURE_CODE,
|
|
58
|
-
"transport_network_error",
|
|
59
|
-
"transport_timeout",
|
|
60
|
-
] as const;
|
|
61
|
-
const RATE_LIMIT_STEALTH_RETRY_ERROR_CODES = ["transport_timeout"] as const;
|
|
62
|
-
const KNOWN_STEALTH_RETRY_METHODS = new Set([
|
|
63
|
-
"GET",
|
|
64
|
-
"HEAD",
|
|
65
|
-
"POST",
|
|
66
|
-
"PUT",
|
|
67
|
-
"DELETE",
|
|
68
|
-
"OPTIONS",
|
|
69
|
-
"TRACE",
|
|
70
|
-
"PATCH",
|
|
71
|
-
]);
|
|
72
|
-
const UNSAFE_STEALTH_RETRY_METHODS = new Set([
|
|
73
|
-
"POST",
|
|
74
|
-
"PUT",
|
|
75
|
-
"PATCH",
|
|
76
|
-
"DELETE",
|
|
77
|
-
"TRACE",
|
|
78
|
-
]);
|
|
79
|
-
const MAX_STEALTH_RETRY_ATTEMPTS = 8;
|
|
61
|
+
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
80
62
|
|
|
81
63
|
export type StealthClientOptions = ProxyResolutionOptions & {
|
|
82
64
|
warn?: (message: string) => void;
|
|
@@ -128,15 +110,15 @@ const FIREFOX_IMPIT_BY_MAJOR: Record<number, ImpitBrowser> = {
|
|
|
128
110
|
|
|
129
111
|
type StealthTransportResponse = Pick<
|
|
130
112
|
ImpitResponse,
|
|
131
|
-
"arrayBuffer" | "headers" | "json" | "ok" | "status" | "text"
|
|
132
|
-
|
|
113
|
+
"arrayBuffer" | "headers" | "json" | "ok" | "status" | "text"
|
|
114
|
+
> & {
|
|
115
|
+
url?: string;
|
|
116
|
+
redirected?: boolean;
|
|
117
|
+
};
|
|
133
118
|
|
|
134
119
|
type StealthMethod = NonNullable<ImpitRequestInit["method"]>;
|
|
135
|
-
type
|
|
136
|
-
|
|
137
|
-
methods: readonly string[];
|
|
138
|
-
errorCodes: readonly string[];
|
|
139
|
-
unsafeMethodPolicy: HttpRetryOptions["unsafeMethodPolicy"];
|
|
120
|
+
type StealthRequestInit = ImpitRequestInit & {
|
|
121
|
+
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
140
122
|
};
|
|
141
123
|
|
|
142
124
|
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
@@ -177,12 +159,37 @@ class CookieJarImpl implements CookieJar {
|
|
|
177
159
|
return { ...this.cookies };
|
|
178
160
|
}
|
|
179
161
|
|
|
162
|
+
has(name: string): boolean {
|
|
163
|
+
return Object.hasOwn(this.cookies, name);
|
|
164
|
+
}
|
|
165
|
+
|
|
180
166
|
toString(): string {
|
|
181
167
|
return Object.entries(this.cookies)
|
|
182
168
|
.map(([name, value]) => `${name}=${value}`)
|
|
183
169
|
.join("; ");
|
|
184
170
|
}
|
|
185
171
|
|
|
172
|
+
toHeader(): string {
|
|
173
|
+
return this.toString();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
snapshot(): Record<string, string> {
|
|
177
|
+
return this.getAll();
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
restore(cookies: Record<string, string>): void {
|
|
181
|
+
this.clear();
|
|
182
|
+
for (const [name, value] of Object.entries(cookies)) {
|
|
183
|
+
if (name) this.cookies[name] = value;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
clear(): void {
|
|
188
|
+
for (const name of Object.keys(this.cookies)) {
|
|
189
|
+
delete this.cookies[name];
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
186
193
|
find(predicate: (cookie: string) => boolean): string | undefined {
|
|
187
194
|
for (const [name, value] of Object.entries(this.cookies)) {
|
|
188
195
|
const cookie = `${name}=${value}`;
|
|
@@ -271,9 +278,7 @@ function normalizeHeaders(
|
|
|
271
278
|
function hasOwn(object: object, key: string): boolean {
|
|
272
279
|
return Object.hasOwn(object, key);
|
|
273
280
|
}
|
|
274
|
-
function toImpitCookieJar(
|
|
275
|
-
cookieJar: CookieJarImpl,
|
|
276
|
-
): NonNullable<ImpitOptions["cookieJar"]> {
|
|
281
|
+
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
277
282
|
return {
|
|
278
283
|
setCookie(cookie: string, _url: string, cb?: (error?: unknown) => void) {
|
|
279
284
|
cookieJar.setFromCookieStrings([cookie]);
|
|
@@ -290,10 +295,8 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
290
295
|
const unsupported: string[] = [];
|
|
291
296
|
if (hasOwn(options, "headerOrder")) unsupported.push("headerOrder");
|
|
292
297
|
const stealth = options.stealth;
|
|
293
|
-
if (isRecord(stealth) && hasOwn(stealth, "ja3"))
|
|
294
|
-
|
|
295
|
-
if (isRecord(stealth) && hasOwn(stealth, "h2"))
|
|
296
|
-
unsupported.push("stealth.h2");
|
|
298
|
+
if (isRecord(stealth) && hasOwn(stealth, "ja3")) unsupported.push("stealth.ja3");
|
|
299
|
+
if (isRecord(stealth) && hasOwn(stealth, "h2")) unsupported.push("stealth.h2");
|
|
297
300
|
if (unsupported.length === 0) return;
|
|
298
301
|
|
|
299
302
|
throw new SDKError(
|
|
@@ -301,9 +304,7 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
301
304
|
);
|
|
302
305
|
}
|
|
303
306
|
|
|
304
|
-
function responseHeadersToRecord(
|
|
305
|
-
headers: Headers,
|
|
306
|
-
): Record<string, string | string[] | undefined> {
|
|
307
|
+
function responseHeadersToRecord(headers: Headers): Record<string, string | string[] | undefined> {
|
|
307
308
|
const record: Record<string, string> = {};
|
|
308
309
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
309
310
|
return record;
|
|
@@ -335,17 +336,22 @@ function splitCombinedSetCookieHeader(headerValue: string): string[] {
|
|
|
335
336
|
|
|
336
337
|
export async function normalizeResponse(
|
|
337
338
|
response: StealthTransportResponse,
|
|
339
|
+
requestUrl?: string,
|
|
338
340
|
): Promise<StealthResponse> {
|
|
339
341
|
const headers = Object.fromEntries(response.headers.entries());
|
|
340
|
-
const cookies = new CookieJarImpl(
|
|
341
|
-
setCookieHeadersFromResponse(response.headers),
|
|
342
|
-
);
|
|
342
|
+
const cookies = new CookieJarImpl(setCookieHeadersFromResponse(response.headers));
|
|
343
343
|
const bodyBytes = await response.arrayBuffer();
|
|
344
344
|
const body = new TextDecoder().decode(bodyBytes);
|
|
345
345
|
|
|
346
346
|
return {
|
|
347
347
|
status: response.status,
|
|
348
348
|
ok: response.status >= 200 && response.status < 300,
|
|
349
|
+
...(response.url ? { url: response.url } : {}),
|
|
350
|
+
...(response.redirected !== undefined
|
|
351
|
+
? { redirected: response.redirected }
|
|
352
|
+
: requestUrl && response.url
|
|
353
|
+
? { redirected: response.url !== requestUrl }
|
|
354
|
+
: {}),
|
|
349
355
|
headers,
|
|
350
356
|
rawHeaders: headerEntriesFromHeaders(response.headers),
|
|
351
357
|
body,
|
|
@@ -383,189 +389,11 @@ function isPolicyManagedProxy(options: StealthClientOptions): boolean {
|
|
|
383
389
|
return Boolean(policy && typeof policy === "object");
|
|
384
390
|
}
|
|
385
391
|
|
|
386
|
-
function
|
|
387
|
-
return
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
function createStealthRetryOptions(
|
|
391
|
-
preset: HttpRetryPreset,
|
|
392
|
-
): NormalizedStealthRetryOptions {
|
|
393
|
-
switch (preset) {
|
|
394
|
-
case HttpRetryPreset.Off:
|
|
395
|
-
return {
|
|
396
|
-
attempts: 1,
|
|
397
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
398
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
399
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
400
|
-
};
|
|
401
|
-
case HttpRetryPreset.AggressiveRead:
|
|
402
|
-
return {
|
|
403
|
-
attempts: 4,
|
|
404
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
405
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
406
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
407
|
-
};
|
|
408
|
-
case HttpRetryPreset.RateLimitAware:
|
|
409
|
-
return {
|
|
410
|
-
attempts: 3,
|
|
411
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
412
|
-
errorCodes: RATE_LIMIT_STEALTH_RETRY_ERROR_CODES,
|
|
413
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
414
|
-
};
|
|
415
|
-
case HttpRetryPreset.SafeRead:
|
|
416
|
-
case HttpRetryPreset.TransportTransient:
|
|
417
|
-
return {
|
|
418
|
-
attempts: 3,
|
|
419
|
-
methods: DEFAULT_STEALTH_RETRY_METHODS,
|
|
420
|
-
errorCodes: DEFAULT_STEALTH_RETRY_ERROR_CODES,
|
|
421
|
-
unsafeMethodPolicy: HttpRetryUnsafeMethodPolicy.Reject,
|
|
422
|
-
};
|
|
423
|
-
}
|
|
424
|
-
throw new ProviderError(`Unknown stealth retry preset: ${preset}`, {
|
|
425
|
-
code: "retry_invalid_policy",
|
|
426
|
-
});
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
function normalizeStealthRetryOptions(
|
|
430
|
-
retry: StealthFetchOptions["retry"],
|
|
431
|
-
): NormalizedStealthRetryOptions | undefined {
|
|
432
|
-
if (retry === undefined) return undefined;
|
|
433
|
-
if (retry === false) return createStealthRetryOptions(HttpRetryPreset.Off);
|
|
434
|
-
if (retry === true)
|
|
435
|
-
return createStealthRetryOptions(HttpRetryPreset.TransportTransient);
|
|
436
|
-
if (typeof retry === "string") {
|
|
437
|
-
if (!Object.values(HttpRetryPreset).includes(retry)) {
|
|
438
|
-
throw new ProviderError(`Unknown stealth retry preset: ${retry}`, {
|
|
439
|
-
code: "retry_invalid_policy",
|
|
440
|
-
});
|
|
441
|
-
}
|
|
442
|
-
return createStealthRetryOptions(retry);
|
|
443
|
-
}
|
|
444
|
-
if (typeof retry !== "object" || retry === null || Array.isArray(retry)) {
|
|
445
|
-
throw new ProviderError("Stealth retry policy must be a plain object", {
|
|
446
|
-
code: "retry_invalid_policy",
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
|
-
if (
|
|
450
|
-
retry.unsafeMethodPolicy !== undefined &&
|
|
451
|
-
!Object.values(HttpRetryUnsafeMethodPolicy).includes(
|
|
452
|
-
retry.unsafeMethodPolicy,
|
|
453
|
-
)
|
|
454
|
-
) {
|
|
455
|
-
throw new ProviderError(
|
|
456
|
-
`Unknown stealth retry unsafe method policy: ${String(retry.unsafeMethodPolicy)}`,
|
|
457
|
-
{ code: "retry_invalid_policy" },
|
|
458
|
-
);
|
|
459
|
-
}
|
|
460
|
-
if (retry.methods !== undefined) {
|
|
461
|
-
if (!Array.isArray(retry.methods)) {
|
|
462
|
-
throw new ProviderError("Stealth retry methods must be an array", {
|
|
463
|
-
code: "retry_invalid_policy",
|
|
464
|
-
});
|
|
465
|
-
}
|
|
466
|
-
const unknownMethods = retry.methods
|
|
467
|
-
.map((method) => (typeof method === "string" ? method.toUpperCase() : ""))
|
|
468
|
-
.filter((method) => !KNOWN_STEALTH_RETRY_METHODS.has(method));
|
|
469
|
-
if (unknownMethods.length > 0) {
|
|
470
|
-
throw new ProviderError(
|
|
471
|
-
`Unknown stealth retry method(s): ${unknownMethods.join(", ")}`,
|
|
472
|
-
{ code: "retry_invalid_policy" },
|
|
473
|
-
);
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
if (retry.errorCodes !== undefined) {
|
|
477
|
-
if (
|
|
478
|
-
!Array.isArray(retry.errorCodes) ||
|
|
479
|
-
retry.errorCodes.some((errorCode) => typeof errorCode !== "string")
|
|
480
|
-
) {
|
|
481
|
-
throw new ProviderError(
|
|
482
|
-
"Stealth retry errorCodes must contain only strings",
|
|
483
|
-
{ code: "retry_invalid_policy" },
|
|
484
|
-
);
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
const base = createStealthRetryOptions(
|
|
489
|
-
retry.preset ?? HttpRetryPreset.TransportTransient,
|
|
490
|
-
);
|
|
491
|
-
const attempts =
|
|
492
|
-
retry.attempts === undefined || !Number.isFinite(retry.attempts)
|
|
493
|
-
? base.attempts
|
|
494
|
-
: Math.max(
|
|
495
|
-
1,
|
|
496
|
-
Math.min(MAX_STEALTH_RETRY_ATTEMPTS, Math.floor(retry.attempts)),
|
|
497
|
-
);
|
|
498
|
-
const normalized: NormalizedStealthRetryOptions = {
|
|
499
|
-
attempts,
|
|
500
|
-
methods:
|
|
501
|
-
retry.methods?.map((method) => method.toUpperCase()) ?? base.methods,
|
|
502
|
-
errorCodes: retry.errorCodes ?? base.errorCodes,
|
|
503
|
-
unsafeMethodPolicy: retry.unsafeMethodPolicy ?? base.unsafeMethodPolicy,
|
|
504
|
-
};
|
|
505
|
-
|
|
506
|
-
if (
|
|
507
|
-
normalized.unsafeMethodPolicy !==
|
|
508
|
-
HttpRetryUnsafeMethodPolicy.AllowExplicitUnsafe
|
|
509
|
-
) {
|
|
510
|
-
const unsafeMethods = normalized.methods.filter((method) =>
|
|
511
|
-
UNSAFE_STEALTH_RETRY_METHODS.has(method.toUpperCase()),
|
|
512
|
-
);
|
|
513
|
-
if (unsafeMethods.length > 0) {
|
|
514
|
-
throw new ProviderError(
|
|
515
|
-
`Stealth retry methods include unsafe method(s): ${unsafeMethods.join(", ")}`,
|
|
516
|
-
{ code: "retry_unsafe_method" },
|
|
517
|
-
);
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
return normalized;
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
function isExplicitStealthRetryAllowed(
|
|
525
|
-
method: StealthMethod,
|
|
526
|
-
error: TransportError,
|
|
527
|
-
retryOptions: NormalizedStealthRetryOptions | undefined,
|
|
528
|
-
): boolean {
|
|
529
|
-
if (!retryOptions || retryOptions.attempts <= 1) return false;
|
|
530
|
-
return (
|
|
531
|
-
retryOptions.methods.includes(method.toUpperCase()) &&
|
|
532
|
-
retryOptions.errorCodes.includes(proxyAttemptErrorCode(error))
|
|
533
|
-
);
|
|
392
|
+
function isProxyConnectFailureResponse(response: StealthTransportResponse, body: string): boolean {
|
|
393
|
+
return response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "");
|
|
534
394
|
}
|
|
535
395
|
|
|
536
|
-
function
|
|
537
|
-
if (error instanceof TransportError) {
|
|
538
|
-
if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
|
|
539
|
-
return false;
|
|
540
|
-
}
|
|
541
|
-
return (
|
|
542
|
-
error.code === PROXY_CONNECT_FAILURE_CODE ||
|
|
543
|
-
error.code === "transport_network_error" ||
|
|
544
|
-
error.code === "transport_timeout"
|
|
545
|
-
);
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
if (error instanceof SDKError) {
|
|
549
|
-
return false;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
553
|
-
return /\bproxy\b|\bnon[\s-]?200\b|\bconnect\b|\btunnel\b/i.test(message);
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
function isProxyConnectFailureResponse(
|
|
557
|
-
response: StealthTransportResponse,
|
|
558
|
-
body: string,
|
|
559
|
-
): boolean {
|
|
560
|
-
return (
|
|
561
|
-
response.status === 0 && PROXY_CONNECT_FAILURE_BODY_PATTERN.test(body ?? "")
|
|
562
|
-
);
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
function createProxyConnectFailureError(
|
|
566
|
-
body: string,
|
|
567
|
-
cause?: Error,
|
|
568
|
-
): TransportError {
|
|
396
|
+
function createProxyConnectFailureError(body: string, cause?: Error): TransportError {
|
|
569
397
|
const bodyExcerpt = (body ?? "").trim().slice(0, 1_000);
|
|
570
398
|
if (isProxyAuthIpDeniedMessage(bodyExcerpt)) {
|
|
571
399
|
return createProxyAuthIpDeniedError(cause);
|
|
@@ -574,10 +402,7 @@ function createProxyConnectFailureError(
|
|
|
574
402
|
return createProxyEdgeAuthRejectedError(cause);
|
|
575
403
|
}
|
|
576
404
|
if (isProxyPoolStaleMessage(bodyExcerpt)) {
|
|
577
|
-
return createProxyPoolStaleError(
|
|
578
|
-
bodyExcerpt.includes("512") ? 512 : 509,
|
|
579
|
-
cause,
|
|
580
|
-
);
|
|
405
|
+
return createProxyPoolStaleError(bodyExcerpt.includes("512") ? 512 : 509, cause);
|
|
581
406
|
}
|
|
582
407
|
return new TransportError(bodyExcerpt || "Proxy CONNECT failed", {
|
|
583
408
|
code: PROXY_CONNECT_FAILURE_CODE,
|
|
@@ -684,33 +509,20 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
684
509
|
}
|
|
685
510
|
|
|
686
511
|
if (isProxyAuthIpDeniedMessage(message)) {
|
|
687
|
-
return createProxyAuthIpDeniedError(
|
|
688
|
-
error instanceof Error ? error : undefined,
|
|
689
|
-
);
|
|
512
|
+
return createProxyAuthIpDeniedError(error instanceof Error ? error : undefined);
|
|
690
513
|
}
|
|
691
514
|
|
|
692
515
|
if (isProxyEdgeAuthRejectedMessage(message)) {
|
|
693
|
-
return createProxyEdgeAuthRejectedError(
|
|
694
|
-
error instanceof Error ? error : undefined,
|
|
695
|
-
);
|
|
516
|
+
return createProxyEdgeAuthRejectedError(error instanceof Error ? error : undefined);
|
|
696
517
|
}
|
|
697
518
|
|
|
698
519
|
const proxyTunnelStatus = getProxyTunnelStatus(error);
|
|
699
|
-
if (
|
|
700
|
-
proxyTunnelStatus
|
|
701
|
-
isProxyPoolStaleStatus(proxyTunnelStatus)
|
|
702
|
-
) {
|
|
703
|
-
return createProxyPoolStaleError(
|
|
704
|
-
proxyTunnelStatus,
|
|
705
|
-
error instanceof Error ? error : undefined,
|
|
706
|
-
);
|
|
520
|
+
if (proxyTunnelStatus !== undefined && isProxyPoolStaleStatus(proxyTunnelStatus)) {
|
|
521
|
+
return createProxyPoolStaleError(proxyTunnelStatus, error instanceof Error ? error : undefined);
|
|
707
522
|
}
|
|
708
523
|
|
|
709
524
|
if (PROXY_CONNECT_FAILURE_BODY_PATTERN.test(message)) {
|
|
710
|
-
return createProxyConnectFailureError(
|
|
711
|
-
message,
|
|
712
|
-
error instanceof Error ? error : undefined,
|
|
713
|
-
);
|
|
525
|
+
return createProxyConnectFailureError(message, error instanceof Error ? error : undefined);
|
|
714
526
|
}
|
|
715
527
|
|
|
716
528
|
return new TransportError("Network error", {
|
|
@@ -720,6 +532,10 @@ function normalizeStealthTransportError(error: unknown): TransportError {
|
|
|
720
532
|
});
|
|
721
533
|
}
|
|
722
534
|
|
|
535
|
+
function sleep(ms: number): Promise<void> {
|
|
536
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
537
|
+
}
|
|
538
|
+
|
|
723
539
|
function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
724
540
|
switch (method.toUpperCase()) {
|
|
725
541
|
case "HEAD":
|
|
@@ -743,6 +559,23 @@ function normalizeMethod(method: HttpMethod | string): StealthMethod {
|
|
|
743
559
|
}
|
|
744
560
|
}
|
|
745
561
|
|
|
562
|
+
function isRedirectStatus(status: number): boolean {
|
|
563
|
+
return [301, 302, 303, 307, 308].includes(status);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function nextRedirectMethod(status: number, method: StealthMethod): StealthMethod {
|
|
567
|
+
if (status === 303 && method !== "HEAD") return "GET";
|
|
568
|
+
if ((status === 301 || status === 302) && method === "POST") return "GET";
|
|
569
|
+
return method;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function locationHeader(headers: Record<string, string>): string | undefined {
|
|
573
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
574
|
+
if (name.toLowerCase() === "location") return value;
|
|
575
|
+
}
|
|
576
|
+
return undefined;
|
|
577
|
+
}
|
|
578
|
+
|
|
746
579
|
function createSessionFetcher(
|
|
747
580
|
baseUrl: string,
|
|
748
581
|
defaultProfile: string,
|
|
@@ -783,19 +616,16 @@ function createSessionFetcher(
|
|
|
783
616
|
options?: StealthFetchOptions,
|
|
784
617
|
proxyAttempt?: number,
|
|
785
618
|
): Promise<ResolvedAttemptProxy> {
|
|
786
|
-
const rawProxyAttemptOffset = options?.proxyAttemptOffset ?? 0;
|
|
787
|
-
const proxyAttemptOffset = Number.isFinite(rawProxyAttemptOffset)
|
|
788
|
-
? Math.max(0, Math.floor(rawProxyAttemptOffset))
|
|
789
|
-
: 0;
|
|
790
619
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
791
620
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
792
621
|
upstream: clientOptions.upstream,
|
|
793
622
|
apifuseConfig: clientOptions.apifuseConfig,
|
|
794
623
|
affinityKey: clientOptions.affinityKey,
|
|
795
|
-
proxyAttempt:
|
|
796
|
-
proxyAttempt
|
|
797
|
-
|
|
798
|
-
|
|
624
|
+
proxyAttempt: computeProxyAttemptIndex({
|
|
625
|
+
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
626
|
+
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
|
+
retryAttemptOffset: proxyAttempt,
|
|
628
|
+
}),
|
|
799
629
|
telemetry: clientOptions.telemetry,
|
|
800
630
|
});
|
|
801
631
|
|
|
@@ -811,27 +641,39 @@ function createSessionFetcher(
|
|
|
811
641
|
};
|
|
812
642
|
}
|
|
813
643
|
|
|
814
|
-
|
|
644
|
+
const session: StealthSession = {
|
|
815
645
|
async fetch(url, options: StealthFetchOptions = {}) {
|
|
816
646
|
const method = normalizeMethod(options.method ?? "GET");
|
|
817
647
|
const hasExplicitRetryPolicy = options.retry !== undefined;
|
|
818
|
-
const stealthRetryOptions =
|
|
648
|
+
const stealthRetryOptions =
|
|
649
|
+
normalizeProxyTransportRetryOptions(options.retry, {
|
|
650
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
651
|
+
label: "Stealth",
|
|
652
|
+
}) ??
|
|
653
|
+
(hasExplicitRetryPolicy
|
|
654
|
+
? undefined
|
|
655
|
+
: createDefaultProxyTransportRetryOptions({
|
|
656
|
+
extraErrorCodes: STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES,
|
|
657
|
+
label: "Stealth",
|
|
658
|
+
}));
|
|
659
|
+
if (stealthRetryOptions) {
|
|
660
|
+
validateUnsafeProxyTransportRetryMethods(stealthRetryOptions, "Stealth");
|
|
661
|
+
}
|
|
819
662
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
820
|
-
const usesPolicyAllocator =
|
|
821
|
-
|
|
822
|
-
const
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
: 1;
|
|
663
|
+
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
|
+
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
665
|
+
const policyProxyAttemptCap = Math.max(
|
|
666
|
+
1,
|
|
667
|
+
Math.min(
|
|
668
|
+
MAX_POLICY_PROXY_RETRY_ATTEMPTS,
|
|
669
|
+
clientOptions.proxyPolicy?.session?.poolSize ??
|
|
670
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
+
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
+
: undefined) ??
|
|
673
|
+
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
|
+
),
|
|
675
|
+
);
|
|
676
|
+
const maxAttempts = usesPolicyAllocator ? policyProxyAttemptCap : retryAttemptCap;
|
|
835
677
|
let lastError: unknown;
|
|
836
678
|
|
|
837
679
|
for (
|
|
@@ -861,9 +703,7 @@ function createSessionFetcher(
|
|
|
861
703
|
...(attemptProxy?.poolIndex === undefined
|
|
862
704
|
? {}
|
|
863
705
|
: { poolIndex: attemptProxy.poolIndex }),
|
|
864
|
-
...(attemptProxy?.proxyHash
|
|
865
|
-
? { proxyHash: attemptProxy.proxyHash }
|
|
866
|
-
: {}),
|
|
706
|
+
...(attemptProxy?.proxyHash ? { proxyHash: attemptProxy.proxyHash } : {}),
|
|
867
707
|
outcome,
|
|
868
708
|
...(errorCode ? { errorCode } : {}),
|
|
869
709
|
...(status === undefined ? {} : { status }),
|
|
@@ -874,7 +714,7 @@ function createSessionFetcher(
|
|
|
874
714
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
875
715
|
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
876
716
|
proxy = attemptProxy.url;
|
|
877
|
-
if (proxy) {
|
|
717
|
+
if (proxy && usesPolicyAllocator) {
|
|
878
718
|
if (attemptedProxies.has(proxy)) {
|
|
879
719
|
break;
|
|
880
720
|
}
|
|
@@ -882,42 +722,32 @@ function createSessionFetcher(
|
|
|
882
722
|
}
|
|
883
723
|
const ignoreTlsErrors = Boolean(
|
|
884
724
|
options.stealth?.insecureSkipVerify ??
|
|
885
|
-
(!hasPolicyProxy &&
|
|
886
|
-
proxy &&
|
|
887
|
-
clientOptions.proxyStealth?.insecureSkipVerify),
|
|
725
|
+
(!hasPolicyProxy && proxy && clientOptions.proxyStealth?.insecureSkipVerify),
|
|
888
726
|
);
|
|
889
727
|
const profileName = options.profile ?? defaultProfile;
|
|
890
|
-
const requestUrl = appendQueryParams(
|
|
891
|
-
resolveUrl(baseUrl, url),
|
|
892
|
-
options.params,
|
|
893
|
-
);
|
|
728
|
+
const requestUrl = appendQueryParams(resolveUrl(baseUrl, url), options.params);
|
|
894
729
|
const headers = { ...(options.headers ?? {}) };
|
|
895
730
|
if (!hasHeader(headers, "Cookie")) {
|
|
896
731
|
const cookieHeader = cookieJar.toString();
|
|
897
732
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
898
733
|
}
|
|
899
|
-
const requestInit:
|
|
734
|
+
const requestInit: StealthRequestInit = {
|
|
900
735
|
headers: normalizeHeaders(headers),
|
|
901
736
|
method,
|
|
737
|
+
...(options.redirect ? { redirect: options.redirect } : {}),
|
|
902
738
|
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
903
739
|
};
|
|
904
740
|
if (options.body !== undefined) {
|
|
905
741
|
requestInit.body = normalizeBody(options.body);
|
|
906
742
|
}
|
|
907
|
-
const response = await getClient(
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
ignoreTlsErrors,
|
|
911
|
-
).fetch(requestUrl, requestInit);
|
|
912
|
-
const normalized = await normalizeResponse(response);
|
|
913
|
-
cookieJar.setFromCookieStrings(
|
|
914
|
-
setCookieHeadersFromResponse(response.headers),
|
|
743
|
+
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(
|
|
744
|
+
requestUrl,
|
|
745
|
+
requestInit,
|
|
915
746
|
);
|
|
747
|
+
const normalized = await normalizeResponse(response, requestUrl);
|
|
748
|
+
cookieJar.setFromCookieStrings(setCookieHeadersFromResponse(response.headers));
|
|
916
749
|
|
|
917
|
-
if (
|
|
918
|
-
proxy &&
|
|
919
|
-
isProxyConnectFailureResponse(response, normalized.body)
|
|
920
|
-
) {
|
|
750
|
+
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
921
751
|
throw createProxyConnectFailureError(normalized.body);
|
|
922
752
|
}
|
|
923
753
|
|
|
@@ -927,10 +757,9 @@ function createSessionFetcher(
|
|
|
927
757
|
usesPolicyAllocator &&
|
|
928
758
|
isProxyEdgeTlsRejectedResponse(
|
|
929
759
|
response.status,
|
|
930
|
-
[
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
].join("\n"),
|
|
760
|
+
[JSON.stringify(responseHeadersToRecord(response.headers)), normalized.body].join(
|
|
761
|
+
"\n",
|
|
762
|
+
),
|
|
934
763
|
)
|
|
935
764
|
) {
|
|
936
765
|
throw createProxyEdgeTlsRejectedError(response.status);
|
|
@@ -951,13 +780,10 @@ function createSessionFetcher(
|
|
|
951
780
|
}
|
|
952
781
|
|
|
953
782
|
if (response.status >= 400 && options.throwOnHttpError !== false) {
|
|
954
|
-
throw new TransportError(
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
status: response.status,
|
|
959
|
-
},
|
|
960
|
-
);
|
|
783
|
+
throw new TransportError(`Upstream request failed with status ${response.status}`, {
|
|
784
|
+
code: "upstream_http_error",
|
|
785
|
+
status: response.status,
|
|
786
|
+
});
|
|
961
787
|
}
|
|
962
788
|
|
|
963
789
|
recordProxyAttempt("ok", undefined, response.status);
|
|
@@ -970,11 +796,7 @@ function createSessionFetcher(
|
|
|
970
796
|
proxyAttemptStatus(normalizedError),
|
|
971
797
|
);
|
|
972
798
|
lastError = normalizedError;
|
|
973
|
-
if (
|
|
974
|
-
proxy &&
|
|
975
|
-
usesPolicyAllocator &&
|
|
976
|
-
isProxyPoolRefreshableError(normalizedError)
|
|
977
|
-
) {
|
|
799
|
+
if (proxy && usesPolicyAllocator && isProxyPoolRefreshableError(normalizedError)) {
|
|
978
800
|
stalePoolError = normalizedError;
|
|
979
801
|
if (shouldRunProxyAuthDiagnostic(normalizedError)) {
|
|
980
802
|
stalePoolDiagnosticProxy = proxy;
|
|
@@ -985,20 +807,21 @@ function createSessionFetcher(
|
|
|
985
807
|
break;
|
|
986
808
|
}
|
|
987
809
|
if (
|
|
988
|
-
proxy &&
|
|
989
810
|
attempt + 1 <
|
|
990
811
|
(stealthRetryOptions
|
|
991
812
|
? Math.min(maxAttempts, stealthRetryOptions.attempts)
|
|
992
813
|
: maxAttempts) &&
|
|
993
|
-
(
|
|
994
|
-
|
|
995
|
-
:
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
isRetryableProxyTransportError(normalizedError)
|
|
814
|
+
shouldRetryProxyTransportAttempt({
|
|
815
|
+
error: normalizedError,
|
|
816
|
+
explicitRetry: hasExplicitRetryPolicy,
|
|
817
|
+
method,
|
|
818
|
+
options: stealthRetryOptions,
|
|
819
|
+
proxyUsed: Boolean(proxy),
|
|
820
|
+
})
|
|
1001
821
|
) {
|
|
822
|
+
if (stealthRetryOptions) {
|
|
823
|
+
await sleep(computeProxyTransportRetryDelayMs(stealthRetryOptions!, attempt + 1));
|
|
824
|
+
}
|
|
1002
825
|
continue;
|
|
1003
826
|
}
|
|
1004
827
|
throw normalizedError;
|
|
@@ -1052,33 +875,138 @@ function createSessionFetcher(
|
|
|
1052
875
|
|
|
1053
876
|
throw normalizeStealthTransportError(lastError);
|
|
1054
877
|
},
|
|
878
|
+
cookies: cookieJar,
|
|
879
|
+
redirects: {
|
|
880
|
+
async run(options) {
|
|
881
|
+
const maxHops =
|
|
882
|
+
options.maxHops === undefined || !Number.isFinite(options.maxHops)
|
|
883
|
+
? 10
|
|
884
|
+
: Math.max(0, Math.floor(options.maxHops));
|
|
885
|
+
const hops: StealthRedirectHop[] = [];
|
|
886
|
+
let currentUrl = resolveUrl(baseUrl, options.url);
|
|
887
|
+
let method = normalizeMethod(options.method ?? "GET");
|
|
888
|
+
let body = options.body;
|
|
889
|
+
let response: StealthResponse | undefined;
|
|
890
|
+
const visitedRequests = new Set<string>();
|
|
891
|
+
|
|
892
|
+
const { url: _url, maxHops: _maxHops, stopWhen, params, ...fetchOptions } = options;
|
|
893
|
+
|
|
894
|
+
for (let hopIndex = 0; hopIndex <= maxHops; hopIndex += 1) {
|
|
895
|
+
visitedRequests.add(`${method} ${currentUrl}`);
|
|
896
|
+
response = await session.fetch(currentUrl, {
|
|
897
|
+
...fetchOptions,
|
|
898
|
+
body,
|
|
899
|
+
method,
|
|
900
|
+
...(hopIndex === 0 && params ? { params } : {}),
|
|
901
|
+
redirect: "manual",
|
|
902
|
+
throwOnHttpError: false,
|
|
903
|
+
});
|
|
904
|
+
|
|
905
|
+
if (!isRedirectStatus(response.status)) {
|
|
906
|
+
return {
|
|
907
|
+
final: response,
|
|
908
|
+
hops,
|
|
909
|
+
reason: "completed",
|
|
910
|
+
cookies: cookieJar.snapshot(),
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
const location = locationHeader(response.headers);
|
|
915
|
+
const nextUrl = location
|
|
916
|
+
? new URL(location, response.url ?? currentUrl).toString()
|
|
917
|
+
: undefined;
|
|
918
|
+
const hop: StealthRedirectHop = {
|
|
919
|
+
url: response.url ?? currentUrl,
|
|
920
|
+
status: response.status,
|
|
921
|
+
method,
|
|
922
|
+
...(location ? { location } : {}),
|
|
923
|
+
...(nextUrl ? { nextUrl } : {}),
|
|
924
|
+
};
|
|
925
|
+
hops.push(hop);
|
|
926
|
+
|
|
927
|
+
if (stopWhen && (await stopWhen(hop))) {
|
|
928
|
+
return {
|
|
929
|
+
final: response,
|
|
930
|
+
hops,
|
|
931
|
+
reason: "stopped",
|
|
932
|
+
cookies: cookieJar.snapshot(),
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
if (!nextUrl) {
|
|
937
|
+
return {
|
|
938
|
+
final: response,
|
|
939
|
+
hops,
|
|
940
|
+
reason: "missing_location",
|
|
941
|
+
cookies: cookieJar.snapshot(),
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
if (hops.length > maxHops) {
|
|
946
|
+
return {
|
|
947
|
+
final: response,
|
|
948
|
+
hops,
|
|
949
|
+
reason: "max_hops",
|
|
950
|
+
cookies: cookieJar.snapshot(),
|
|
951
|
+
};
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
const nextMethod = nextRedirectMethod(response.status, method);
|
|
955
|
+
if (nextMethod !== method) {
|
|
956
|
+
body = undefined;
|
|
957
|
+
}
|
|
958
|
+
if (visitedRequests.has(`${nextMethod} ${nextUrl}`)) {
|
|
959
|
+
return {
|
|
960
|
+
final: response,
|
|
961
|
+
hops,
|
|
962
|
+
reason: "loop",
|
|
963
|
+
cookies: cookieJar.snapshot(),
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
method = nextMethod;
|
|
967
|
+
currentUrl = nextUrl;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
if (!response) {
|
|
971
|
+
response = await session.fetch(currentUrl, {
|
|
972
|
+
...fetchOptions,
|
|
973
|
+
body,
|
|
974
|
+
method,
|
|
975
|
+
...(params ? { params } : {}),
|
|
976
|
+
redirect: "manual",
|
|
977
|
+
throwOnHttpError: false,
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
return {
|
|
981
|
+
final: response,
|
|
982
|
+
hops,
|
|
983
|
+
reason: "max_hops",
|
|
984
|
+
cookies: cookieJar.snapshot(),
|
|
985
|
+
};
|
|
986
|
+
},
|
|
987
|
+
},
|
|
1055
988
|
close() {
|
|
1056
989
|
closed = true;
|
|
1057
990
|
clients.clear();
|
|
1058
991
|
},
|
|
1059
992
|
};
|
|
993
|
+
return session;
|
|
1060
994
|
|
|
1061
995
|
async function classifyProxyAuthDiagnostic(
|
|
1062
996
|
profileName: string,
|
|
1063
997
|
proxy: string,
|
|
1064
998
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
1065
999
|
try {
|
|
1066
|
-
const response = await getClient(profileName, proxy, false).fetch(
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1071
|
-
},
|
|
1072
|
-
);
|
|
1000
|
+
const response = await getClient(profileName, proxy, false).fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1001
|
+
method: "GET",
|
|
1002
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1003
|
+
});
|
|
1073
1004
|
const normalized = await normalizeResponse(response);
|
|
1074
1005
|
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1075
1006
|
} catch (error) {
|
|
1076
1007
|
const message =
|
|
1077
1008
|
error instanceof Error
|
|
1078
|
-
? [
|
|
1079
|
-
error.message,
|
|
1080
|
-
error.cause instanceof Error ? error.cause.message : "",
|
|
1081
|
-
]
|
|
1009
|
+
? [error.message, error.cause instanceof Error ? error.cause.message : ""]
|
|
1082
1010
|
.filter(Boolean)
|
|
1083
1011
|
.join(" ")
|
|
1084
1012
|
: String(error);
|
|
@@ -1118,22 +1046,14 @@ export function createStealthClient(
|
|
|
1118
1046
|
clientOptions: StealthClientOptions = {},
|
|
1119
1047
|
): StealthClient {
|
|
1120
1048
|
const defaultProfile =
|
|
1121
|
-
typeof defaultProfileOrOptions === "string"
|
|
1122
|
-
? defaultProfileOrOptions
|
|
1123
|
-
: DEFAULT_PROFILE;
|
|
1049
|
+
typeof defaultProfileOrOptions === "string" ? defaultProfileOrOptions : DEFAULT_PROFILE;
|
|
1124
1050
|
const resolvedClientOptions =
|
|
1125
|
-
typeof defaultProfileOrOptions === "string"
|
|
1126
|
-
? clientOptions
|
|
1127
|
-
: defaultProfileOrOptions;
|
|
1051
|
+
typeof defaultProfileOrOptions === "string" ? clientOptions : defaultProfileOrOptions;
|
|
1128
1052
|
let sharedSession: StealthSession | null = null;
|
|
1129
1053
|
|
|
1130
1054
|
function getSharedSession(): StealthSession {
|
|
1131
1055
|
if (!sharedSession) {
|
|
1132
|
-
sharedSession = createSessionFetcher(
|
|
1133
|
-
baseUrl,
|
|
1134
|
-
defaultProfile,
|
|
1135
|
-
resolvedClientOptions,
|
|
1136
|
-
);
|
|
1056
|
+
sharedSession = createSessionFetcher(baseUrl, defaultProfile, resolvedClientOptions);
|
|
1137
1057
|
}
|
|
1138
1058
|
|
|
1139
1059
|
return sharedSession;
|
|
@@ -1145,11 +1065,7 @@ export function createStealthClient(
|
|
|
1145
1065
|
},
|
|
1146
1066
|
createSession(opts?: { profile?: string }) {
|
|
1147
1067
|
const sessionProfile = opts?.profile ?? defaultProfile;
|
|
1148
|
-
return createSessionFetcher(
|
|
1149
|
-
baseUrl,
|
|
1150
|
-
sessionProfile,
|
|
1151
|
-
resolvedClientOptions,
|
|
1152
|
-
);
|
|
1068
|
+
return createSessionFetcher(baseUrl, sessionProfile, resolvedClientOptions);
|
|
1153
1069
|
},
|
|
1154
1070
|
close() {
|
|
1155
1071
|
sharedSession?.close();
|