@apifuse/provider-sdk 2.2.0-beta.25 → 2.2.0-beta.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +7 -6
- package/CHANGELOG.md +5 -1
- package/README.md +3 -3
- package/bin/apifuse-check.ts +62 -3
- package/bin/apifuse-pack-check.ts +8 -2
- package/bin/apifuse-pack-smoke.ts +43 -2
- package/bin/apifuse-pack-types.ts +58 -0
- package/dist/auth.js +29 -0
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/contract-serialization.d.ts +20 -1
- package/dist/contract-serialization.js +583 -8
- package/dist/contract.d.ts +2 -0
- package/dist/contract.js +9 -5
- package/dist/declaration-validation.d.ts +23 -0
- package/dist/declaration-validation.js +159 -0
- package/dist/define.d.ts +1 -1
- package/dist/define.js +13 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -2
- package/dist/lint.js +85 -3
- package/dist/provider.d.ts +1 -1
- package/dist/provider.js +1 -1
- package/dist/runtime/resolver-vendors/bindings.d.ts +42 -2
- package/dist/runtime/resolver-vendors/bindings.js +31 -6
- package/dist/runtime/resolver-vendors/browser.d.ts +3 -7
- package/dist/runtime/resolver-vendors/browser.js +7 -22
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +23 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +264 -0
- package/dist/runtime/resolver-vendors/types.d.ts +44 -3
- package/dist/runtime/resolver-vendors/types.js +10 -0
- package/dist/runtime/resolver.d.ts +17 -2
- package/dist/runtime/resolver.js +237 -15
- package/dist/runtime/stealth.d.ts +26 -4
- package/dist/runtime/stealth.js +224 -114
- package/dist/schema.d.ts +63 -0
- package/dist/schema.js +808 -8
- package/dist/server/serve.js +8 -0
- package/dist/stealth/profiles.js +16 -7
- package/dist/types.d.ts +34 -1
- package/package.json +2 -2
- package/src/auth.ts +40 -0
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/contract-serialization.ts +857 -8
- package/src/contract.ts +16 -5
- package/src/declaration-validation.ts +202 -0
- package/src/define.ts +23 -2
- package/src/index.ts +12 -0
- package/src/lint.ts +98 -3
- package/src/provider.ts +10 -0
- package/src/runtime/resolver-vendors/bindings.ts +40 -15
- package/src/runtime/resolver-vendors/browser.ts +9 -31
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +366 -0
- package/src/runtime/resolver-vendors/types.ts +54 -0
- package/src/runtime/resolver.ts +304 -24
- package/src/runtime/stealth.ts +317 -136
- package/src/schema.ts +1060 -9
- package/src/server/serve.ts +8 -0
- package/src/stealth/profiles.ts +17 -7
- package/src/types.ts +36 -3
package/src/runtime/stealth.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import type { Browser, ImpitOptions, ImpitResponse, RequestInit } from "impit";
|
|
3
|
-
import { Impit } from "impit";
|
|
4
2
|
import { Cookie, CookieJar as ToughCookieJar } from "tough-cookie";
|
|
3
|
+
import type {
|
|
4
|
+
BrowserProfile,
|
|
5
|
+
EmulationOS,
|
|
6
|
+
RequestInit as WreqRequestInit,
|
|
7
|
+
Session as WreqSession,
|
|
8
|
+
} from "wreq-js";
|
|
5
9
|
|
|
6
10
|
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
7
11
|
import {
|
|
@@ -53,15 +57,16 @@ import {
|
|
|
53
57
|
import {
|
|
54
58
|
evaluateRedirectHop,
|
|
55
59
|
isRedirectStatus,
|
|
60
|
+
nextRedirectMethod,
|
|
56
61
|
resolveRedirectUrl,
|
|
57
62
|
} from "./redirects.js";
|
|
58
63
|
import {
|
|
59
64
|
isSensitiveKey,
|
|
65
|
+
normalizeSensitiveParams,
|
|
60
66
|
redactSensitiveError,
|
|
61
67
|
redactSensitiveRequestError,
|
|
62
68
|
redactSensitiveText,
|
|
63
69
|
redactUrlQueryParams,
|
|
64
|
-
normalizeSensitiveParams,
|
|
65
70
|
serializeRequestUrl,
|
|
66
71
|
} from "./request-options.js";
|
|
67
72
|
|
|
@@ -77,6 +82,13 @@ const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
|
77
82
|
const PROXY_AUTH_DIAGNOSTIC_URL = "http://example.com/";
|
|
78
83
|
const PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS = 5_000;
|
|
79
84
|
const STEALTH_PROXY_TRANSPORT_RETRY_ERROR_CODES = [PROXY_CONNECT_FAILURE_CODE] as const;
|
|
85
|
+
const MAX_STEALTH_REDIRECT_HOPS = 10;
|
|
86
|
+
const REDIRECT_BODY_HEADERS = new Set([
|
|
87
|
+
"content-encoding",
|
|
88
|
+
"content-language",
|
|
89
|
+
"content-location",
|
|
90
|
+
"content-type",
|
|
91
|
+
]);
|
|
80
92
|
|
|
81
93
|
function sensitiveQueryParamNames(url: string): string[] {
|
|
82
94
|
const queryStart = url.indexOf("?");
|
|
@@ -110,45 +122,40 @@ const REMOVED_CHROME_PROFILE_NAMES = new Set([
|
|
|
110
122
|
"edge-131",
|
|
111
123
|
]);
|
|
112
124
|
|
|
113
|
-
type
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
100: "chrome100",
|
|
118
|
-
101: "chrome101",
|
|
119
|
-
104: "chrome104",
|
|
120
|
-
107: "chrome107",
|
|
121
|
-
110: "chrome110",
|
|
122
|
-
116: "chrome116",
|
|
123
|
-
124: "chrome124",
|
|
124
|
-
125: "chrome125",
|
|
125
|
-
131: "chrome131",
|
|
126
|
-
136: "chrome136",
|
|
127
|
-
142: "chrome142",
|
|
125
|
+
type StealthTransportHeaders = {
|
|
126
|
+
entries(): IterableIterator<[string, string]>;
|
|
127
|
+
get(name: string): string | null;
|
|
128
|
+
getSetCookie?: () => string[];
|
|
128
129
|
};
|
|
129
130
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
131
|
+
type StealthTransportBody = {
|
|
132
|
+
cancel(): Promise<void>;
|
|
133
|
+
getReader(): {
|
|
134
|
+
read(): Promise<{ done: boolean; value?: Uint8Array }>;
|
|
135
|
+
cancel(): Promise<void>;
|
|
136
|
+
releaseLock(): void;
|
|
137
|
+
};
|
|
135
138
|
};
|
|
136
139
|
|
|
137
|
-
type StealthTransportResponse =
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
body?:
|
|
142
|
-
abort?: () => void;
|
|
140
|
+
type StealthTransportResponse = {
|
|
141
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
142
|
+
headers: StealthTransportHeaders;
|
|
143
|
+
status: number;
|
|
144
|
+
body?: StealthTransportBody | null;
|
|
143
145
|
url?: string;
|
|
144
146
|
redirected?: boolean;
|
|
145
147
|
};
|
|
146
148
|
|
|
147
|
-
type StealthMethod =
|
|
148
|
-
type StealthRequestInit =
|
|
149
|
+
type StealthMethod = HttpMethod | "TRACE";
|
|
150
|
+
type StealthRequestInit = WreqRequestInit & {
|
|
149
151
|
redirect?: NonNullable<StealthFetchOptions["redirect"]>;
|
|
150
152
|
};
|
|
151
153
|
|
|
154
|
+
type WreqSessionCacheEntry = {
|
|
155
|
+
session: Promise<WreqSession>;
|
|
156
|
+
tail: Promise<void>;
|
|
157
|
+
};
|
|
158
|
+
|
|
152
159
|
function isRecord(value: unknown): value is Record<PropertyKey, unknown> {
|
|
153
160
|
return typeof value === "object" && value !== null;
|
|
154
161
|
}
|
|
@@ -294,65 +301,134 @@ class CookieJarImpl implements CookieJar {
|
|
|
294
301
|
}
|
|
295
302
|
}
|
|
296
303
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
304
|
+
type WreqModule = typeof import("wreq-js");
|
|
305
|
+
|
|
306
|
+
let wreqModulePromise: Promise<WreqModule> | undefined;
|
|
307
|
+
|
|
308
|
+
function getWreqModule(): Promise<WreqModule> {
|
|
309
|
+
if (!wreqModulePromise) {
|
|
310
|
+
wreqModulePromise = import("wreq-js").catch((error: unknown) => {
|
|
311
|
+
throw new SDKError(
|
|
312
|
+
`Stealth transport is unavailable on ${process.platform}-${process.arch}: the wreq-js native binary could not be loaded.`,
|
|
313
|
+
{
|
|
314
|
+
code: "stealth_transport_unavailable",
|
|
315
|
+
cause: error instanceof Error ? error : undefined,
|
|
316
|
+
},
|
|
317
|
+
);
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
return wreqModulePromise;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function parseProfileIdentifier(identifier: string): {
|
|
325
|
+
family: string;
|
|
326
|
+
version: number[];
|
|
327
|
+
} | null {
|
|
328
|
+
const match = /^(safari_ios|safari_ipad|firefox_android|firefox_private|chrome|edge|firefox|opera|safari|okhttp)_(\d+(?:[._]\d+)*)$/.exec(
|
|
329
|
+
identifier.toLowerCase(),
|
|
330
|
+
);
|
|
331
|
+
if (!match?.[1] || !match[2]) return null;
|
|
332
|
+
return {
|
|
333
|
+
family: match[1],
|
|
334
|
+
version: match[2].split(/[._]/).map(Number),
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function compareVersionDistance(target: number[], left: number[], right: number[]): number {
|
|
339
|
+
const width = Math.max(target.length, left.length, right.length);
|
|
340
|
+
for (let index = 0; index < width; index += 1) {
|
|
341
|
+
const targetPart = target[index] ?? 0;
|
|
342
|
+
const leftDistance = Math.abs((left[index] ?? 0) - targetPart);
|
|
343
|
+
const rightDistance = Math.abs((right[index] ?? 0) - targetPart);
|
|
344
|
+
if (leftDistance !== rightDistance) return leftDistance - rightDistance;
|
|
345
|
+
}
|
|
346
|
+
return 0;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function closestWreqProfile(
|
|
350
|
+
identifier: string,
|
|
351
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
352
|
+
): BrowserProfile | undefined {
|
|
353
|
+
const requested = parseProfileIdentifier(identifier);
|
|
354
|
+
if (!requested) return undefined;
|
|
355
|
+
|
|
356
|
+
let closest: { name: BrowserProfile; version: number[] } | undefined;
|
|
357
|
+
for (const candidateName of wreqProfiles) {
|
|
358
|
+
const candidate = parseProfileIdentifier(candidateName);
|
|
359
|
+
if (!candidate || candidate.family !== requested.family) continue;
|
|
305
360
|
if (
|
|
306
|
-
|
|
307
|
-
|
|
361
|
+
!closest ||
|
|
362
|
+
compareVersionDistance(requested.version, candidate.version, closest.version) < 0
|
|
308
363
|
) {
|
|
309
|
-
|
|
310
|
-
closestBrowser = browser;
|
|
364
|
+
closest = { name: candidateName, version: candidate.version };
|
|
311
365
|
}
|
|
312
366
|
}
|
|
313
|
-
return
|
|
367
|
+
return closest?.name;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function resolveDefaultWreqProfileMapping(): { identifier: string; os: EmulationOS } {
|
|
371
|
+
let profile: ReturnType<typeof getStealthProfile>;
|
|
372
|
+
try {
|
|
373
|
+
profile = getStealthProfile(DEFAULT_PROFILE);
|
|
374
|
+
} catch (error) {
|
|
375
|
+
throw new SDKError(
|
|
376
|
+
`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`,
|
|
377
|
+
{ cause: error instanceof Error ? error : undefined },
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
const identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
382
|
+
if (!parseProfileIdentifier(identifier)) {
|
|
383
|
+
throw new SDKError(
|
|
384
|
+
`Default stealth profile "${DEFAULT_PROFILE}" cannot be mapped to a wreq-js browser profile.`,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
return { identifier, os: profile.platform };
|
|
314
388
|
}
|
|
315
389
|
|
|
316
|
-
|
|
390
|
+
const DEFAULT_WREQ_PROFILE_MAPPING = resolveDefaultWreqProfileMapping();
|
|
391
|
+
|
|
392
|
+
export function resolveWreqProfile(
|
|
393
|
+
profileName: string,
|
|
394
|
+
wreqProfiles: readonly BrowserProfile[],
|
|
395
|
+
): {
|
|
396
|
+
browser: BrowserProfile;
|
|
397
|
+
os: EmulationOS;
|
|
398
|
+
} {
|
|
317
399
|
if (REMOVED_CHROME_PROFILE_NAMES.has(profileName)) {
|
|
318
400
|
throw new SDKError(`Unknown stealth profile: ${profileName}`);
|
|
319
401
|
}
|
|
320
402
|
|
|
321
|
-
let
|
|
403
|
+
let identifier: string;
|
|
404
|
+
let os: EmulationOS;
|
|
322
405
|
try {
|
|
323
|
-
profile = getStealthProfile(profileName);
|
|
406
|
+
const profile = getStealthProfile(profileName);
|
|
407
|
+
identifier = profile.tlsClientIdentifier?.toLowerCase() ?? "";
|
|
408
|
+
os = profile.platform;
|
|
324
409
|
} catch {
|
|
325
410
|
// Preserve the previous ctx.stealth.fetch() compatibility behavior: unknown
|
|
326
411
|
// profile strings still run with the transport default instead of failing
|
|
327
412
|
// before the request starts. Removed built-in profile aliases above remain
|
|
328
413
|
// explicit errors so callers do not accidentally pin retired fingerprints.
|
|
329
|
-
|
|
414
|
+
identifier = DEFAULT_WREQ_PROFILE_MAPPING.identifier;
|
|
415
|
+
os = DEFAULT_WREQ_PROFILE_MAPPING.os;
|
|
330
416
|
}
|
|
331
417
|
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
if (chromeMatch?.[1]) {
|
|
335
|
-
return closestImpitBrowser(Number(chromeMatch[1]), CHROME_IMPIT_BY_MAJOR);
|
|
336
|
-
}
|
|
337
|
-
const firefoxMatch = /^firefox_(\d+)/.exec(identifier);
|
|
338
|
-
if (firefoxMatch?.[1]) {
|
|
339
|
-
return closestImpitBrowser(Number(firefoxMatch[1]), FIREFOX_IMPIT_BY_MAJOR);
|
|
340
|
-
}
|
|
341
|
-
if (identifier.startsWith("safari_")) {
|
|
418
|
+
const browser = closestWreqProfile(identifier, wreqProfiles);
|
|
419
|
+
if (!browser) {
|
|
342
420
|
throw new SDKError(
|
|
343
|
-
`Stealth profile "${profileName}"
|
|
421
|
+
`Stealth profile "${profileName}" cannot be mapped to a wreq-js browser profile.`,
|
|
344
422
|
);
|
|
345
423
|
}
|
|
346
|
-
|
|
347
|
-
`Stealth profile "${profileName}" cannot be mapped to an impit browser profile.`,
|
|
348
|
-
);
|
|
424
|
+
return { browser, os };
|
|
349
425
|
}
|
|
350
426
|
|
|
351
427
|
function resolveUrl(baseUrl: string, url: string): string {
|
|
352
428
|
return new URL(url, baseUrl).toString();
|
|
353
429
|
}
|
|
354
430
|
|
|
355
|
-
function headerEntriesFromHeaders(headers:
|
|
431
|
+
function headerEntriesFromHeaders(headers: StealthTransportHeaders): [string, string][] {
|
|
356
432
|
return Array.from(headers.entries());
|
|
357
433
|
}
|
|
358
434
|
|
|
@@ -370,17 +446,6 @@ function normalizeHeaders(
|
|
|
370
446
|
function hasOwn(object: object, key: string): boolean {
|
|
371
447
|
return Object.hasOwn(object, key);
|
|
372
448
|
}
|
|
373
|
-
function toImpitCookieJar(cookieJar: CookieJarImpl): NonNullable<ImpitOptions["cookieJar"]> {
|
|
374
|
-
return {
|
|
375
|
-
setCookie(cookie: string, url: string, cb?: (error?: unknown) => void) {
|
|
376
|
-
cookieJar.setFromCookieStrings([cookie], url);
|
|
377
|
-
if (typeof cb === "function") cb();
|
|
378
|
-
},
|
|
379
|
-
getCookieString(url: string) {
|
|
380
|
-
return cookieJar.toHeader(url);
|
|
381
|
-
},
|
|
382
|
-
};
|
|
383
|
-
}
|
|
384
449
|
|
|
385
450
|
function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
386
451
|
if (!isRecord(options)) return;
|
|
@@ -392,17 +457,19 @@ function assertNoUnsupportedFingerprintOverrides(options: unknown): void {
|
|
|
392
457
|
if (unsupported.length === 0) return;
|
|
393
458
|
|
|
394
459
|
throw new SDKError(
|
|
395
|
-
`ctx.stealth.fetch uses
|
|
460
|
+
`ctx.stealth.fetch uses transport-managed browser fingerprints and no longer accepts low-level stealth overrides: ${unsupported.join(", ")}. Use the profile option instead.`,
|
|
396
461
|
);
|
|
397
462
|
}
|
|
398
463
|
|
|
399
|
-
function responseHeadersToRecord(
|
|
464
|
+
function responseHeadersToRecord(
|
|
465
|
+
headers: StealthTransportHeaders,
|
|
466
|
+
): Record<string, string | string[] | undefined> {
|
|
400
467
|
const record: Record<string, string> = {};
|
|
401
468
|
for (const [name, value] of headers.entries()) record[name] = value;
|
|
402
469
|
return record;
|
|
403
470
|
}
|
|
404
471
|
|
|
405
|
-
function setCookieHeadersFromResponse(headers:
|
|
472
|
+
function setCookieHeadersFromResponse(headers: StealthTransportHeaders): string[] {
|
|
406
473
|
const getSetCookie = headers.getSetCookie;
|
|
407
474
|
if (typeof getSetCookie === "function") return getSetCookie.call(headers);
|
|
408
475
|
const setCookie = headers.get("set-cookie");
|
|
@@ -479,32 +546,20 @@ function responseTooLargeError(maxBodyBytes: number, observedBytes: number): Tra
|
|
|
479
546
|
);
|
|
480
547
|
}
|
|
481
548
|
|
|
482
|
-
function declaredContentLength(headers:
|
|
549
|
+
function declaredContentLength(headers: StealthTransportHeaders): number | undefined {
|
|
483
550
|
const contentLength = headers.get("content-length")?.trim();
|
|
484
551
|
if (!contentLength || !/^\d+$/.test(contentLength)) return undefined;
|
|
485
552
|
const parsed = Number(contentLength);
|
|
486
553
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
487
554
|
}
|
|
488
555
|
|
|
489
|
-
function abortTransportResponse(response: StealthTransportResponse): boolean {
|
|
490
|
-
if (!response.abort) return false;
|
|
491
|
-
try {
|
|
492
|
-
response.abort();
|
|
493
|
-
} catch {
|
|
494
|
-
// The size error remains the primary failure if impit has already closed the response.
|
|
495
|
-
}
|
|
496
|
-
return true;
|
|
497
|
-
}
|
|
498
|
-
|
|
499
556
|
async function readResponseBodyWithLimit(
|
|
500
557
|
response: StealthTransportResponse,
|
|
501
558
|
maxBodyBytes: number,
|
|
502
559
|
): Promise<ArrayBuffer> {
|
|
503
560
|
const contentLength = declaredContentLength(response.headers);
|
|
504
561
|
if (contentLength !== undefined && contentLength > maxBodyBytes) {
|
|
505
|
-
|
|
506
|
-
await response.body?.cancel().catch(() => undefined);
|
|
507
|
-
}
|
|
562
|
+
await response.body?.cancel().catch(() => undefined);
|
|
508
563
|
throw responseTooLargeError(maxBodyBytes, contentLength);
|
|
509
564
|
}
|
|
510
565
|
|
|
@@ -523,10 +578,10 @@ async function readResponseBodyWithLimit(
|
|
|
523
578
|
while (true) {
|
|
524
579
|
const { done, value } = await reader.read();
|
|
525
580
|
if (done) break;
|
|
581
|
+
if (!value) continue;
|
|
526
582
|
receivedBytes += value.byteLength;
|
|
527
583
|
if (receivedBytes > maxBodyBytes) {
|
|
528
584
|
await reader.cancel().catch(() => undefined);
|
|
529
|
-
abortTransportResponse(response);
|
|
530
585
|
throw responseTooLargeError(maxBodyBytes, receivedBytes);
|
|
531
586
|
}
|
|
532
587
|
chunks.push(value);
|
|
@@ -743,40 +798,171 @@ function locationHeader(headers: Record<string, string>): string | undefined {
|
|
|
743
798
|
return undefined;
|
|
744
799
|
}
|
|
745
800
|
|
|
801
|
+
function withoutRedirectBodyHeaders(headers: Record<string, string>): Record<string, string> {
|
|
802
|
+
return Object.fromEntries(
|
|
803
|
+
Object.entries(headers).filter(([name]) => !REDIRECT_BODY_HEADERS.has(name.toLowerCase())),
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function assertStealthRedirectUrl(url: string): void {
|
|
808
|
+
const protocol = new URL(url).protocol;
|
|
809
|
+
if (protocol !== "http:" && protocol !== "https:") {
|
|
810
|
+
throw new TransportError(`Stealth redirect target scheme "${protocol}" is not allowed`, {
|
|
811
|
+
code: "transport_invalid_url",
|
|
812
|
+
status: 0,
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
function discardStealthRedirectBody(response: StealthTransportResponse): void {
|
|
818
|
+
try {
|
|
819
|
+
const cancellation = response.body?.cancel();
|
|
820
|
+
if (cancellation) void cancellation.catch(() => undefined);
|
|
821
|
+
} catch {
|
|
822
|
+
// Redirect handling is decided from status and headers. A cancellation
|
|
823
|
+
// failure must not replace or delay that decision.
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
async function fetchStealthRedirectChain(
|
|
828
|
+
transport: WreqSession,
|
|
829
|
+
cookieJar: CookieJarImpl,
|
|
830
|
+
requestUrl: string,
|
|
831
|
+
method: StealthMethod,
|
|
832
|
+
options: StealthFetchOptions,
|
|
833
|
+
): Promise<{ normalized: StealthResponse; response: StealthTransportResponse }> {
|
|
834
|
+
let currentUrl = requestUrl;
|
|
835
|
+
let currentMethod = method;
|
|
836
|
+
let currentBody = options.body === undefined ? undefined : normalizeBody(options.body);
|
|
837
|
+
let currentHeaders = { ...(options.headers ?? {}) };
|
|
838
|
+
let followedHops = 0;
|
|
839
|
+
let response: StealthTransportResponse;
|
|
840
|
+
const deadline = options.timeout ? performance.now() + options.timeout : undefined;
|
|
841
|
+
|
|
842
|
+
while (true) {
|
|
843
|
+
const headers = { ...currentHeaders };
|
|
844
|
+
if (!hasHeader(headers, "Cookie")) {
|
|
845
|
+
const cookieHeader = cookieJar.toHeader(currentUrl);
|
|
846
|
+
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
847
|
+
}
|
|
848
|
+
const requestInit: StealthRequestInit = {
|
|
849
|
+
headers: normalizeHeaders(headers),
|
|
850
|
+
method: currentMethod,
|
|
851
|
+
redirect: "manual",
|
|
852
|
+
};
|
|
853
|
+
if (currentBody !== undefined) requestInit.body = currentBody;
|
|
854
|
+
|
|
855
|
+
await transport.clearCookies();
|
|
856
|
+
const remainingTimeout =
|
|
857
|
+
deadline === undefined ? undefined : Math.ceil(deadline - performance.now());
|
|
858
|
+
if (remainingTimeout !== undefined && remainingTimeout <= 0) {
|
|
859
|
+
throw new TransportError("Request timed out", {
|
|
860
|
+
code: "transport_timeout",
|
|
861
|
+
status: 0,
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
if (remainingTimeout !== undefined) requestInit.timeout = remainingTimeout;
|
|
865
|
+
response = await transport.fetch(currentUrl, requestInit);
|
|
866
|
+
cookieJar.setFromCookieStrings(
|
|
867
|
+
setCookieHeadersFromResponse(response.headers),
|
|
868
|
+
response.url ?? currentUrl,
|
|
869
|
+
);
|
|
870
|
+
|
|
871
|
+
if (!isRedirectStatus(response.status) || options.redirect === "manual") break;
|
|
872
|
+
if (options.redirect === "error") {
|
|
873
|
+
discardStealthRedirectBody(response);
|
|
874
|
+
throw new TransportError("Stealth request encountered a redirect", {
|
|
875
|
+
code: "transport_network_error",
|
|
876
|
+
status: 0,
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
const nextUrl = resolveRedirectUrl(
|
|
881
|
+
response.headers.get("location") ?? undefined,
|
|
882
|
+
response.url ?? currentUrl,
|
|
883
|
+
);
|
|
884
|
+
if (!nextUrl) break;
|
|
885
|
+
if (followedHops >= MAX_STEALTH_REDIRECT_HOPS) {
|
|
886
|
+
discardStealthRedirectBody(response);
|
|
887
|
+
throw new TransportError(
|
|
888
|
+
`Stealth request exceeded the ${MAX_STEALTH_REDIRECT_HOPS}-redirect limit`,
|
|
889
|
+
{ code: "transport_network_error", status: 0 },
|
|
890
|
+
);
|
|
891
|
+
}
|
|
892
|
+
assertStealthRedirectUrl(nextUrl);
|
|
893
|
+
discardStealthRedirectBody(response);
|
|
894
|
+
const nextMethod = nextRedirectMethod(response.status, currentMethod);
|
|
895
|
+
if (nextMethod !== currentMethod) {
|
|
896
|
+
currentBody = undefined;
|
|
897
|
+
currentHeaders = withoutRedirectBodyHeaders(currentHeaders);
|
|
898
|
+
}
|
|
899
|
+
currentMethod = nextMethod;
|
|
900
|
+
currentUrl = nextUrl;
|
|
901
|
+
followedHops += 1;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
const normalized = await normalizeResponse(response, currentUrl, options.maxBodyBytes);
|
|
905
|
+
if (followedHops > 0) normalized.redirected = true;
|
|
906
|
+
return { normalized, response };
|
|
907
|
+
}
|
|
908
|
+
|
|
746
909
|
function createSessionFetcher(
|
|
747
910
|
baseUrl: string,
|
|
748
911
|
defaultProfile: string,
|
|
749
912
|
clientOptions: StealthClientOptions,
|
|
750
913
|
): StealthSession {
|
|
751
|
-
const clients = new Map<string,
|
|
914
|
+
const clients = new Map<string, WreqSessionCacheEntry>();
|
|
752
915
|
let closed = false;
|
|
753
916
|
let hasWarnedMissingProxy = false;
|
|
754
917
|
const warn = clientOptions.warn ?? console.warn;
|
|
755
918
|
const cookieJar = new CookieJarImpl([], baseUrl);
|
|
756
|
-
const impitCookieJar = toImpitCookieJar(cookieJar);
|
|
757
919
|
|
|
758
|
-
function
|
|
920
|
+
async function getClientEntry(
|
|
759
921
|
profileName: string,
|
|
760
922
|
proxyUrl: string | undefined,
|
|
761
923
|
ignoreTlsErrors: boolean,
|
|
762
|
-
):
|
|
924
|
+
): Promise<WreqSessionCacheEntry> {
|
|
763
925
|
if (closed) {
|
|
764
926
|
throw new TransportError("Stealth session is closed", { status: 0 });
|
|
765
927
|
}
|
|
766
|
-
const
|
|
928
|
+
const wreq = await getWreqModule();
|
|
929
|
+
const { browser, os } = resolveWreqProfile(profileName, wreq.getProfiles());
|
|
767
930
|
const cacheKey = JSON.stringify({ browser, proxyUrl, ignoreTlsErrors });
|
|
768
|
-
let
|
|
769
|
-
if (!
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
931
|
+
let entry = clients.get(cacheKey);
|
|
932
|
+
if (!entry) {
|
|
933
|
+
entry = {
|
|
934
|
+
session: wreq.createSession({
|
|
935
|
+
browser,
|
|
936
|
+
os,
|
|
937
|
+
...(proxyUrl ? { proxy: proxyUrl } : {}),
|
|
938
|
+
...(ignoreTlsErrors ? { insecure: true } : {}),
|
|
939
|
+
timeout: 30_000,
|
|
940
|
+
}),
|
|
941
|
+
tail: Promise.resolve(),
|
|
942
|
+
};
|
|
943
|
+
clients.set(cacheKey, entry);
|
|
778
944
|
}
|
|
779
|
-
return
|
|
945
|
+
return entry;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
async function withClient<T>(
|
|
949
|
+
profileName: string,
|
|
950
|
+
proxyUrl: string | undefined,
|
|
951
|
+
ignoreTlsErrors: boolean,
|
|
952
|
+
operation: (client: WreqSession) => Promise<T>,
|
|
953
|
+
): Promise<T> {
|
|
954
|
+
const entry = await getClientEntry(profileName, proxyUrl, ignoreTlsErrors);
|
|
955
|
+
const previous = entry.tail;
|
|
956
|
+
let release!: () => void;
|
|
957
|
+
entry.tail = new Promise<void>((resolve) => {
|
|
958
|
+
release = resolve;
|
|
959
|
+
});
|
|
960
|
+
await previous;
|
|
961
|
+
try {
|
|
962
|
+
return await operation(await entry.session);
|
|
963
|
+
} finally {
|
|
964
|
+
release();
|
|
965
|
+
}
|
|
780
966
|
}
|
|
781
967
|
|
|
782
968
|
async function resolveRequestProxy(
|
|
@@ -793,7 +979,7 @@ function createSessionFetcher(
|
|
|
793
979
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
794
980
|
retryAttemptOffset: proxyAttempt,
|
|
795
981
|
}),
|
|
796
|
-
// The
|
|
982
|
+
// The stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
797
983
|
// preserving the client TLS fingerprint end-to-end.
|
|
798
984
|
transportProtocols: ["http", "socks5"],
|
|
799
985
|
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
@@ -943,28 +1129,12 @@ function createSessionFetcher(
|
|
|
943
1129
|
sensitiveParams,
|
|
944
1130
|
);
|
|
945
1131
|
const { requestUrl } = serializedUrl;
|
|
946
|
-
const
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
headers: normalizeHeaders(headers),
|
|
953
|
-
method,
|
|
954
|
-
...(options.redirect ? { redirect: options.redirect } : {}),
|
|
955
|
-
...(options.timeout ? { timeout: options.timeout } : {}),
|
|
956
|
-
};
|
|
957
|
-
if (options.body !== undefined) {
|
|
958
|
-
requestInit.body = normalizeBody(options.body);
|
|
959
|
-
}
|
|
960
|
-
const response = await getClient(profileName, proxy, ignoreTlsErrors).fetch(
|
|
961
|
-
requestUrl,
|
|
962
|
-
requestInit,
|
|
963
|
-
);
|
|
964
|
-
const normalized = await normalizeResponse(response, requestUrl, options.maxBodyBytes);
|
|
965
|
-
cookieJar.setFromCookieStrings(
|
|
966
|
-
setCookieHeadersFromResponse(response.headers),
|
|
967
|
-
response.url ?? requestUrl,
|
|
1132
|
+
const { normalized, response } = await withClient(
|
|
1133
|
+
profileName,
|
|
1134
|
+
proxy,
|
|
1135
|
+
ignoreTlsErrors,
|
|
1136
|
+
(transport) =>
|
|
1137
|
+
fetchStealthRedirectChain(transport, cookieJar, requestUrl, method, options),
|
|
968
1138
|
);
|
|
969
1139
|
|
|
970
1140
|
if (proxy && isProxyConnectFailureResponse(response, normalized.body)) {
|
|
@@ -1335,6 +1505,14 @@ function createSessionFetcher(
|
|
|
1335
1505
|
},
|
|
1336
1506
|
close() {
|
|
1337
1507
|
closed = true;
|
|
1508
|
+
for (const client of clients.values()) {
|
|
1509
|
+
void client.session
|
|
1510
|
+
.then((session) => session.close())
|
|
1511
|
+
.catch((error: unknown) => {
|
|
1512
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1513
|
+
warn(`[provider-sdk] Failed to close stealth transport session: ${message}`);
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1338
1516
|
clients.clear();
|
|
1339
1517
|
},
|
|
1340
1518
|
};
|
|
@@ -1345,12 +1523,15 @@ function createSessionFetcher(
|
|
|
1345
1523
|
proxy: string,
|
|
1346
1524
|
): Promise<"source_ip_denied" | "edge_auth_rejected" | undefined> {
|
|
1347
1525
|
try {
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1526
|
+
return await withClient(profileName, proxy, false, async (client) => {
|
|
1527
|
+
await client.clearCookies();
|
|
1528
|
+
const response = await client.fetch(PROXY_AUTH_DIAGNOSTIC_URL, {
|
|
1529
|
+
method: "GET",
|
|
1530
|
+
timeout: PROXY_AUTH_DIAGNOSTIC_TIMEOUT_MS,
|
|
1531
|
+
});
|
|
1532
|
+
const normalized = await normalizeResponse(response);
|
|
1533
|
+
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1351
1534
|
});
|
|
1352
|
-
const normalized = await normalizeResponse(response);
|
|
1353
|
-
return classifyProxyAuthDiagnosticMessage(normalized.body);
|
|
1354
1535
|
} catch (error) {
|
|
1355
1536
|
const message =
|
|
1356
1537
|
error instanceof Error
|